Hadoop之HDFS04【JavaAPI操作】
【摘要】
前面项目中我们是创建的java项目来演示的，但是hadoop相关的依赖太多了，不方便，本文通过maven项目来演示HDFS的java API操作
创建maven项目
相关的依赖
<depe...
前面项目中我们是创建的java项目来演示的，但是hadoop相关的依赖太多了，不方便，本文通过maven项目来演示HDFS的java API操作
创建maven项目
相关的依赖
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.5.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
</dependencies>
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
CRUD 操作
package com.sxt.test;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.Iterator;
import java.util.Map.Entry;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
public class HdfsTest {
FileSystem fs = null;
Configuration conf = null;
@Before
public void test() throws Exception {
// 1.加载配置文件
conf = new Configuration(true);
// 2.获取FileSystem对象
fs = FileSystem.get(new URI("hdfs://hadoop-node01:9000"),conf,"root");
}
/**
* 上传文件
* @throws IOException
* @throws IllegalArgumentException
*/
@Test
public void updateFile() throws IllegalArgumentException, IOException{
fs.copyFromLocalFile(true, true, new Path("c:/unintall.log"), new Path("/"));
fs.close();
}
/**
* 下载文件
* @throws Exception
*/
@Test
public void testDownload() throws Exception {
fs.copyToLocalFile(new Path("/a.txt"), new Path("c:/tools"));
fs.close();
}
/**
* 打印参数
*/
@Test
public void testConf(){
Iterator<Entry<String, String>> it = conf.iterator();
while(it.hasNext()){
Entry<String, String> ent = it.next();
System.out.println(ent.getKey() + " : " + ent.getValue());
}
}
/**
* 创建文件夹
* @throws Exception
*/
@Test
public void testMkdir() throws Exception {
boolean mkdirs = fs.mkdirs(new Path("/testmkdir/aaa/bbb"));
System.out.println(mkdirs);
}
/**
* 删除文件夹
* @throws Exception
*/
@Test
public void testDelete() throws Exception {
// recursive 是否递归删除
boolean flag = fs.delete(new Path("/testmkdir/aaa"), true);
System.out.println(flag);
}
/**
* 递归列出指定目录下所有子文件夹中的文件
* @throws Exception
*/
@Test
public void testLs() throws Exception {
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
while(listFiles.hasNext()){
LocatedFileStatus fileStatus = listFiles.next();
System.out.println("blocksize: " +fileStatus.getBlockSize());
System.out.println("owner: " +fileStatus.getOwner());
System.out.println("Replication: " +fileStatus.getReplication());
System.out.println("Permission: " +fileStatus.getPermission());
System.out.println("Name: " +fileStatus.getPath().getName());
System.out.println("------------------");
BlockLocation[] blockLocations = fileStatus.getBlockLocations();
for(BlockLocation b:blockLocations){
System.out.println("块起始偏移量: " +b.getOffset());
System.out.println("块长度:" + b.getLength());
//块所在的datanode节点
String[] datanodes = b.getHosts();
for(String dn:datanodes){
System.out.println("datanode:" + dn);
}
}
}
}
/**
* 获取文件的类型
* @throws Exception
*/
@Test
public void testLs2() throws Exception {
FileStatus[] listStatus = fs.listStatus(new Path("/"));
for(FileStatus file :listStatus){
System.out.println("name: " + file.getPath().getName());
System.out.println((file.isFile()?"file":"directory"));
}
}
/**
* 通过流的方式将文件上传到HDFS系统中
* @throws Exception
* @throws IOException
*/
@Test
public void testStreamUpload() throws Exception, IOException{
// 输出流
FSDataOutputStream out = fs.create(new Path("/dpb.txt"), true);
// 字节输入流
InputStream in = new FileInputStream("c:/tools/aaaaa.txt");
IOUtils.copy(in, out);
in.close();
out.close();
}
/**
* 通过流的方式获取HDFS上的数据
* @throws Exception
* @throws IllegalArgumentException
*/
@Test
public void testStreamDownload() throws IllegalArgumentException, Exception{
FSDataInputStream in = fs.open(new Path("/dpb.txt"));
OutputStream out = new FileOutputStream("c:/tools/a1.txt");
IOUtils.copy(in, out);
out.close();
in.close();
}
/**
* 获取HDFS中的文件的一部分内容
* @throws Exception
* @throws IllegalArgumentException
*/
@Test
public void testRandomAccess() throws IllegalArgumentException, Exception{
FSDataInputStream in = fs.open(new Path("/dpb.txt"));
in.seek(4);
OutputStream out = new FileOutputStream("c:/tools/a2.txt");
IOUtils.copy(in, out);
out.close();
in.close();
}
/**
* 显示HDFS上文件的内容
* @throws IOException
* @throws IllegalArgumentException
*/
@Test
public void testCat() throws IllegalArgumentException, IOException{
FSDataInputStream in = fs.open(new Path("/dpb.txt"));
// 将输出的目的地执行控制台即可~
IOUtils.copy(in, System.out);
}
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
- 126
- 127
- 128
- 129
- 130
- 131
- 132
- 133
- 134
- 135
- 136
- 137
- 138
- 139
- 140
- 141
- 142
- 143
- 144
- 145
- 146
- 147
- 148
- 149
- 150
- 151
- 152
- 153
- 154
- 155
- 156
- 157
- 158
- 159
- 160
- 161
- 162
- 163
- 164
- 165
- 166
- 167
- 168
- 169
- 170
- 171
- 172
- 173
- 174
- 175
- 176
- 177
- 178
- 179
- 180
- 181
- 182
- 183
- 184
- 185
- 186
- 187
- 188
- 189
- 190
- 191
- 192
- 193
- 194
- 195
只放出最后一个方法的输出。其他方法执行效果请各自尝试
文章来源: dpb-bobokaoya-sm.blog.csdn.net，作者：波波烤鸭，版权归原作者所有，如需转载，请联系作者。
原文链接：dpb-bobokaoya-sm.blog.csdn.net/article/details/88981296
【版权声明】本文为华为云社区用户转载文章，如果您发现本社区中有涉嫌抄袭的内容，欢迎发送邮件进行举报，并提供相关证据，一经查实，本社区将立刻删除涉嫌侵权内容，举报邮箱：
cloudbbs@huaweicloud.com
- 点赞
- 收藏
- 关注作者
评论（0）