Java: Hadoop文件系统的读写操作

所需jar包路径：

hadoop-2.8.5/share/hadoop/common
hadoop-2.8.5/share/hadoop/common/bin

hadoop-2.8.5/share/hadoop/hdfs
hadoop-2.8.5/share/hadoop/hdfs/bin

  
 

  
1
  
2
  
3
  
4
  
5
 

java代码实例

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * HDFS操作，需要导入common和hdsf文件夹下的jar
 */
public class HDSFDemo { private FileSystem fs = null; HDSFDemo() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); fs = FileSystem.get(new URI("hdfs://master:9000"), conf, "root"); } /** * 下载文件 * * @throws IOException */ public void download() throws IOException { InputStream in = fs.open(new Path("/java")); OutputStream out = new FileOutputStream("/Users/Desktop"); IOUtils.copyBytes(in, out, 4096, true); } /** * 上传文件 * * @throws IOException */ public void upload() throws IOException { System.out.println("开始上传..."); InputStream in = new FileInputStream("/Users/Desktop/compare.py"); OutputStream out = this.fs.create(new Path("/compare.py")); IOUtils.copyBytes(in, out, 4096, true); } /** * 删除文件 * * @throws IOException */ public void delete() throws IOException { boolean ret = this.fs.delete(new Path("/java"), true); System.out.println("文件删除结果：" + ret); } /** * 创建文件夹 * * @throws IOException */ public void mkdir() throws IOException { boolean ret = this.fs.mkdirs(new Path("/temp")); System.out.println("文件创建成功：" + ret); } public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException { HDSFDemo hdfs = new HDSFDemo(); hdfs.upload(); }
}
  
 

  
1
  
2
  
3
  
4
  
5
  
6
  
7
  
8
  
9
  
10
  
11
  
12
  
13
  
14
  
15
  
16
  
17
  
18
  
19
  
20
  
21
  
22
  
23
  
24
  
25
  
26
  
27
  
28
  
29
  
30
  
31
  
32
  
33
  
34
  
35
  
36
  
37
  
38
  
39
  
40
  
41
  
42
  
43
  
44
  
45
  
46
  
47
  
48
  
49
  
50
  
51
  
52
  
53
  
54
  
55
  
56
  
57
  
58
  
59
  
60
  
61
  
62
  
63
  
64
  
65
  
66
  
67
  
68
  
69
  
70
  
71
  
72
  
73
  
74
 

备注：
URI的端口号一定要和配置文件中的端口号一致

问题：

log4j:WARN Please initialize the log4j system properly

可参考：log4j:WARN Please initialize the log4j system properly 问题解决

文章来源: pengshiyu.blog.csdn.net，作者：彭世瑜，版权归原作者所有，如需转载，请联系作者。

原文链接：pengshiyu.blog.csdn.net/article/details/84405252