java1 6 years ago
parent
commit
400a7cb9a1
2 changed files with 19 additions and 5 deletions
  1. 3 3
      pom.xml
  2. 16 2
      src/main/java/com/ekexiu/project/hdfs/HdfsService.java

+ 3 - 3
pom.xml

10
    <packaging>war</packaging>
10
    <packaging>war</packaging>
11
    <properties>
11
    <properties>
12
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
12
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
13
        <maven.compiler.source>1.8</maven.compiler.source>
14
        <maven.compiler.target>1.8</maven.compiler.target>
13
        <maven.compiler.source>1.7</maven.compiler.source>
14
        <maven.compiler.target>1.7</maven.compiler.target>
15
        <maven.war.filteringDeploymentDescriptors>true</maven.war.filteringDeploymentDescriptors>
15
        <maven.war.filteringDeploymentDescriptors>true</maven.war.filteringDeploymentDescriptors>
16
        <maven.tomcat.path>/</maven.tomcat.path>
16
        <maven.tomcat.path>/</maven.tomcat.path>
17
    </properties>
17
    </properties>
73
		<dependency>
73
		<dependency>
74
			<groupId>org.apache.hadoop</groupId>
74
			<groupId>org.apache.hadoop</groupId>
75
			<artifactId>hadoop-client</artifactId>
75
			<artifactId>hadoop-client</artifactId>
76
			<version>3.0.3</version>
76
			<version>2.9.2</version>
77
		</dependency>
77
		</dependency>
78
<!-- 		<dependency>
78
<!-- 		<dependency>
79
			<groupId>org.apache.hadoop</groupId>
79
			<groupId>org.apache.hadoop</groupId>

+ 16 - 2
src/main/java/com/ekexiu/project/hdfs/HdfsService.java

29
		System.setProperty("HADOOP_USER_NAME", "hadoop");
29
		System.setProperty("HADOOP_USER_NAME", "hadoop");
30
		this.remoteUrl = remoteUrl;
30
		this.remoteUrl = remoteUrl;
31
		configuration.set("fs.default.name", this.remoteUrl);
31
		configuration.set("fs.default.name", this.remoteUrl);
32
		configuration.set("dfs.client.use.datanode.hostname", "true");
32
		try {
33
		try {
33
			dfs = (DistributedFileSystem) FileSystem.get(configuration);
34
			dfs = (DistributedFileSystem) FileSystem.get(configuration);
34
		} catch (IOException e) {
35
		} catch (IOException e) {
37
	}
38
	}
38
	
39
	
39
	public void upload(String path,File src) throws IOException{
40
	public void upload(String path,File src) throws IOException{
41
//		byte[] buffer = new byte[8192];
40
		Path dstPath = new Path(root,path);
42
		Path dstPath = new Path(root,path);
41
		if(src.exists() && src.isDirectory()){
43
		if(src.exists() && src.isDirectory()){
42
			if(!this.dfs.exists(dstPath)){
44
			if(!this.dfs.exists(dstPath)){
53
						throw new IOException("delete "+configuration.get("fs.default.name")+"/"+path+"/"+file.getName()+" error");
55
						throw new IOException("delete "+configuration.get("fs.default.name")+"/"+path+"/"+file.getName()+" error");
54
					}
56
					}
55
				}
57
				}
56
				this.dfs.copyFromLocalFile(new Path(src.getAbsolutePath()), dst);
58
				this.dfs.copyFromLocalFile(new Path(file.getAbsolutePath()), dst);
59
				
60
//				FSDataOutputStream outStream = dfs.create(dst);
61
//				try {
62
//					InputStream in = new FileInputStream(file);
63
//					try {
64
//						IoUtil.copy(in, outStream, buffer);
65
//					}finally {
66
//						in.close();
67
//					}
68
//				}finally {
69
//					outStream.close();
70
//				}
57
			}
71
			}
58
		}
72
		}
59
		
73
		
63
	public static void main(String args[]) throws Exception{
77
	public static void main(String args[]) throws Exception{
64
		HdfsService service = new HdfsService();
78
		HdfsService service = new HdfsService();
65
		service.setRemoteUrl("hdfs://39.97.161.48:9000");
79
		service.setRemoteUrl("hdfs://39.97.161.48:9000");
66
		service.upload("25",new File("/ekexiu/storage_asd/task/result/test/data_desen"));
80
		service.upload("22",new File("/home/java1/test"));
67
		
81
		
68
		System.out.println("11111111111111111111111111111111");
82
		System.out.println("11111111111111111111111111111111");
69
	}
83
	}