(1). 命令参考地址
https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/FileSystemShell.html
(2). HDFS API(pom.xml)
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>help.lixin.hadoop.example</groupId>
<artifactId>hadoop-example</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>central</id>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<layout>default</layout>
<!-- 是否开启发布版构件下载 -->
<releases>
<enabled>true</enabled>
</releases>
<!-- 是否开启快照版构件下载 -->
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.7.5</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<!-- 指定maven编译的jdk版本,如果不指定,maven3默认用jdk 1.5 maven2默认用jdk1.3 -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<!-- 一般而言，target与source是保持一致的，但是，有时候为了让程序能在其他版本的jdk中运行(对于低版本目标jdk，源代码中不能使用低版本jdk中不支持的语法)，会存在target不同于source的情况 -->
<source>1.8</source> <!-- 源代码使用的JDK版本 -->
<target>1.8</target> <!-- 需要生成的目标class文件的编译版本 -->
<encoding>UTF-8</encoding><!-- 字符集编码 -->
<verbose>true</verbose>
<showWarnings>true</showWarnings>
<fork>true</fork><!-- 要使compilerVersion标签生效，还需要将fork设为true，用于明确表示编译版本配置的可用 -->
<executable><!-- path-to-javac --></executable><!-- 使用指定的javac命令，例如：<executable>${JAVA_1_4_HOME}/bin/javac</executable> -->
<compilerVersion>1.3</compilerVersion><!-- 指定插件将使用的编译器的版本 -->
<meminitial>128m</meminitial><!-- 编译器使用的初始内存 -->
<maxmem>512m</maxmem><!-- 编译器使用的最大内存 -->
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<transformers>
<!--
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.demo.App</mainClass>
</transformer>
-->
</transformers>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
(3). HDFS API
package help.lixin.hdfs.example;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URL;
public class HdfsTest {
private static FileSystem fileSystem = null;
@BeforeClass
public static void init() throws Exception {
fileSystem = FileSystem.get(new URI("hdfs://localhost:9000"), new Configuration());
}
public static void close() throws Exception {
if (null != fileSystem) {
fileSystem.close();
}
}
@Test
public void testGetFile() throws Exception {
// 注册URL工厂
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
// 获得输入流
InputStream inputStream = new URL("hdfs://loclahost:9000/query-log/SogouQ2.txt").openStream();
OutputStream outputStream = new FileOutputStream(new File("/Users/lixin/IDEAWorkspace/hadoop-example/target/SogouQ2.txt"));
IOUtils.copy(inputStream, outputStream);
IOUtils.closeQuietly(inputStream);
IOUtils.closeQuietly(outputStream);
}
@Test
public void testEachFiles() throws Exception {
RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/"), true);
while (locatedFileStatusRemoteIterator.hasNext()) {
LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
System.out.println(next.getPath().toString());
}
}
@Test
public void testMkDir() throws Exception {
boolean mkdirs = fileSystem.mkdirs(new Path("/logs"));
if (mkdirs) {
System.out.println("create dir success");
} else {
System.out.println("create dir fail");
}
}
@Test
public void testUploadFile() throws Exception {
Path logs = new Path("/logs");
if (!fileSystem.exists(logs)) {
fileSystem.mkdirs(logs);
}
Path src = new Path("/Users/lixin/IDEAWorkspace/hadoop-example/target/SogouQ2.txt");
fileSystem.copyFromLocalFile(src, logs);
System.out.println("copy local file to hdfs success");
}
@Test
public void testGetFile2() throws Exception {
Path src = new Path("/logs/SogouQ2.txt");
Path dest = new Path("/Users/lixin/IDEAWorkspace/hadoop-example/target/SogouQ3.txt");
fileSystem.copyToLocalFile(src, dest);
// 打开文件并转换成流
FSDataInputStream dataInputStream = fileSystem.open(src);
FileOutputStream outputStream = new FileOutputStream(new File("/Users/lixin/IDEAWorkspace/hadoop-example/target/SogouQ4.txt"));
IOUtils.copy(dataInputStream, outputStream);
IOUtils.closeQuietly(dataInputStream);
IOUtils.closeQuietly(outputStream);
System.out.println("copy remote file to local success");
}
// 测试文件拷贝之前,先修改权限:
// hdfs dfs -chmod 600 /logs/SogouQ2.txt
@Test
public void testPermission() throws Exception {
FileSystem fileSystem2 = FileSystem.get(
// HDFS URL
new URI("hdfs://localhost:9000"),
// Configuration
new Configuration(),
// ********************执行任务时,可以模拟哪个用户去运行*****************
"test");
Path src = new Path("/logs/SogouQ2.txt");
// 打开文件并转换成流
FSDataInputStream dataInputStream = fileSystem2.open(src);
// 抛出如下异常
// org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ, inode="/logs/SogouQ2.txt":lixin:supergroup:-rw-------
FileOutputStream outputStream = new FileOutputStream(new File("/Users/lixin/IDEAWorkspace/hadoop-example/target/SogouQ4.txt"));
IOUtils.copy(dataInputStream, outputStream);
IOUtils.closeQuietly(dataInputStream);
IOUtils.closeQuietly(outputStream);
System.out.println("test testPermission copy remote file to local success");
}
}
