javaAPI操作HDFS

/**
 * javaAPI操作HDFS客户端
 */
public class test {
    public static final String  HDFS_PATH="hdfs://10.23.12.188:8020";

    FileSystem fileSystem = null;
    Configuration configuration = null;
    @Before
    public void setUp() throws  Exception{
        System.out.println("HDFS_PATH setup");
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"zikang");
    }

    //在HDFS上创建文件
    @Test
    public void getInstance()  throws  Exception {
        fileSystem.mkdirs(new Path("/javaApi"));
    }

    //往HDFS上面的文件写入字符
    @Test
    public void create()  throws  Exception{
        FSDataOutputStream output  = fileSystem.create(new Path("/javaApi/test.txt"));
        output.write("hello world".getBytes());
        output.flush();
        output.close();
    }
    //查看HDFS上面文件的内容
    @Test
    public void cat() throws Exception{
        FSDataInputStream in = fileSystem.open(new Path("/javaApi/test.txt"));
        IOUtils.copyBytes(in ,System.out,1024);
        in.close();
    }

    //对HDFS上的文件进行重命名
    @Test
    public void rename() throws Exception{
        Path oldPath = new Path("/javaApi/test.txt");
        Path newPath = new Path("/javaApi/change.txt");
        fileSystem.rename(oldPath,newPath);
    }

    //从本地文件复制到HDFS
    @Test
    public void copyFromLocalFile()  throws Exception{
        Path localPath = new Path("");
        Path hdfsPath = new Path("");
        fileSystem.copyFromLocalFile(localPath,hdfsPath);
    }

    //大文件上传到HDFS时候输出。。作为进度
    @Test
    public  void copyFromLocalFileWithProgress()  throws Exception{
        InputStream in = new BufferedInputStream(
                  new FileInputStream(
                           new File("")));

        FSDataOutputStream output = fileSystem.create(new Path(""),
                new Progressable() {
                    public void progress() {
                        System.out.println("..");
                    }
                });
    }

    //下载HDFS文件
    @Test
    public  void copyToLocalFile() throws Exception{
        Path oldPath = new Path("/javaApi/test.txt");
        Path newPath = new Path("/javaApi/change.txt");
        fileSystem.copyToLocalFile(oldPath,newPath);
    }

    /**
     * 查看目录下的所有文件
     * 注：虽然设置了副本的系数为3，但是通过HDFS shell的方式put上的，那么
     * 将采用默认的副本系数1，如果是java API的话那么就是默认的3
     */
    @Test
    public void listFiles() throws Exception{
        //文件 3 11 hdfs://10.23.12.188:8020/javaApi/change.txt
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/javaApi"));
        for(FileStatus fileStatus : fileStatuses){
            String isDir = fileStatus.isDirectory()?"文件夹":"文件";
            short replication = fileStatus.getReplication();
            long len = fileStatus.getLen();
            String path = fileStatus.getPath().toString();
            System.out.println(isDir+" "+ replication+ " "+len+" "+path);
        }
    }
    //删除HDFS上面的文件
    @Test
    public  void  delete()  throws Exception{
        fileSystem.delete(new Path(""),true);
    }

    @After
    public  void tearDown() throws  Exception{
        System.out.println("HDFS_PATH tearDown");
        configuration = null;
        fileSystem = null;

    }