HDFS 单节点环境搭建

2022-10-31  本文已影响0人  wayyyy

HDFS 作为hadoop 生态的一部分，部署HDFS即需要部署Hadoop。

这里由于节点数量有限，所以我们选择单节点（伪分布式部署）。

部署HDFS3.2.2
使用java-api 操作hdfs
  1. 首先需要配置 window 的 hadoop 环境，过程很多，可以参考：https://github.com/autopear/Intellij-Hadoop
  1. 建立工程，代码如下：
package edu.ucr.cs.merlin;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.log4j.BasicConfigurator;
import org.junit.Before;
import org.junit.Test;


public class HDFSDemo {

    FileSystem fs = null;

    @Before
    public void init() throws Exception {
        fs = FileSystem.get(new URI("hdfs://192.168.48.140:9000"), new Configuration(), "root");
    }

    @Test
    public void testUpload() throws Exception {
        InputStream in = new FileInputStream("/root/install.log");
        OutputStream out = fs.create(new Path("/log123.log"));
        IOUtils.copyBytes(in, out, 1024, true);
    }

    @Test
    public void testMkdir() throws IllegalArgumentException, IOException {
        boolean flag = fs.mkdirs(new Path("/a/aa"));
        System.out.println(flag);
    }

    @Test
    public void testDel() throws IllegalArgumentException, IOException {
        boolean flag = fs.delete(new Path("/a"), true);
        System.out.println(flag);
    }

    public static void main(String[] args) throws Exception {
        BasicConfigurator.configure();

        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.48.140:9000"), new Configuration(), "root");
//        InputStream in = fs.open(new Path("/jdk"));
//        OutputStream out = new FileOutputStream("/home/jdk1.7.tar.gz");
//        IOUtils.copyBytes(in, out, 4096, true);
        FSDataOutputStream fsout = fs.create(new Path("/test2.txt"));
        fsout.write("hello world".getBytes());
    }
}

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>wordcount</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <hadoop.version>3.2.2</hadoop.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13.2</version>
            <scope>compile</scope>
        </dependency>
    </dependencies>

    <repositories>
        <repository>
            <id>apache</id>
            <url>http://maven.apache.org</url>
        </repository>
    </repositories>

</project>

log4j.properties 如下：

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
hdfs 常用命令

显示当前目录结构

hadoop fs -ls  <path>

# 递归显示当前目录结构
hadoop fs -ls  -R  <path>

创建目录

hadoop fs -mkdir  <path> 

# 递归创建目录
hadoop fs -mkdir -p  <path>

删除操作

# 删除文件
hadoop fs -rm  <path>

# 递归删除目录和文件
hadoop fs -rm -R  <path>

本地加载文件到HDFS

# 二选一执行即可
hadoop fs -put  [localsrc] [dst] 
hadoop fs - copyFromLocal [localsrc] [dst]

参考资料
1、https://blog.csdn.net/y5201h/article/details/120024999
2、https://blog.csdn.net/S1124654/article/details/125522613
3、https://blog.csdn.net/S1124654/article/details/125522613

上一篇下一篇

猜你喜欢

热点阅读