Hadoop 安装配置

2016-12-20 本文已影响0人 StonyBlue

1.创建用户组和用户

groupadd hadoops  
useradd hadoop -g hadoops
passwd hadoop

2.解压jdk和hadoop

tar -zxvf jdk1.7.0_80.tar.gz -C /usr/local
mv /usr/local/jdk1.7.0_80 /usr/local/java

tar -zxvf hadoop-2.7.3.tar.gz -C /u01
mv /u01/hadoop-2.7.3  /u01/hadoop

chown -R hadoop:hadoop /u01/hadoop/etc/hadoop/**
chown -R hadoop:hadoop /u01/hadoop/bin/**
chown -R hadoop:hadoop /u01/hadoop/sbin/**

3.配置环境变量

vi /etc/profile

export HADOOP_HOME=/u01/hadoop
export PATH=$HADOOP_HOME/bin:$PATH
export JAVA_HOME=/usr/local/java
export PATH=$JAVA_HOME/bin:$PATH

source /etc/profile

4.配置hosts

vi /etc/hosts

192.168.0.9 hadmaster
192.168.0.27 hadslave1
192.168.0.28 hadslave2

5.创建目录

mkdri /u01/hadoopData
chown hadoop:hadoop /u01/hadoopData

vi /u01/hadoop/etc/hadoop/slaves

hadslave1
hadslave2

vi /u01/hadoop/etc/hadoop/masters
hadmaster

6.配置hadoop

6.1.编辑hadoop-env.sh

vi /u01/hadoop/etc/hadoop/hadoop-env.sh

export JAVA_HOME=/usr/local/java
export HADOOP_OPTS=-Djava.net.preferIPv4Stack=true

6.2.编辑core-site.xml

vi /u01/hadoop/etc/hadoop/core-site.xml

<property>
    <name>hadoop.tmp.dir</name>
    <value>/u01/hadoopData/tmp</value>
</property>
<property>
        <name>fs.default.name</name>
        <value>hdfs://hadmaster:9000</value>
</property>
<property>
    <name>fs.defaultFS</name>
    <value>hdfs://hadmaster:9000</value>
</property>
<property>  
    <name>hadoop.proxyuser.hadoop.hosts</name>
    <value>*</value>  
</property>  
<property>  
    <name>hadoop.proxyuser.hadoop.groups</name>  
    <value>*</value>  
</property>

6.3.编辑hdfs-site.xml

vi /u01/hadoop/etc/hadoop/hdfs-site.xml

<property>
    <name>dfs.replication</name>
    <value>2</value>
</property>
<property>
    <name>dfs.namenode.data.dir</name>
    <value>file:/u01/hadoopData/data,file:/u01/hadoopData/data2</value>
</property>
<property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/u01/hadoopData/name,file:/u01/hadoopData/name2</value>
</property>

6.4.编辑mapred-site.xml

cp /u01/hadoop/etc/hadoop/mapred-site.xml.template  /u01/hadoop/etc/hadoop/mapred-site.xml
vi /u01/hadoop/etc/hadoop/mapred-site.xml

<property>
    <name>mapred.job.tracker</name>
    <value>hadmaster:9001</value>
</property>
<property>
    <name>mapred.local.dir</name>
    <value>/u01/hadoopData/tmp</value>
</property>

6.5.编辑yarn-site.xml

vi /u01/hadoop/etc/hadoop/mapred-site.xml

<property>
    <name>yarn.resourcemanager.resource-tracker.address</name>
    <value>hadmaster:8025</value>
</property>
<property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>hadmaster:8035</value>
</property>
<property>
        <name>yarn.resourcemanager.address</name>
        <value>hadmaster:8050</value>
</property>
<property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>hadmaster:8058</value>
</property>

7.ssh免密码登陆

su - hadoop
cd /home/hadoop
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cp id_rsa.pub authorized_keys
cd /home/hadoop/.ssh
scp authorized_keys hadslave1:/home/hadoop/.ssh
scp authorized_keys hadslave2:/home/hadoop/.ssh
ssh hadmaster 
exit
ssh hadslave1
exit
ssh hadslave2
exit
rsync -avxP /u01/hadoop/ hadoop@hadslave1:/u01/hadoop/
rsync -avxP /u01/hadoop/ hadoop@hadslave2:/u01/hadoop/
rsync -avxP /u01/hadoop/etc/hadoop/ hadoop@hadslave1:/u01/hadoop/etc/hadoop/
rsync -avxP /u01/hadoop/etc/hadoop/ hadoop@hadslave2:/u01/hadoop/etc/hadoop/

8.hive元数据

create database hive_meta; 
create user 'hive' identified by 'hive'; 
grant all privileges on hive_meta.* to
'hive'@'5'
identified by 'hive' WITH GRANT OPTION; 
flush privileges;

9.hive环境变量

tar -zxvf apache-hive-2.1.0-bin.tar.gz -C /u01
mv /u01/apache-hive-2.1.0-bin /u01/hive
chown hadoop:hadoop  /u01/hive
vi /etc/profile
export HIVE_HOME=/u01/hive
export PATH=$PATH:$HIVE_HOME/bin
source /etc/profile

10.hive配置

10.1.copy配置文件

cd /u01/hive/conf
cp hive-env.sh.template hive-env.sh
cp hive-default.xml.template hive-site.xml  
cp hive-log4j2.properties.template hive-log4j2.properties
cp hive-exec-log4j2.properties.template hive-exec-log4j2.properties

10.2.配置hive-env.sh

vi hive-env.sh
export JAVA_HOME=/usr/local/java
export HADOOP_HOME=/u01/hadoop
export HIVE_HOME=/u01/hive
export HIVE_CONF_DIR=/u01/hive/conf

10.3.Mysql驱动

cp  mysql-connector-java-5.1.39.jar $HIVE_HOME/lib
chown hadoop:hadoop $HIVE_HOME/lib/**

10.4.配置hive-site.xml

##替换hive-site.xml文件中的 ${system:java.io.tmpdir} 和 ${system:user.name}
<property>
    <name>hive.exec.scratchdir</name>
    <value>/tmp/hive-${user.name}</value>
    <description>HDFS root scratch dir for Hive jobs which gets created with write all (733) permission. For each
        connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/<username> is created, with
        ${hive.scratch.dir.permission}.
    </description>
</property>
<property>
    <name>hive.exec.local.scratchdir</name>
    <value>/tmp/${user.name}</value>
    <description>Local scratch space for Hive jobs</description>
</property>
<property>
    <name>hive.downloaded.resources.dir</name>
    <value>/tmp/hive/resources</value>
    <description>Temporary local directory for added resources in the remote file system.</description>
</property>
<property>
    <name>hive.querylog.location</name>
    <value>/tmp/${user.name}</value>
    <description>Location of Hive run time structured log file</description>
</property>
<property>
    <name>hive.server2.logging.operation.log.location</name>
    <value>/tmp/${user.name}/operation_logs</value>
    <description>Top level directory where operation logs are stored if logging functionality is enabled
    </description>
</property>

##配置Hive Metastore
<property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://192.168.0.7:3306/hive_meta?createDatabaseIfNotExist=true&characterEncoding=UTF-8&useSSL=false</value>
</property>
<property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
</property>
<property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>hive</value>
</property>
<property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>hive</value>
</property>

11.hdfs目录

hdfs dfs -mkdir /tmp
hdfs dfs -mkdir -p /user/hive/warehouse
hdfs dfs -chmod g+w /tmp
hdfs dfs -chmod g+w /user/hive/warehouse

12.初始化hive元数据

cd /u01/hive/bin
./schematool -dbType mysql -initSchema

13.测试hive

hive
show tables

14.连接hive

hive --service metastore >/dev/null 2>/dev/null &
hive --service hiveserver2 10000 >/dev/null 2>/dev/null &

$HIVE_HOME/bin/hive --service hiveserver2 10000 >/dev/null 2>/dev/null &
连接Hive JDBC URL：jdbc:hive2://192.168.6.116:10000/default     （Hive默认端口：10000  默认数据库名：default）

beeline
!connect jdbc:hive2://localhost:10000 hadoop hadoop

15.hiveServer2WebUI

<property>
    <name>hive.server2.webui.host</name>
    <value>192.168.0.9</value>
</property>

<property>
    <name>hive.server2.webui.port</name>
    <value>10002</value>
</property>

Hadoop 安装配置

1.创建用户组和用户

2.解压jdk和hadoop

3.配置环境变量

4.配置hosts

5.创建目录

6.配置hadoop

7.ssh免密码登陆

8.hive元数据

9.hive环境变量

10.hive配置

11.hdfs目录

12.初始化hive元数据

13.测试hive

14.连接hive

15.hiveServer2WebUI

git仓库地址

猜你喜欢

热点阅读