Spark-On-Yarn模式

[server@hadoop102 ~]$ cd /opt/software/
[server@hadoop102 software]$ tar -zxvf spark-3.0.0-bin-hadoop3.2.tgz -C /opt/module/
[server@hadoop102 module]$ mv spark-3.0.0-bin-hadoop3.2/ spark-yarn

[server@hadoop102 module]$ cd /opt/module/hadoop-3.1.3/etc/hadoop/
[server@hadoop102 hadoop]$ vim yarn-site.xml

增加

    <property>
             <name>yarn.nodemanager.pmem-check-enabled</name>
             <value>false</value>
    </property>
    
    <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
    </property>

[server@hadoop102 hadoop]$ cd /opt/module/spark-yarn/conf/
[server@hadoop102 conf]$ mv spark-env.sh.template spark-env.sh
[server@hadoop102 conf]$ vim spark-env.sh 

增加

export JAVA_HOME=/opt/module/jdk1.8.0_65
YARN_CONF_DIR=/opt/module/hadoop-3.1.3/etc/hadoop/

export SPARK_HISTORY_OPTS="
-Dspark.history.ui.port=18080
-Dspark.history.fs.logDirectory=hdfs://hadoop102/spark
-Dspark.history.retainedApplications=30"

启动hadoop集群创建目录

[server@hadoop102 conf]$ cd ~
[server@hadoop102 ~]$ cd bin
[server@hadoop102 bin]$ myhadoop.sh start
=================启动 Hadoop集群========================
------------------启动 hdfs-----------------------------
Starting namenodes on [hadoop102]
Starting datanodes
Starting secondary namenodes [hadoop104]
------------------启动 yarn-----------------------------
Starting resourcemanager
Starting nodemanagers
------------------启动 historyserver--------------------
[server@hadoop102 bin]$ hadoop fs -mkdir /spark

配置历史服务

[server@hadoop102 conf]$ mv spark-defaults.conf.template spark-defaults.conf
[server@hadoop102 conf]$ vim spark-defaults.conf 

增加

spark.eventLog.enabled           true
spark.eventLog.dir               hdfs://hadoop102/spark

spark.yarn.historyServer.address=hadoop102:18080
spark.history.ui.port=18080 

[server@hadoop102 conf]$ cd /opt/module/spark-yarn/
[server@hadoop102 spark-yarn]$ sbin/start-history-server.sh 
starting org.apache.spark.deploy.history.HistoryServer, logging to /opt/module/spark-yarn/logs/spark-server-org.apache.spark.deploy.history.HistoryServer-1-hadoop102.out

提交作业

bin/spark-submit \
--class org.apache.spark.examples.SparkPi \
--master yarn \
--deploy-mode client \
./examples/jars/spark-examples_2.12-3.0.0.jar \
10

[server@hadoop102 spark-yarn]$ bin/spark-submit \
> --class org.apache.spark.examples.SparkPi \
> --master yarn \
> --deploy-mode client \
> ./examples/jars/spark-examples_2.12-3.0.0.jar \
> 10

(http://hadoop103:8088/cluster


编写Java进程脚本并查看https://www.jianshu.com/p/fdbde4bc01ae

[server@hadoop102 spark-yarn]$ jpsall
=================hadoop102===================
8640 HistoryServer
7938 NodeManager
9256 Jps
7485 NameNode
8110 JobHistoryServer
7599 DataNode
=================hadoop103===================
7216 DataNode
7408 ResourceManager
7520 NodeManager
8412 Jps
=================hadoop104===================
7412 NodeManager
7301 SecondaryNameNode
8105 Jps
7195 DataNode