Byzer-lang 集成 Hive 2.3.9

根据 Spark 文档， hive 2.x 只支持到 2.3.7。Hive 2.3.9 经测试能启动，但可能依然有兼容性问题, 这里还没测试。

环境介绍

单机安装以下应用. ubuntu 20.04 JDK 8。

Hive 2.3.9 下载解压

请参考 Ubuntu 安装 Hive 2.x

Spark 3.1.1 适配

建立 hive-site.xml 软链接，Spark 可以找到 Hive MetaStore 连接信息

cd /work/server/spark_3.1.1-hive_2.3.9
ln -s /work/server/apache-hive-2.3.9-bin/conf/hive-site.xml hive-site.xml

配置 Hive MetaStore Jar

# 修改 spark-defaults.conf
cp spark-defaults.conf.template spark-defaults.conf
vi spark-defaults.conf

# 加入以下内容
# Hive 版本
# 2.3.9 导致报错，Available options are 0.12.0 through 2.3.7 and 3.0.0 through 3.1.2. 考虑到小版本差异，我们尝试一下。
spark.sql.hive.metastore.version=2.3.7
# 从本地路径加载 Hive Metastore Jar包。 
spark.sql.hive.metastore.jars=path
# 每个 jar 用 , 分割
spark.sql.hive.metastore.jars.path=file:///work/server/apache-hive-2.3.9-bin/lib/hive-metastore-2.3.9.jar,file:///work/server/apache-hive-2.3.9-bin/lib/hive-exec-2.3.9.jar,file:///work/server/apache-hive-2.3.9-bin/lib/commons-logging-1.2.jar,file:///work/server/apache-hive-2.3.9-bin/lib/commons-io-2.4.jar,file:///work/server/apache-hive-2.3.9-bin/lib/javax.servlet-api-3.1.0.jar,file:///work/server/apache-hive-2.3.9-bin/lib/commons-codec-1.4.jar,file:///work/server/apache-hive-2.3.9-bin/lib/libfb303-0.9.3.jar,file:///work/server/apache-hive-2.3.9-bin/lib/metrics-core-3.1.0.jar,file:///work/server/apache-hive-2.3.9-bin/lib/datanucleus-core-4.1.17.jar,file:///work/server/apache-hive-2.3.9-bin/lib/datanucleus-api-jdo-4.2.4.jar,file:///work/server/apache-hive-2.3.9-bin/lib/javax.jdo-3.2.0-m3.jar,file:///work/server/apache-hive-2.3.9-bin/lib/datanucleus-rdbms-4.1.19.jar,file:///work/server/apache-hive-2.3.9-bin/lib/HikariCP-2.5.1.jar,file:///work/server/apache-hive-2.3.9-bin/lib/mysql-connector-java-5.1.48.jar,file:///work/server/spark_3.1.1-hive_2.3.9/jars/commons-collections-3.2.2.jar,file:///work/server/apache-hive-2.3.9-bin/lib/antlr-runtime-3.5.2.jar,file:///work/server/apache-hive-2.3.9-bin/lib/jackson-core-2.6.5.jar,file:///work/server/apache-hive-2.3.9-bin/lib/jackson-annotations-2.6.0.jar,file:///work/server/apache-hive-2.3.9-bin/lib/jackson-databind-2.6.5.jar,file:///work/server/apache-hive-2.3.9-bin/lib/jackson-mapper-asl-1.9.13.jar,file:///work/server/apache-hive-2.3.9-bin/lib/jackson-core-asl-1.9.13.jar

启动 Byzer-lang & Notebook

在本机 ~/bin/ 目录创建 Byzer-lang 启动脚本

vi start-mlsql-3.0-latest-yarn-hive_2.3.9.sh

#!/bin/bash

set -e
set -o pipefail
# yarn-client 模式启动 Byzer-lang , 同时访问本机 Hive 2.3.9
# 适配 Hive 2.3.9 的 Byzer-lang
MLSQL_HOME=/work/server/mlsql-engine_3.0-2.3.0-SNAPSHOT-hive_2.3.9
# 适配 Hive 2.3.9 的 Spark
SPARK_HOME=/work/server/spark_3.1.1-hive_2.3.9

JARS=$(echo ${MLSQL_HOME}/libs/*.jar | tr ' ' ',')
MAIN_JAR=$(ls ${MLSQL_HOME}/libs|grep 'streamingpro-mlsql')
export DRIVER_MEMORY=${DRIVER_MEMORY:-1g}

echo "##############################"
echo "Run with spark : $SPARK_HOME"
echo "With DRIVER_MEMORY=${DRIVER_MEMORY:-1g}"
echo "JARS: ${JARS}"
echo "MAIN_JAR: ${MLSQL_HOME}/libs/${MAIN_JAR}"
echo "##############################"

nohup $SPARK_HOME/bin/spark-submit --class streaming.core.StreamingApp \
        --driver-memory "${DRIVER_MEMORY}" \
        --jars "${JARS}" \
        --master yarn \
        --deploy-mode client \
        --name mlsql \
        --conf "spark.executor.memory=1024m" \
        --conf "spark.executor.instances=1" \
        --conf "spark.sql.hive.thriftServer.singleSession=true" \
        --conf "spark.kryoserializer.buffer=256k" \
        --conf "spark.kryoserializer.buffer.max=64m" \
        --conf "spark.serializer=org.apache.spark.serializer.KryoSerializer" \
        --conf "spark.scheduler.mode=FAIR" \
        "${MLSQL_HOME}/libs/${MAIN_JAR}" \
        -streaming.name mlsql \
        -streaming.platform spark \
        -streaming.rest true \
        -streaming.driver.port 9005   \
        -streaming.spark.service true \
        -streaming.thrift false \
        -streaming.enableHiveSupport true \
        -streaming.datalake.path /work/data/mlsql \
> /work/logs/mlsql-3.0-2.3.0-SNAPSHOT-hive_2.3.9.log 2>&1 &

修改 Notebook 配置文件( notebook.properties)

notebook.mlsql.engine-url=http://localhost:9005

启动 Notebook, 再访问 localhost:9002，执行以下 Byzer-lang 代码，均成功。

select 2 as c1 as new_data;
-- 数据 覆盖写入表 zjc_11.zjc_0305
save overwrite new_data as hive.`zjc_11.zjc_0305`;
-- 读取数据
load hive.`zjc_11.zjc_0305` as hive_zjc_12;