superset 0.36.0 搭建编译镜像

2020-11-19  本文已影响0人  郭青耀

准备环境

  1. centos7.3基础镜像
  2. python3.7.9 环境准备
  3. superset安装包准备

拉取基础镜像并进入

docker  pull  centos:centos7.3.1611
docker  run  -ti centos:centos7.3.1611 /bin/bash

替换centos的源

cd /etc/yum.repos.d/
mkdir bak
mv *.repo bak/
wget  http://mirrors.aliyun.com/repo/Centos-7.repo
yum clean all 
yum makecache

安装编译环境依赖

yum install gcc gcc-c++  -y
yum install binutils -y
yum install postgresql-devel*
wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient18.3-basic-18.3.0.0.0-3.x86_64.rpm
yum install libaio
rpm -ivh oracle-instantclient18.3-basic-18.3.0.0.0-3.x86_64.rpm

yum install postgresql-devel*
yum install unixODBC-devel

当前准备编译路径

mkdir /opt
export SUPERSET_HOME_BUILD=/opt

python3.7.9环境准备

使用conda创建python环境

  1. 配置conda环境变量
[root@34e2d48a8660 ~]# cat .condarc 
auto_activate_base: false
show_channel_urls: true
channel_alias: https://mirrors.tuna.tsinghua.edu.cn/anaconda
default_channels:
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/pro
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
custom_channels:
  conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud

1.安装anaconda

cd $SUPERSET_HOME_BUILD
wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh
chmod +x Miniconda2-latest-Linux-x86_64.sh

./Miniconda2-latest-Linux-x86_64.sh -p /opt/py3.7
cd /opt/py3.7/bin

设置不自动加载base环境

conda config --set auto_activate_base false

2.创建python环境

python_home=${SUPERSET_HOME_BUILD}/python3.7
mkdir ${python_home}
#创建python3.7环境
./susconda create  -p ${python_home} python=3.7
#查看并试运行
cd ${python_home} && cd bin 
./python --version
#显示 Python 3.7.6 即表示成功

superset源码下载编译安装

下载superset源码
mkdir -p ${SUPERSET_HOME_BUILD}
cd ${SUPERSET_HOME_BUILD}
git clone https://github.com/apache/incubator-superset
export SUPERSET_SOURCE_PATH=/opt/code/incubator-superset
创建对应的superset依赖
export PATH=${SUPERSET_HOME_BUILD}/python3.7/bin:$PATH
#安装依赖
python -m pip install -e ${SUPERSET_SOURCE_PATH}/
#安装mysql 连接依赖
yum install mariadb* -y
python -m pip install -i https://pypi.tuna.tsinghua.edu.cn/simple mysqlclient
下载nodejs
mkdir ${SUPERSET_HOME_BUILD}
export NODE_PATH=${SUPERSET_HOME_BUILD}/node-v12.18.2-linux-x64
https://nodejs.org/dist/v12.18.2/node-v12.18.2-linux-x64.tar.xz
tar -xvf node-v12.18.2-linux-x64.tar.xz
编译前端
export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PATH=$PATH:$NODE_PATH/bin
mkdir -p ${SUPERSET_PATH}/app/superset-frontend
mkdir -p ${SUPERSET_PATH}/app/superset/assets
export SUPERSET_SOURCE_PATH=${SUPERSET_HOME_BUILD}/code/incubator-superset

复制前端源码

cp -r ${SUPERSET_SOURCE_PATH}/superset-frontend/* ${SUPERSET_PATH}/app/superset-frontend
npm config set registry http://registry.npm.taobao.org/
npm config get registry
#直接使用默认的运行有问题，提前需要安装下面的包
npm install -D babel-loader @babel/core @babel/preset-env webpack
#编译安装 // docker容器内编译需要调整 docker daemon ：   mem 4G swap 2G
cd ${SUPERSET_PATH}/app/superset-frontend && npm ci && npm run build && rm -rf node_modules
编译后端
#设置环境变量
export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PYTHON_HOME=${SUPERSET_HOME_BUILD}/python3.7
export LANG=C.UTF-8
export LC_ALL=C.UTF-8
npm config set registry http://registry.npm.taobao.org/
npm config get registry
#直接使用默认的运行有问题，提前需要安装下面的包
npm install -D babel-loader @babel/core @babel/preset-env webpack
export FLASK_ENV=production
export FLASK_APP="superset.app:create_app()"
export PYTHONPATH="${PYTHON_HOME}"
export SUPERSET_HOME="$SUPERSET_PATH/app/superset"
export SUPERSET_PORT=8080
export PATH=$PATH:$PYTHON_HOME/bin
export npm config set registry http://registry.npm.taobao.org/
npm config get registry
#直接使用默认的运行有问题，提前需要安装下面的包
npm install -D babel-loader @babel/core @babel/preset-env webpackPATH=$PYTHON_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${SUPERSET_HOME_BUILD}/mysql

cp -r ${SUPERSET_SOURCE_PATH}/superset ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/setup.py ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/MANIFEnpm config set registry http://registry.npm.taobao.org/
npm config get registryST.in ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/README.md ${SUPERSET_PATH}/app/

替换python脚本内的解释执行器

grep /bin/python ${PYTHON_HOME}/bin/*  -l | xargs sed -i '1s/^.*$/#!\/usr\/bin\/env python/

安装superset

cd ${SUPERSET_PATH}/app/
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -e .
cd -

pip uninstall marshmallow
pip uninstall marshmallow-sqlalchemy
pip install marshmallow-sqlalchemy==0.20.0
pip install  marshmallow==2.18.0

superset初始化

#设置初始化admin角色的用户名密码
export ADMIN_USERNAME=admin
export ADMIN_FIRSTNAME=Superset
export ADMIN_LASTNAME=Admin
export ADMIN_PASSWORD=admin
export ADMIN_EMAIL=admin@superset.com

STEP_CNT=4

echo_step() {
cat <<EOF
######################################################################
Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
######################################################################
EOF
}

# Create an admin user
  echo_step "1" "Starting" "Setting up admin user ( admin / admin )"
  superset fab create-admin \
                --username ${ADMIN_USERNAME} \
                --firstname ${ADMIN_FIRSTNAME} \
                --lastname ${ADMIN_LASTNAME} \
                --email ${ADMIN_EMAIL} \
                --password ${ADMIN_PASSWORD}
  echo_step "1" "Complete" "Setting up admin user"

  # Initialize the database
  echo_step "2" "Starting" "Applying DB migrations"
  superset db upgrade
  echo_step "2" "Complete" "Applying DB migrations"

  if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
      # Load some data to play with
      echo_step "3" "Starting" "Loading examples"
      superset load_examples
      echo_step "3" "Complete" "Loading examples"
  fi

  # Create default roles and permissions
  echo_step "4" "Starting" "Setting up roles and perms"
  superset init
  echo_step "4" "Complete" "Setting up roles and perms"
启动
#设置环境变量export SUPERSET_HOME_BUILD=/opt
python_home=${SUPERSET_HOME_BUILD}/python3.7
export SUPERSET_SOURCE_PATH=/opt/code/incubator-superset
export PATH=${SUPERSET_HOME_BUILD}/python3.7/bin:$PATH
export NODE_PATH=${SUPERSET_HOME_BUILD}/node-v12.18.2-linux-x64
export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PATH=$PATH:$NODE_PATH/bin
export SUPERSET_SOURCE_PATH=${SUPERSET_HOME_BUILD}/code/incubator-superset
#当前编译根路径，当前选择的是/opt
export SUPERSET_HOME_BUILD=`pwd`
export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PYTHON_HOME=${SUPERSET_HOME_BUILD}/python3.7
#export LANG=C.UTF-8
#export LC_ALL=C.UTF-8
export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8
export FLASK_ENV=production
export FLASK_APP="superset.app:create_app()"
export PYTHONPATH="${PYTHON_HOME}"
export SUPERSET_HOME="$SUPERSET_PATH/app/superset"
export SUPERSET_PORT=8080
export PATH=$PATH:$PYTHON_HOME/bin
export PATH=$PYTHON_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${SUPERSET_HOME_BUILD}/mysql

gunicorn \
        --bind  "0.0.0.0:${SUPERSET_PORT}" \
        --access-logfile '-' \
        --error-logfile '-' \
        --workers 1 \
        --worker-class gthread \
        --threads 20 \
        --timeout 60 \
        --limit-request-line 0 \
        --limit-request-field_size 0 \
        "${FLASK_APP}"
上一篇下一篇

猜你喜欢

热点阅读