准备主机

image.png

在各个主机上初始设置

设置hostname

echo "k8s-N" > /etc/hostname # N是数字

设置初始环境

#关闭防火墙
systemctl stop firewalld && systemctl disable firewalld

#CentOS关闭selinux
setenforce 0
sed -i "s/^SELINUX=.*/SELINUX=disabled/g" /etc/sysconfig/selinux
sed -i "s/^SELINUX=.*/SELINUX=disabled/g" /etc/selinux/config
sed -i "s/^SELINUX=.*/SELINUX=disabled/g" /etc/sysconfig/selinux
sed -i "s/^SELINUX=.*/SELINUX=disabled/g" /etc/selinux/config

#修改时区
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

#修改系统语言环境
echo 'LANG="en_US.UTF-8"' >> /etc/profile;source /etc/profile

#kernel设置
  modprobe br_netfilter

    echo "
        net.ipv4.ip_forward = 1
        net.bridge.bridge-nf-call-ip6tables = 1
        net.bridge.bridge-nf-call-iptables = 1
        net.ipv4.neigh.default.gc_thresh1=4096
        net.ipv4.neigh.default.gc_thresh2=6144
        net.ipv4.neigh.default.gc_thresh3=8192
    " > /etc/sysctl.d/k8s.conf

    sysctl -p /etc/sysctl.d/k8s.conf

    # 加载ipvs相关内核模块
    # 如果重新开机，需要重新加载
    modprobe ip_vs
    modprobe ip_vs_rr
    modprobe ip_vs_wrr
    modprobe ip_vs_sh
    modprobe nf_conntrack_ipv4
    lsmod | grep ip_vs

#设置dns
echo "nameserver 8.8.8.8" > /etc/resolv.conf

#设置yum源
sudo cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.bak
echo '
[base]
name=CentOS-$releasever - Base - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
#released updates 
[updates]
name=CentOS-$releasever - Updates - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/centosplus/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
#contrib - packages by Centos Users
[contrib]
name=CentOS-$releasever - Contrib - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/contrib/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

' > /etc/yum.repos.d/CentOS-Base.repo

# 卸载旧版本Docker软件
sudo yum -y remove docker \
              docker-ce \
              docker-client \
              docker-client-latest \
              docker-common \
              docker-latest \
              docker-latest-logrotate \
              docker-logrotate \
              docker-selinux \
              docker-engine-selinux \
              docker-engine \
              container*
rm -rf /var/lib/docker*

#修改docker配置文件
mkdir -p /etc/docker
echo '
{
  "max-concurrent-downloads": 3,
  "max-concurrent-uploads": 5,
  "registry-mirrors": ["https://7bezldxe.mirror.aliyuncs.com/"],
  "insecure-registries": ["192.168.1.100"],
  "storage-driver": "overlay2",
  "storage-opts": ["overlay2.override_kernel_check=true"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m",
    "max-file": "3"
    }
}
' > /etc/docker/daemon.json

# 定义安装版本
export docker_version=17.03.2
# step 1: 安装必要的一些系统工具
yum clean all && yum makecache
sudo yum update -y
sudo yum install -y yum-utils device-mapper-persistent-data lvm2 bash-completion
# Step 2: 添加软件源信息
sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# Step 3: 更新并安装 Docker-CE
sudo yum makecache all
version=$(yum list docker-ce.x86_64 --showduplicates | sort -r|grep ${docker_version}|awk '{print $2}')
sudo yum -y install --setopt=obsoletes=0 docker-ce-${version} docker-ce-selinux-${version}
# 如果已经安装高版本Docker,可进行降级安装(可选)
# yum downgrade --setopt=obsoletes=0 -y docker-ce-${version} docker-ce-selinux-${version}

# 启动docker
systemctl start docker

# 设置开机启动
sudo systemctl enable docker

部署Rancher

启动

sudo docker run -d -v <主机路径>:/var/lib/rancher/ --restart=unless-stopped -p 80:80 -p 443:443 rancher/rancher:stable

访问

Rancher Server容器启动很快速，不到一分钟你就可以通过https://<server_ip>访问Rancher UI。

image.png

创建集群

非公有云VM或物理机，选择custom。

image.png

选择“demo项目>主机>编辑集群”

image.png

添加主机

添加

登录预添加集群的主机，执行以上复制的命令。添加过程需要拉取镜像，可能有速度慢问题

失败处理

由于网络或其他原因失败后，执行下面清除命令，并重新添加

df -h|grep kubelet |awk -F % '{print $2}'|xargs umount 
rm /var/lib/kubelet/* -rf
rm /etc/kubernetes/* -rf
rm /var/lib/rancher/* -rf
rm /var/lib/etcd/* -rf
rm /var/lib/cni/* -rf
iptables -F && iptables -t nat –F
ip link del flannel.1
docker ps -a|awk '{print $1}'|xargs docker rm -f
docker volume ls|awk '{print $2}'|xargs docker volume rm
systemctl stop docker
rm -rf /var/lib/docker*

安装kubectl

下载

wget https://www.cnrancher.com/download/kubectl/kubectl_amd64-linux

安装

chmod +x kubectl_amd64-linux && mv kubectl_amd64-linux kubectl && mv ./kubectl /usr/local/bin/kubectl

配置

在部署rancher主机上查找kube_config_cluster.yml
find / -name kube_config_cluster.yml
对接一个集群时复制到对应目录
cp 路径/kube_config_rancher-cluster.yml $HOME/.kube/config
对接多个集群时导出环境变量
export KUBECONFIG=路径/kube_config_rancher-cluster.yml

相关问题

问题1.执行kubectl命令：Unable to connect to the server: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kube-ca")
原因：ca证书没有配对

问题2.执行kubectl命令：The connection to the server <server-name:port> was refused - did you specify the right host or port?
原因：KUBE_APISERVER地址不对

rancher页面获取kubeconfig文件

image.png

手动生成kubeconfig文件

$ export KUBE_APISERVER="[https://172.20.0.113:6443](https://172.20.0.113:6443/)"
$ # 设置集群参数
$ kubectl config set-cluster kubernetes 
--certificate-authority=/etc/kubernetes/ssl/ca.pem 
--embed-certs=true 
--server=${KUBE_APISERVER}
$ # 设置客户端认证参数
$ kubectl config set-credentials admin 
--client-certificate=/etc/kubernetes/ssl/admin.pem 
--embed-certs=true 
--client-key=/etc/kubernetes/ssl/admin-key.pem
$ # 设置上下文参数
$ kubectl config set-context kubernetes 
--cluster=kubernetes 
--user=admin
$ # 设置默认上下文
$ kubectl config use-context kubernetes

部署Kubernetes Dashboard

从应用商店部署

image.png
若镜像下载失败，找到Kubernetes Dashboard应用，编辑yml文件，修改镜像名称为国内镜像名称，点击重新部署即可。
修改为：mirrorgooglecontainers/kubernetes-dashboard-amd64:v1.10.0
image.png

访问

kubectl cluster-info 查看kubernetes-dashboard访问地址

image.png
或通过rancher页面查看
image.png

创建dashboard用户

创建dashboard管理用户
kubectl create serviceaccount dashboard-admin -n kube-system
绑定用户为集群管理用户
kubectl create clusterrolebinding dashboard-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin

获取登录kubeconfig

使用rancher页面的kubeconfig
修改加入新建的用户，用户token。

apiVersion: v1
kind: Config
clusters:
- name: "zuozhu"
  cluster:
    server: "https://10.155.200.212/k8s/clusters/c-h6bfb"
    api-version: v1
    certificate-authority-data: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM3akNDQ\
      WRhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFvTVJJd0VBWURWUVFLRXdsMGFHVXQKY\
      21GdVkyZ3hFakFRQmdOVkJBTVRDV05oZEhSc1pTMWpZVEFlRncweE9UQXhNekV4TlRNNE1qZGFGd\
      zB5T1RBeApNamd4TlRNNE1qZGFNQ2d4RWpBUUJnTlZCQW9UQ1hSb1pTMXlZVzVqYURFU01CQUdBM\
      VVFQXhNSlkyRjBkR3hsCkxXTmhNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ\
      0tDQVFFQXJpTUlOUno0amVxdnBFMDEKMVllNHI1U3o3L2RNaUtnWWZlOTA0S0xMMGM2NUZqcGx3S\
      k95Ylo0OFdhVHBzbDdNWnJhZnJSZjJzb2JreHlkaAp5MFIrejNRRDM1dEs5cVM3SzQyL0VXN1p6Z\
      FF3R2treXp2aU1seGF0UGN1YmRPOGNZbkMzb1FGNDJ4NURxMCtqCm9acW1kMGVXcElTaFFJOFJYd\
      1JsUjBKSUlHWm1wejZJZUpBZ0lTOGo2SER5MDRsVVNicXNsamVLdW1WcnM1RHYKSlhzTVRvSzc1d\
      W9ialRtK1VDL2lKazRSZmhPcmxhN0VkMnpzWGQrVHpsK1Y5OElNNVdwcmUrUk9HOUN0eUVqTgpRc\
      mw0UlN6eGNsZDYxM2YwZ1dIeFlwQ1pnNkFqYWsvMnRjem5VcVBzWUZmY2g0WmFsTmNkeHJ3NGsyU\
      XY1VjdECkY5anNTd0lEQVFBQm95TXdJVEFPQmdOVkhROEJBZjhFQkFNQ0FxUXdEd1lEVlIwVEFRS\
      C9CQVV3QXdFQi96QU4KQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBUTFualRZN0V0Qm1kSjhPQ0Nwe\
      W9xSFVCNXZQa0NzcXVYVEVXODU5cgordk9GcHVPKytmU0pGNGFVUUZCRW0vM0NTemxnRkVCTDF6Q\
      lF2eUN6WGJvck5qelZ6Vzh2SXNqM3l2eHROTXJ5CmFiUkh1b1BaSkpYcEprMGhLbmRPRmozVEJMa\
      kdoT3c5STYraGN1WnB5ME4yT2taTmhPSFhpd01vbnBSTTFEU3kKUVBlbUxZYWpSempTSU8reHhsW\
      VYxalFWOFcwZVVJd2t1UlkyRkpsU3I0S1lodHZOeVExNFlLakJwcWdvSWJodwpZU0VmNEVvLy9zU\
      WFJQVRJaTBKN3hjT05tVGExRDlYYThIZysvS3k0d3RHNGNIN2k1bTFwYWRLdXZ5TEFiVWlwCm9Wc\
      2ROR3VQSW9yM1RVY3VaT2FhamN1Qk5YMnlHZDJDQlVUdGNGV0FHL3RXM1E9PQotLS0tLUVORCBDR\
      VJUSUZJQ0FURS0tLS0t"

users:
- name: "dashboard-admin"
  user:
    token: "eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4taHd4bG4iLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiZTI2ODJlZDQtMjYzMC0xMWU5LTgzOWItMDAwYzI5NDVkM2JkIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.Q2RlKw1PPNpFnBBf4T7pVDpQSIzHlHKOxmADGEhZAjxshydHq5PPr4tRqItU-E8bu6Edtkm9cC7GqrqtAWWk2C08bFSLO-KvXuVTYrrp4WQ0q7m5KmEhAxK1Ao-IoiCYssUiPdpiqIimH81DqGJ7u90fzuF7DrD7_wQeZpbgnUJj_DedF-1pIDmzRxT_0neLSuiNKck64KUgerNcxbrUVNZHxTmvDukZssltO24h6QbqSOjMfar54M_08VzfHoD7G4Z-7SFkZtc8TMmCAr473w1KgqTwW809WSjermilb4FL2RHY10TC6vGzs1pPxMfjBQ3RPI6_ljLwfxHiGLkDMg"

contexts:
- name: "zuozhu"
  context:
    user: "dashboard-admin"
    cluster: "zuozhu"

current-context: "zuozhu"

获取登录token

kubectl get secret --namespace=kube-system|grep dashboard-token|awk '{print $1}'|xargs -i kubectl describe secret {} --namespace=kube-system

image.png

部署镜像仓库

这里不使用helm或应用商店直接在k8s集群上安装harbor，而采用单独主机docker-compose离线安装。
参考Harbor单节点安装

搭建docker环境

同上面docker搭建步骤

搭建python、docker-compose环境

docker-compose用于在一台宿主机上快速部署一组docker服务。
1）
yum -y install epel-release
2）安装python-pip包
yum -y install python-pip
3）对安装好的pip进行升级
pip install --upgrade pip
查看
pip -V
4）安装docker-compose
pip --default-timeout=200 install -U docker-compose
查看
docker-compose -version

下载harbor离线包

image.png
上图为harbor github上版本资源
wget https://storage.googleapis.com/harbor-releases/release-1.7.0/harbor-offline-installer-v1.7.1.tgz
tar xvf harbor-offline-installer-<version>.tgz

修改配置文件

修改harbor.cfg文件

• hostname = 10.155.200.117 (主机IP)
• db_password = admin123 (harbor使用数据库密码)
• max_job_workers = 10（最大并发数量）
• log_rotate_count = 1 （删除文件时，日志回转次数）
• harbor_admin_password = admin123 （UI admin账户密码）

安装

./install.sh
默认镜像存储路径：/data/
默认日志存储路径：/var/lib/harbor/

管理镜像仓库生命周期

docker-compose start/stop/restart
更新配置

  docker-compose down -v
  vim harbor.cfg
  prepare
  docker-compose up -d

为镜像仓库添加https认证

创建root CA私钥

openssl req \
-newkey rsa:4096 -nodes -sha256 -keyout ca.key \
-x509 -days 365 -out ca.crt

Country Name(2 letter code)[AU]: CN
State or Province Name(full name)[Some-State]: Beijing
Locality Name(eg, city)[]: Beijing
Organization Name(eg, company)[Internet Widgits Pty Ltd]: xxx
Organizational Unit Name(eg, section)[]: info technology
Common Name(e.g. server FQDN or YOUR name)[]: 域名或IP
Email Address []: xxx@sina.com

为服务端(web)生成证书签名请求文件

openssl req \
-newkey rsa:4096 -nodes -sha256 -keyout demo.xxx.com.key \
-out  demo.xxx.com.csr

Country Name(2 letter code)[AU]: CN
State or Province Name(full name)[Some-State]: Beijing
Locality Name(eg, city)[]: Beijing
Organization Name(eg, company)[Internet Widgits Pty Ltd]: xxx
Organizational Unit Name(eg, section)[]: info technology
Common Name(e.g. server FQDN or YOUR name)[]: 域名或IP（不可与上面相同）
Email Address []: xxx@sina.com
A challenge password []: 回车
An optional company name []: xxx

用第一步创建的CA证书给第二步生成的签名请求进行签名

openssl x509 -req -days 365 -in demo.xxx.com.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out demo.xxx.com.crt
如果使用IP访问
echo 'subjectAltName = IP:xxx.xxx.xxx.xxx' > extfile.cnf openssl x509 -req -days 365 -in demo.xxx.com.csr -CA ca.crt -CAkey ca.key -CAcreateserial -extfile extfile.cnf -out demo.xxx.com.crt

修改配置文件

修改harbor.cfg

• ui_url_protocol = https
• ssl_cert = /root/cert/yourdomain.com.crt
• /root/cert/yourdomain.com.key

重启harbor

停止容器
docker-compose down -v
重新生成配置
./prepare
启动
docker-compose up -d

访问

https://<server-IP>

image.png

docker访问

image.png

因为我们添加了https认证，当前docker client不具有证书，所以不能够访问镜像仓库

docker client添加证书

TO DO
daemon.json官网全部配置

添加非安全镜像仓库

/etc/docker/daemon.json中加入，地址不能写成https://10.155.200.117
"insecure-registries": ["10.155.200.117"],

image.png

---

TO BE Continued