k8s证书配置，dns，dashboard
2017-03-11 本文已影响2481人
秋幻旎苏
一.签发证书
TLS双向认知需要预先自建CA签发证书，权威CA机构的证书应该不可用，因为大部分k8s都是在内网中部署，而内网应该都会采用私有IP地址通讯，权威CA好像只能签署域名证书，对签署到IP可能无法实现.
master:192.168.23.128
node1:192.168.23.129
node2:192.168.23.131
node3:192.168.23.130
1.自签CA
对于私有证书签发首先要自签署一个CA根证书
创建证书存放的目录,
创建CA私钥
自签CA
[root@master ~]# mkdir /etc/kubernetes/ssl && cd /etc/kubernetes/ssl
openssl genrsa -out ca-key.pem 2048
[root@master ssl]# openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"
[root@master ssl]# ls
ca-key.pem ca.pem
2.签署apiserver 证书
自签 CA 后就需要使用这个根 CA 签署 apiserver 相关的证书了，首先先修改 openssl 的配置。
# vim openssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local
IP.1 = 10.254.0.1 #k8s 集群service ip
IP.2 = 192.168.23.128 #k8s master ip
然后开始签署apiserver相关的证书
# 生成 apiserver 私钥
openssl genrsa -out apiserver-key.pem 2048
# 生成签署请求
openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf
# 使用自建 CA 签署
openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
3.生成集群管理证书
openssl genrsa -out admin-key.pem 2048
openssl req -new -key admin-key.pem -out admin.csr -subj "/CN=kube-admin"
openssl x509 -req -in admin.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin.pem -days 365
4.签署node证书
先修改一下 openssl 配置
[root@master ssl]# cp openssl.cnf worker-openssl.cnf
[root@master ssl]# cat worker-openssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
IP.1 = 192.168.23.129
IP.2 = 192.168.23.131
IP.3 = 192.168.23.130
生成各个结点的证书并且拷贝到每个节点的目录下
[root@master ssl]# for i in {node1,node2,node3}
> do
> openssl genrsa -out $i-worker-key.pem 2048
> openssl req -new -key $i-worker-key.pem -out $i-worker.csr -subj "/CN=$i" -config worker-openssl.cnf
> openssl x509 -req -in $i-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out $i-worker.pem -days 365 -extensions v3_req -extfile worker-openssl.cnf
> ssh root@$i "mkdir /etc/kubernetes/ssl;chown kube:kube -R /etc/kubernetes/ssl"
> scp /etc/kubernetes/ssl/ca.pem /etc/kubernetes/ssl/$i* root@$i:/etc/kubernetes/ssl
> done
二.配置k8s
1.配置master
apiserver文件
KUBE_API_ADDRESS="--bind-address=192.168.23.128 --insecure-bind-address=127.0.0.1 "
# The port on the local server to listen on.
KUBE_API_PORT=="--secure-port=6443 --insecure-port=8080"
# Port minions listen on
# KUBELET_PORT="--kubelet-port=10250"
# Comma separated list of nodes in the etcd cluster
KUBE_ETCD_SERVERS="--etcd-servers=http://192.168.23.128:2379"
# Address range to use for services
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
# default admission control policies
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
#KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota"
# Add your own!
KUBE_API_ARGS="--tls-cert-file=/etc/kubernetes/ssl/apiserver.pem --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem --client-ca-file=/etc/kubernetes/ssl/ca.pem --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem"
config文件
KUBE_MASTER="--master=https://192.168.23.128:6443"
scheduler文件
KUBE_SCHEDULER_ARGS="--kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml --master=http://127.0.0.1:8080"
controller-manager
KUBE_CONTROLLER_MANAGER_ARGS="--service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem --root-ca-file=/etc/kubernetes/ssl/ca.pem --master=http://127.0.0.1:8080 --kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"
创建一个/etc/kubernetes/cm-kubeconfig.yaml 文件
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
certificate-authority: /etc/kubernetes/ssl/ca.pem
users:
- name: controllermanager
user:
client-certificate: /etc/kubernetes/ssl/apiserver.pem
client-key: /etc/kubernetes/ssl/apiserver-key.pem
contexts:
- context:
cluster: local
user: controllermanager
name: kubelet-context
current-context: kubelet-context
重启服务
systemctl restart etcd kube-apiserver.service kube-controller-manager.service kube-scheduler.service
2.配置node结点（以node1为例子）
config文件
KUBE_MASTER="--master=https://192.168.23.128:6443"
kubelet文件
KUBELET_ADDRESS="--address=192.168.23.129"
# The port for the info server to serve on
KUBELET_PORT="--port=10250"
# You may leave this blank to use the actual hostname
KUBELET_HOSTNAME="--hostname-override=node1"
# location of the api-server
KUBELET_API_SERVER="--api-servers=https://192.168.23.128:6443"
# pod infrastructure container
#KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=kubernetes/pause:latest"
# Add your own!
KUBELET_ARGS="--cluster_dns=10.254.0.3 --cluster_domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/node1-worker.pem --tls-private-key-file=/etc/kubernetes/ssl/node1-worker-key.pem --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml --allow-privileged=true"
proxy文件
KUBE_PROXY_ARGS="--proxy-mode=iptables --master=https://192.168.23.128:6443 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"
创建一个文件worker-kubeconfig.yaml
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
server: https://192.168.23.128:6443
certificate-authority: /etc/kubernetes/ssl/ca.pem
users:
- name: kubelet
user:
client-certificate: /etc/kubernetes/ssl/node1-worker.pem
client-key: /etc/kubernetes/ssl/node1-worker-key.pem
contexts:
- context:
cluster: local
user: kubelet
name: kubelet-context
current-context: kubelet-context
node1
#重启服务
systemctl restart kubelet kube-proxy
#查看 状态
systemctl status kubelet kube-proxy -l
#验证证书
curl https://192.168.23.128:6443/api/v1/nodes --cert /etc/kubernetes/ssl/node1-worker.pem --key /etc/kubernetes/ssl/node1-worker-key.pem --cacert /ec/kubernetes/ssl/ca.pem
curl这行要手打出来。不然会显示ca有问题
PS 如果显示certificate signed by unknown authority检查kubelet文件里面的配置。
三.配置dns
skydns-rc.yaml
apiVersion: v1
kind: ReplicationController
metadata:
name: kube-dns-v9
namespace: kube-system
labels:
k8s-app: kube-dns
version: v9
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
k8s-app: kube-dns
version: v9
template:
metadata:
labels:
k8s-app: kube-dns
version: v9
kubernetes.io/cluster-service: "true"
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
spec:
containers:
- name: etcd
image: test-registry:5000/etcd
resources:
limits:
cpu: 100m
memory: 50Mi
command:
- /usr/local/bin/etcd
- -data-dir
- /var/etcd/data
- -listen-client-urls
- http://127.0.0.1:2379,http://127.0.0.1:4001
- -advertise-client-urls
- http://127.0.0.1:2379,http://127.0.0.1:4001
- -initial-cluster-token
- skydns-etcd
volumeMounts:
- name: etcd-storage
mountPath: /var/etcd/data
- name: kube2sky
image: test-registry:5000/kube2sky
resources:
limits:
cpu: 100m
memory: 50Mi
args:
- -domain=cluster.local #设置k8s集群中Service所属的域名
- -kube_master_url=https://192.168.23.128:6443 #k8s中master的ip地址和apiserver中配置的端口号
- -kubecfg_file=/etc/kubernetes/worker-kubeconfig.yaml
volumeMounts:
- mountPath: /etc/kubernetes/ssl
name: ssl-certs-kubernetes
- mountPath: /etc/ssl/certs
name: ssl-certs-host
- mountPath: /etc/kubernetes/worker-kubeconfig.yaml
name: config
- name: skydns
image: test-registry:5000/skydns
resources:
limits:
cpu: 100m
memory: 50Mi
args:
- -machines=http://localhost:4001
- -addr=0.0.0.0:53
- -domain=cluster.local
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
volumes:
- name: etcd-storage
emptyDir: {}
- hostPath:
path: /etc/kubernetes/ssl
name: ssl-certs-kubernetes
- hostPath:
path: /etc/pki/tls/certs
name: ssl-certs-host
- hostPath:
path: /etc/kubernetes/worker-kubeconfig.yaml
name: config
skydns-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "KubeDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.254.0.3 #/etc/kubernetes/kubelet中已经设定好clusterIP
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
启动rc和svckubectl create -f skydns-svc.yaml,skydns-rc.yaml
排错
kubectl describe pod name --namespace=kube-system
kubectl logs podname -c containersname --namespace=kube-system
docker run test-registry:5000/kube2sky --help
检测
启动一个带有nslookup命令的容器解析同一命名空间内的service。
进入容器查看etcd里面获取到service的信息
[root@master build]# kubectl exec -it kube-dns-v9-b8a4z --namespace=kube-system -c etcd etcdctl ls /skydns/local/cluster
/skydns/local/cluster/default
/skydns/local/cluster/svc
/skydns/local/cluster/kube-system
四.配置dashboard
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
labels:
app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
app: kubernetes-dashboard
template:
metadata:
labels:
app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: index.tenxcloud.com/google_containers/kubernetes-dashboard-amd64:v1.4.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9090
protocol: TCP
args:
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
- --apiserver-host=https://192.168.23.128:6443
- --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml
livenessProbe:
httpGet:
path: /
port: 9090
initialDelaySeconds: 30
timeoutSeconds: 30
volumeMounts:
- mountPath: /etc/kubernetes/ssl
name: ssl-certs-kubernetes
- mountPath: /etc/ssl/certs
name: ssl-certs-host
- mountPath: /etc/kubernetes/worker-kubeconfig.yaml
name: config
volumes:
- hostPath:
path: /etc/kubernetes/ssl
name: ssl-certs-kubernetes
- hostPath:
path: /etc/pki/tls/certs
name: ssl-certs-host
- hostPath:
path: /etc/kubernetes/worker-kubeconfig.yaml
name: config
kind: Service
apiVersion: v1
metadata:
labels:
app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
type: NodePort
ports:
- port: 80
nodePort: 30010
targetPort: 9090
selector:
app: kubernetes-dashboard
访问方法 http://nodeip:30010
