k8s ingress
2021-07-15 本文已影响0人
sjyu_eadd
image.png
ingress-controller有许多种（https://kubernetes.io/zh/docs/concepts/services-networking/ingress-controllers/），这里选用haproxy
ingress-controller及default-backend部署如下，也可以参考：https://www.haproxy.com/documentation/kubernetes/latest/installation/community/kubernetes/
---
apiVersion: v1
kind: Namespace
metadata:
name: haproxy-controller
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: haproxy-ingress-service-account
namespace: haproxy-controller
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: haproxy-ingress-cluster-role
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- services
- namespaces
- events
- serviceaccounts
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
- "networking.k8s.io"
resources:
- ingresses
- ingresses/status
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
- "networking.k8s.io"
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- watch
- create
- patch
- update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: haproxy-ingress-cluster-role-binding
namespace: haproxy-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: haproxy-ingress-cluster-role
subjects:
- kind: ServiceAccount
name: haproxy-ingress-service-account
namespace: haproxy-controller
---
apiVersion: v1
kind: ConfigMap
metadata:
name: haproxy
namespace: haproxy-controller
data:
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
run: ingress-default-backend
name: ingress-default-backend
namespace: haproxy-controller
spec:
replicas: 1
selector:
matchLabels:
run: ingress-default-backend
template:
metadata:
labels:
run: ingress-default-backend
spec:
containers:
- name: ingress-default-backend
image: gcr.io/google_containers/defaultbackend:1.0
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
labels:
run: ingress-default-backend
name: ingress-default-backend
namespace: haproxy-controller
spec:
selector:
run: ingress-default-backend
ports:
- name: port-1
port: 8080
protocol: TCP
targetPort: 8080
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
run: haproxy-ingress
name: haproxy-ingress
namespace: haproxy-controller
spec:
replicas: 1
selector:
matchLabels:
run: haproxy-ingress
template:
metadata:
labels:
run: haproxy-ingress
spec:
serviceAccountName: haproxy-ingress-service-account
containers:
- name: haproxy-ingress
image: haproxytech/kubernetes-ingress:1.6.4
args:
- --configmap=haproxy-controller/haproxy
- --default-backend-service=haproxy-controller/ingress-default-backend
securityContext:
runAsUser: 1000
runAsGroup: 1000
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
resources:
requests:
cpu: "500m"
memory: "50Mi"
livenessProbe:
httpGet:
path: /healthz
port: 1042
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
- name: stat
containerPort: 1024
env:
- name: TZ
value: "Etc/UTC"
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
initContainers:
- name: sysctl
image: busybox:musl
command:
- /bin/sh
- -c
- sysctl -w net.ipv4.ip_unprivileged_port_start=0
securityContext:
privileged: true
---
apiVersion: v1
kind: Service
metadata:
labels:
run: haproxy-ingress
name: haproxy-ingress
namespace: haproxy-controller
spec:
selector:
run: haproxy-ingress
type: NodePort
ports:
- name: http
port: 80
protocol: TCP
targetPort: 80
- name: https
port: 443
protocol: TCP
targetPort: 443
- name: stat
port: 1024
protocol: TCP
targetPort: 1024
ingress-controller和default-backend状态查询
root@master-1:~# kubectl get pod -o wide -n haproxy-controller
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
haproxy-ingress-5986944f76-c2s2g 1/1 Running 0 18h 10.244.2.63 master-3 <none> <none>
ingress-default-backend-78f5cc7d4c-z7fqh 1/1 Running 0 18h 10.244.2.62 master-3 <none> <none>
root@master-1:~# kubectl get svc -o wide -n haproxy-controller
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
haproxy-ingress NodePort 10.97.67.216 <none> 80:49296/TCP,443:57045/TCP,1024:52602/TCP 18h run=haproxy-ingress
ingress-default-backend ClusterIP 10.106.162.115 <none> 8080/TCP 18h run=ingress-default-backend
查询ingress-controller中haproxy配置
root@master-1:~# kubectl -n haproxy-controller exec -it haproxy-ingress-5986944f76-c2s2g -- sh
/ $ cat /usr/local/etc/haproxy/haproxy.cfg
# _version=29
# HAProxy Technologies
# https://www.haproxy.com/
# this file is not meant to be changed directly
# it is under haproxy ingress controller management
global
daemon
localpeer local
master-worker
pidfile /var/run/haproxy.pid
stats socket /var/run/haproxy-runtime-api.sock level admin expose-fd listeners
stats timeout 1m
tune.ssl.default-dh-param 2048
ssl-default-bind-options no-sslv3 no-tls-tickets no-tlsv10
ssl-default-bind-ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK
log 127.0.0.1:514 local0 notice
server-state-file global
server-state-base /var/state/haproxy/
defaults
log global
log-format '%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs "%HM %[var(txn.base)] %HV"'
option redispatch
option dontlognull
option http-keep-alive
timeout http-request 5s
timeout connect 5s
timeout client 50s
timeout queue 5s
timeout server 50s
timeout tunnel 1h
timeout http-keep-alive 1m
load-server-state-from-file global
peers localinstance
peer local 127.0.0.1:10000
frontend healthz
mode http
bind 0.0.0.0:1042 name v4
bind :::1042 name v6 v4v6
monitor-uri /healthz
option dontlog-normal
frontend http
mode http
bind 0.0.0.0:80 name v4
bind :::80 name v6
http-request set-var(txn.base) base
http-request set-var(txn.path) path
http-request set-var(txn.host) req.hdr(Host),field(1,:),lower
http-request set-var(txn.host_match) var(txn.host),map(/etc/haproxy/maps/host.map)
http-request set-var(txn.host_match) var(txn.host),regsub(^[^.]*,,),map(/etc/haproxy/maps/host.map,'') if !{ var(txn.host_match) -m found }
http-request set-var(txn.path_match) var(txn.host_match),concat(,txn.path,),map(/etc/haproxy/maps/path-exact.map)
http-request set-var(txn.path_match) var(txn.host_match),concat(,txn.path,),map_beg(/etc/haproxy/maps/path-prefix.map) if !{ var(txn.path_match) -m found }
use_backend %[var(txn.path_match),field(1,.)]
default_backend haproxy-controller-ingress-default-backend-port-1
frontend https
mode http
bind 0.0.0.0:443 name v4
bind :::443 name v6
http-request set-var(txn.base) base
http-request set-var(txn.path) path
http-request set-var(txn.host) req.hdr(Host),field(1,:),lower
http-request set-var(txn.host_match) var(txn.host),map(/etc/haproxy/maps/host.map)
http-request set-var(txn.host_match) var(txn.host),regsub(^[^.]*,,),map(/etc/haproxy/maps/host.map,'') if !{ var(txn.host_match) -m found }
http-request set-var(txn.path_match) var(txn.host_match),concat(,txn.path,),map(/etc/haproxy/maps/path-exact.map)
http-request set-var(txn.path_match) var(txn.host_match),concat(,txn.path,),map_beg(/etc/haproxy/maps/path-prefix.map) if !{ var(txn.path_match) -m found }
http-request set-header X-Forwarded-Proto https
use_backend %[var(txn.path_match),field(1,.)]
default_backend haproxy-controller-ingress-default-backend-port-1
frontend stats
mode http
bind *:1024
bind :::1024 name v6
stats enable
stats uri /
stats refresh 10s
http-request set-var(txn.base) base
http-request use-service prometheus-exporter if { path /metrics }
backend haproxy-controller-ingress-default-backend-port-1
mode http
balance roundrobin
option forwardfor
server SRV_1 10.244.2.62:8080 check weight 128
server SRV_2 127.0.0.1:8080 disabled check weight 128
server SRV_3 127.0.0.1:8080 disabled check weight 128
部署应用
kind: Deployment
apiVersion: apps/v1
metadata:
name: tomcat-deployment
spec:
replicas: 3
selector:
matchLabels:
app: tomcat
template:
metadata:
labels:
app: tomcat
spec:
containers:
- name: tomcat
image: tomcat:8.5.34-jre8-alpine
---
apiVersion: v1
kind: Service
metadata:
name: tomcat-svc
spec:
type: NodePort
selector:
app: tomcat
ports:
- protocol: TCP
port: 8080
targetPort: 8080
kind: Deployment
apiVersion: apps/v1
metadata:
name: httpd-deployment
spec:
replicas: 3
selector:
matchLabels:
app: httpd
template:
metadata:
labels:
app: httpd
spec:
containers:
- name: httpd
image: httpd
---
apiVersion: v1
kind: Service
metadata:
name: httpd-svc
spec:
selector:
app: httpd
ports:
- protocol: TCP
port: 80
targetPort: 80
应用状态查询
root@master-1:~# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
httpd-deployment-757fb56c8d-czwb2 1/1 Running 0 21h 10.244.3.150 node-1 <none> <none>
httpd-deployment-757fb56c8d-l8kjm 1/1 Running 0 3h28m 10.244.4.125 node-2 <none> <none>
httpd-deployment-757fb56c8d-zgm9v 1/1 Running 0 3h30m 10.244.2.65 master-3 <none> <none>
tomcat-deployment-57cb88956b-gss9n 1/1 Running 0 3h46m 10.244.3.155 node-1 <none> <none>
tomcat-deployment-57cb88956b-hggzr 1/1 Running 0 3h28m 10.244.2.66 master-3 <none> <none>
tomcat-deployment-57cb88956b-m4zk5 1/1 Running 0 3h24m 10.244.4.126 node-2 <none> <none>
应用服务端口查询
root@master-1:~# kubectl get svc -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
httpd-svc NodePort 10.111.172.142 <none> 80:2200/TCP 21h app=httpd
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 56d <none>
tomcat-svc NodePort 10.104.4.139 <none> 8080:46101/TCP 21h app=tomcat
通过NodePort端口号访问应用，其中10.101.x.x为主机ip
root@master-1:~# curl http://10.101.x.x:2200
<html><body><h1>It works!</h1></body></html>
root@master-1:~# curl http://10.101.x.x:46101
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>Apache Tomcat/8.5.34</title>
<link href="favicon.ico" rel="icon" type="image/x-icon" />
<link href="favicon.ico" rel="shortcut icon" type="image/x-icon" />
<link href="tomcat.css" rel="stylesheet" type="text/css" />
</head>
<body>
<div id="wrapper">
<div id="navigation" class="curved container">
部署ingress
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress
annotations:
haproxy.org/path-rewrite: "/"
spec:
rules:
- host: web.com
http:
paths:
- path: /httpd
pathType: Prefix
backend:
service:
name: httpd-svc
port:
number: 80
- path: /tomcat
pathType: Prefix
backend:
service:
name: tomcat-svc
port:
number: 8080
再次查询haproxy配置发生变化，增加如下字段：
backend default-tomcat-svc-8080
mode http
balance roundrobin
option forwardfor
server SRV_1 10.244.3.155:8080 check weight 128
server SRV_2 10.244.2.66:8080 check weight 128
server SRV_3 10.244.4.126:8080 check weight 128
backend default-httpd-svc-80
mode http
balance roundrobin
option forwardfor
server SRV_1 10.244.3.150:80 check weight 128
server SRV_2 10.244.2.65:80 check weight 128
server SRV_3 10.244.4.125:80 check weight 128
其中，server字段即为对应的pod ip+port
再次访问应用
/etc/hosts添加10.101.x.x web.com
root@master-1:~# curl http://web.com:49296/httpd
<html><body><h1>It works!</h1></body></html>
root@master-1:~# curl http://web.com:49296/tomcat
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>Apache Tomcat/8.5.34</title>
<link href="favicon.ico" rel="icon" type="image/x-icon" />
<link href="favicon.ico" rel="shortcut icon" type="image/x-icon" />
<link href="tomcat.css" rel="stylesheet" type="text/css" />
</head>
