通过kubeadm部署v1.13版本高可用kubernetes集群
目录：
环境
/etc/hosts
172.19.0.145 node-02
172.19.0.8 node-01
172.19.0.141 haproxy-02
172.19.0.138 haproxy-01
172.19.0.118 etcd-03
172.19.0.4 etcd-02
172.19.0.84 etcd-01
172.19.0.48 master-03
172.19.0.142 master-02
172.19.0.57 master-01
这里是通过ipvs+haproxy实现的高可用负载，依赖的是组播的功能，部分云服务并不支持组播，可以直接使用的云服务的四层负载来实现，但是也要依赖haproxy，因为LB不支持后端ECS实例既作为Real Server又作为客户端向所在的负载均衡实例发送请求。因为返回的数据包只在云服务器内部转发，不经过负载均衡，所以在后端ECS实例上去访问负载均衡的服务地址是不通的。
机器预处理
关闭selinux
$ sed -ri 's#(SELINUX=).*#\1disabled#' /etc/selinux/config
$ setenforce 0
关闭防火墙
$ systemctl disable firewalld
$ systemctl stop firewalld
关闭swap
$ swapoff -a
配置内核转发相关参数
$ cat <<EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness=0
EOF
$
$ sysctl --system
加载ipvs模块
使用云服务器的负载均衡跳过这一步
$ cat << EOF > /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
ipvs_modules_dir="/usr/lib/modules/\`uname -r\`/kernel/net/netfilter/ipvs"
for i in \`ls \$ipvs_modules_dir | sed -r 's#(.*).ko.*#\1#'\`; do
/sbin/modinfo -F filename \$i &> /dev/null
if [ \$? -eq 0 ]; then
/sbin/modprobe \$i
fi
done
EOF
$ chmod +x /etc/sysconfig/modules/ipvs.modules
$ bash /etc/sysconfig/modules/ipvs.modules
master节点安装cfssl
$ wget -O /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
$ wget -O /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
$ wget -O /bin/cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
$ for cfssl in `ls /bin/cfssl*`;do chmod +x $cfssl;done;
安装docker
$ yum install -y yum-utils device-mapper-persistent-data lvm2
$ yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
$ yum install -y docker-ce
$ systemctl enable docker && systemctl start docker
安装kubeadm
$ cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kube*
EOF
$ yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
$ systemctl enable kubelet && systemctl start kubelet
搭建etcd
证书配置
$ mkdir -pv $HOME/ssl && cd $HOME/ssl
$ cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
EOF
$ cat > etcd-ca-csr.json << EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
EOF
$ cat > etcd-csr.json << EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"172.19.0.118",
"172.19.0.4",
"172.19.0.84"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
EOF
生成证书并复制证书至其他etcd节点
$ cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
$ cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
$ mkdir -pv /etc/etcd/ssl
$ mkdir -pv /etc/kubernetes/pki/etcd
$ cp etcd*.pem /etc/etcd/ssl
$ cp etcd*.pem /etc/kubernetes/pki/etcd
$ scp -r /etc/etcd etcd-01:/etc/
$ scp -r /etc/etcd etcd-02:/etc/
$ scp -r /etc/etcd etcd-03:/etc/
配置etcd
安装etcd
$ yum install -y etcd
修改配置文件/etc/etcd/etcd.conf，同理配置三台
#[Member]
#ETCD_CORS=""
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
ETCD_LISTEN_PEER_URLS="https://172.19.0.84:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://172.19.0.84:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="etcd-01"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.19.0.84:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://172.19.0.84:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_DISCOVERY_SRV=""
ETCD_INITIAL_CLUSTER="etcd-03=https://172.19.0.118:2380,etcd-02=https://172.19.0.4:2380,etcd-01=https://172.19.0.84:2380"
ETCD_INITIAL_CLUSTER_TOKEN="Why"
#ETCD_INITIAL_CLUSTER_STATE="new"
#ETCD_STRICT_RECONFIG_CHECK="true"
#ETCD_ENABLE_V2="true"
#
#[Proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"
#
#[Security]
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
#ETCD_CLIENT_CERT_AUTH="false"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem"
#ETCD_AUTO_TLS="false"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
#ETCD_PEER_CLIENT_CERT_AUTH="false"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem"
#ETCD_PEER_AUTO_TLS="false"
#
#[Logging]
#ETCD_DEBUG="false"
#ETCD_LOG_PACKAGE_LEVELS=""
#ETCD_LOG_OUTPUT="default"
#
#[Unsafe]
#ETCD_FORCE_NEW_CLUSTER="false"
#
#[Version]
#ETCD_VERSION="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"
#
#[Profiling]
#ETCD_ENABLE_PPROF="false"
#ETCD_METRICS="basic"
#
#[Auth]
#ETCD_AUTH_TOKEN="simple"
启动并检查
$ chown -R etcd.etcd /etc/etcd
$ systemctl enable etcd
$ systemctl start etcd
服务全部启动后进行检查
$ etcdctl --endpoints "https://172.19.0.84:2379,https://172.19.0.4:2379,https://172.19.0.118:2379" --ca-file=/etc/etcd/ssl/etcd-ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem cluster-health
配置keepalived+haproxy或者CLB+haproxy
yum安装keepalived
yum install -y keepalived
配置keepalived
$ cat << EOF > /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost #发送邮箱
}
notification_email_from keepalived@localhost #邮箱地址
smtp_server 127.0.0.1 #邮件服务器地址
smtp_connect_timeout 30
router_id haproxy-01 #主机名，每个节点不同即可
vrrp_mcast_group4 224.0.100.100 #组播地址
}
vrrp_instance VI_1 {
state BACKUP #在另一个节点上为MASTER
interface eth0 #IP地址漂移到的网卡
virtual_router_id 6 #多个节点必须相同
priority 80 #优先级，备用节点的值必须低于主节点的值
advert_int 1 #通告间隔1秒
authentication {
auth_type PASS #预共享密钥认证
auth_pass 571f97b2 #密钥
}
virtual_ipaddress {
172.19.0.95/16 #漂移过来的IP地址
}
}
EOF
启动服务
$ systemctl enable keepalived
$ systemctl start keepalived
如果是CLB就直接是内网CLB的IP地址即可，监听6443端口，后端的机器为haproxy，端口为6443
安装haproxy
yum install -y haproxy
配置haproxy
cat << EOF > /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
defaults
mode tcp
log global
retries 3
timeout connect 10s
timeout client 1m
timeout server 1m
frontend kubernetes
bind *:6443
mode tcp
default_backend kubernetes-master
backend kubernetes-master
balance roundrobin
server master-01 172.19.0.57:6443 check maxconn 2000
server master-02 172.19.0.142:6443 check maxconn 2000
server master-03 172.19.0.48:6443 check maxconn 2000
EOF
启动服务
systemctl enable haproxy
systemctl start haproxy
kubernetes
从v1.12版本开始kubeadm的api改变了，可以通过旧的yaml配置升级为新版api的yaml配置，但是由于v1.12版本有漏洞，直接下载了v1.13，但是v1.13版本的kubeadm不支持转换，转换完成是一个空的yaml，必须通过v1.12版本的kubeadm进行转换
这是提示
Please use kubeadm v1.12 instead and run 'kubeadm config migrate --old-config old.yaml --new-config new.yaml'
当然也可以使用kubeadm config print init-defaults，就能看到默认的apiVersion了
这是我转换之后的
apiEndpoint:
advertiseAddress: 172.19.0.95
bindPort: 6443
apiVersion: kubeadm.k8s.io/v1alpha3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: jhl1hh.p25woojjwy7uw1g4
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: master-03
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServerCertSANs:
- master-01
- master-02
- master-03
- 172.19.0.48
- 172.19.0.142
- 172.19.0.57
- 172.19.0.84
- 172.19.0.4
- 172.19.0.118
- 172.19.0.95
- 127.0.0.1
apiVersion: kubeadm.k8s.io/v1alpha3
auditPolicy:
logDir: /var/log/kubernetes/audit
logMaxAge: 2
path: ""
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 172.19.0.95:6443
etcd:
external:
caFile: /etc/kubernetes/pki/etcd/etcd-ca.pem
certFile: /etc/kubernetes/pki/etcd/etcd.pem
endpoints:
- https://172.19.0.4:2379
- https://172.19.0.84:2379
- https://172.19.0.118:2379
keyFile: /etc/kubernetes/pki/etcd/etcd-key.pem
featureGates:
CoreDNS: true
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.13.0
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16
serviceSubnet: 10.96.0.0/12
unifiedControlPlaneImage: ""
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
acceptContentTypes: ""
burst: 10
contentType: application/vnd.kubernetes.protobuf
kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
qps: 5
clusterCIDR: 10.244.0.0/16
configSyncPeriod: 15m0s
conntrack:
max: null
maxPerCore: 32768
min: 131072
tcpCloseWaitTimeout: 1h0m0s
tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
masqueradeAll: false
masqueradeBit: 14
minSyncPeriod: 0s
syncPeriod: 30s
ipvs:
excludeCIDRs: null
minSyncPeriod: 0s
scheduler: ""
syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: ipvs
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
resourceContainer: /kube-proxy
udpIdleTimeout: 250ms
当然我没用这个配置，我参考官方的hubernetes高可用集群
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
apiServer:
certSANs:
- 172.19.0.95
controlPlaneEndpoint: 172.19.0.95:6443
etcd:
external:
caFile: /etc/kubernetes/pki/etcd/etcd-ca.pem
certFile: /etc/kubernetes/pki/etcd/etcd.pem
endpoints:
- https://172.19.0.4:2379
- https://172.19.0.84:2379
- https://172.19.0.118:2379
keyFile: /etc/kubernetes/pki/etcd/etcd-key.pem
kubernetesVersion: v1.13.0
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16
serviceSubnet: 10.96.0.0/12
进行部署
kubeadm init --config /root/kubeadm-init.yaml
执行结果
Your Kubernetes master has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of machines by running the following on each node
as root:
kubeadm join 172.19.0.95:6443 --token nrbrt0.d1lyjst4cl780oos --discovery-token-ca-cert-hash sha256:88f993ba0c0e829b29e4811a03845892c8df6d82a023715f13d091d536996ea6
根据提示
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
cat << EOF > /etc/profile.d/kubernetes.sh
source <(kubectl completion bash)
EOF
source /etc/profile.d/kubernetes.sh
查看一下Node
[root@master-01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
master-01 NotReady master 7m44s v1.13.1
使用flannel网络
$ kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.extensions/kube-flannel-ds-amd64 created
daemonset.extensions/kube-flannel-ds-arm64 created
daemonset.extensions/kube-flannel-ds-arm created
daemonset.extensions/kube-flannel-ds-ppc64le created
daemonset.extensions/kube-flannel-ds-s390x created
拷贝配置文件到其他master节点
export master='master-03'
scp /etc/kubernetes/pki/ca.crt $master:/etc/kubernetes/pki/ca.crt
scp /etc/kubernetes/pki/ca.key $master:/etc/kubernetes/pki/ca.key
scp /etc/kubernetes/pki/sa.pub $master:/etc/kubernetes/pki/sa.pub
scp /etc/kubernetes/pki/sa.key $master:/etc/kubernetes/pki/sa.key
scp /etc/kubernetes/pki/front-proxy-ca.crt $master:/etc/kubernetes/pki/front-proxy-ca.crt
scp /etc/kubernetes/pki/front-proxy-ca.key $master:/etc/kubernetes/pki/front-proxy-ca.key
scp -r /etc/kubernetes/pki/etcd/ $master:/etc/kubernetes/pki/
scp -r /etc/kubernetes/admin.conf $master:/etc/kubernetes/admin.conf
在master节点执行
kubeadm join 172.19.0.95:6443 --token nrbrt0.d1lyjst4cl780oos --discovery-token-ca-cert-hash sha256:88f993ba0c0e829b29e4811a03845892c8df6d82a023715f13d091d536996ea6 --experimental-control-plane
查看Node
[root@master-01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
master-01 Ready master 16m v1.13.1
master-02 Ready master 4m42s v1.13.1
master-03 Ready master 3m47s v1.13.1
token过期后如何加入新节点
kubeadm init执行后输出的kubeadm join所用密钥24h就会过期，我们可以在主节点上创建自己的BootStrap密钥：
首先查看现有密钥
$ kubeadm token list
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
8ewj1p.9r9hcjoqgajrj4gi 23h 2018-06-12T02:51:28Z authentication, The default bootstrap system:
signing token generated by bootstrappers:
'kubeadm init'. kubeadm:
default-node-token
创建新密钥
$ kubeadm token create
5didvk.d09sbcov8ph2amjw
查看密钥的hash值
$ openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
8cb2de97839780a412b93877f8507ad6c94f73add17d5d7058e91741c9d5ec78
在节点上执行
$ kubeadm reset
$ kubeadm join --token 5didvk.d09sbcov8ph2amjw 172.19.0.95:6443 --discovery-token-ca-cert-hash sha256:8cb2de97839780a412b93877f8507ad6c94f73add17d5d7058e91741c9d5ec78
其他Node节点加入集群
kubeadm join 172.19.0.95:6443 --token nrbrt0.d1lyjst4cl780oos --discovery-token-ca-cert-hash sha256:88f993ba0c0e829b29e4811a03845892c8df6d82a023715f13d091d536996ea6
查看集群
$ kubectl get node
NAME STATUS ROLES AGE VERSION
master-01 Ready master 24m v1.13.1
master-02 Ready master 12m v1.13.1
master-03 Ready master 11m v1.13.1
node-01 Ready <none> 46s v1.13.1
node-02 Ready <none> 64s v1.13.1
如果需要删除节点
$ kubectl drain <nodename> --delete-local-data
$ kubectl delete node <nodename>
高可用测试
创建Service
cat << EOF > nginx.yaml
---
apiVersion: v1
kind: Service
metadata:
name: nginx
spec:
selector:
app: nginx
type: NodePort
ports:
- port: 80
nodePort: 31000
name: nginx-port
targetPort: 80
protocol: TCP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
spec:
replicas: 2
selector:
matchLabels:
app: nginx
template:
metadata:
name: nginx
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx
ports:
- containerPort: 80
EOF
kubectl apply -f nginx.yaml
检测DNS
$ kubectl run curl --image=radial/busyboxplus:curl -i --tty
kubectl run --generator=deployment/apps.v1 is DEPRECATED and will be removed in a future version. Use kubectl run --generator=run-pod/v1 or kubectl create instead.
If you don't see a command prompt, try pressing enter.
[ root@curl-66959f6557-trz25:/ ]$ nslookup kubernetes
Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name: kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local
[ root@curl-66959f6557-trz25:/ ]$ nslookup nginx
Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name: nginx
Address 1: 10.99.169.192 nginx.default.svc.cluster.local
[ root@curl-66959f6557-trz25:/ ]$ curl nginx
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
可以测试宕机master节点或者haproxy节点进行测试
