kubeadm高可用集群安装

发表于 | 分类于 | 浏览 次

前提

最近总有小伙伴说kubeadm的高可用集群怎么安装，故写了这篇文章。

创建高可用首先先有一个 Master 节点，然后再让其他服务器加入组成三个 Master 节点高可用，然后再将工作节点 Node 加入。

Kuberadm 功能

  • kubeadm init： 启动一个 Kubernetes 主节点
  • kubeadm join： 启动一个 Kubernetes 工作节点并且将其加入到集群
  • kubeadm upgrade： 更新一个 Kubernetes 集群到新版本
  • kubeadm config： 如果使用 v1.7.x 或者更低版本的 kubeadm 初始化集群，您需要对集群做一些配置以便使用 kubeadm upgrade 命令
  • kubeadm token： 管理 kubeadm join 使用的令牌
  • kubeadm reset： 还原 kubeadm init 或者 kubeadm join 对主机所做的任何更改
  • kubeadm version： 打印 kubeadm 版本
  • kubeadm alpha： 预览一组可用的新功能以便从社区搜集反馈

初始化

主机名称解析

分别进入不同服务器，进入 /etc/hosts 进行编辑

1
2
3
4
10.16.16.100    master.sy.com      k8s-vip
10.16.16.19    master01.sy.com    sy1
10.16.16.20    master02.sy.com    sy2
10.16.16.28    master03.sy.com    sy3

我用的openstack虚拟机，vip需要提前开通，具体开通vip地址可以看我另一篇文章

https://shenshengkun.github.io/posts/dk456akf.html

安装依赖包

以下操作均在所有机器操作

1
yum install -y epel-release conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget unzip net-tools

关闭防火墙

1
2
3
4
systemctl stop firewalld
systemctl disable firewalld
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT

关闭 swap 分区

1
2
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

关闭 SELinux

1
2
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

设置系统参数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
modprobe ip_vs_rr
modprobe br_netfilter
cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
cp kubernetes.conf  /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf

Keepalived安装

安装

1
yum install -y keepalived

配置Keepalived

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
cat <<EOF > /etc/keepalived/keepalived.conf
! Configuration File for keepalived

# 主要是配置故障发生时的通知对象以及机器标识。
global_defs {
   # 标识本节点的字条串，通常为 hostname，但不一定非得是 hostname。故障发生时，邮件通知会用到。
   router_id LVS_k8s
}

# 用来做健康检查的，当时检查失败时会将 vrrp_instance 的 priority 减少相应的值。
vrrp_script check_haproxy {
    script "killall -0 haproxy"   #根据进程名称检测进程是否存活
    interval 3
    weight -2
    fall 10
    rise 2
}

# rp_instance用来定义对外提供服务的 VIP 区域及其相关属性。
vrrp_instance VI_1 {
    state MASTER   #当前节点为MASTER，其他两个节点设置为BACKUP
    interface eth0 #改为自己的网卡
    virtual_router_id 51
    priority 250
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 3sqP05dQgMSlzrxHj
    }
    virtual_ipaddress {
        10.16.16.100   #虚拟ip，即VIP
    }
    track_script {
        check_haproxy
    }

}
EOF

启动

1
systemctl enable keepalived && systemctl start keepalived

安装haproxy

1
yum install -y haproxy

此处的haproxy为apiserver提供反向代理，haproxy将所有请求轮询转发到每个master节点上。相对于仅仅使用keepalived主备模式仅单个master节点承载流量，这种方式更加合理、健壮。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
cat > /etc/haproxy/haproxy.cfg << EOF
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2
    
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon 
       
    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------  
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
#---------------------------------------------------------------------
# kubernetes apiserver frontend which proxys to the backends
#--------------------------------------------------------------------- 
frontend kubernetes-apiserver
    mode                 tcp
    bind                 *:8443
    option               tcplog
    default_backend      kubernetes-apiserver    
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiserver
    mode        tcp
    balance     roundrobin
    server      master01.sy.com   10.16.16.19:6443 check
    server      master02.sy.com   10.16.16.20:6443 check
    server      master03.sy.com   10.16.16.28:6443 check
#---------------------------------------------------------------------
# collection haproxy statistics message
#---------------------------------------------------------------------
listen stats
    bind                 *:1080
    stats auth           admin:admin
    stats refresh        5s
    stats realm          HAProxy\ Statistics
    stats uri            /admin?stats
EOF

启动haproxy

1
systemctl enable haproxy && systemctl start haproxy

安装Docker (所有节点)

1
2
3
4
5
6
7
8
9
yum install -y yum-utils \
  device-mapper-persistent-data \
  lvm2
  
yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo
    
yum install docker-ce-19.03.1-3.el7 -y

安加速，并且修改cgroup driver

根据文档CRI installation中的内容，对于使用systemd作为init system的Linux的发行版，使用systemd作为docker的cgroup driver可以确保服务器节点在资源紧张的情况更加稳定，因此这里修改各个节点上docker的cgroup driver为systemd。

1
2
3
4
5
6
7
8
9
[root@sy1 ~]# systemctl start docker
[root@sy1 ~]# cat /etc/docker/daemon.json 
{"exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirrors": ["https://dockerhub.mirrors.nwafu.edu.cn/"],
    "bip": "192.17.10.1/24"
}
[root@sy1 ~]# systemctl restart docker
[root@sy1 ~]# systemctl enable docker 
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.

安装kubeadm、kubelet

配置yum源

1
2
3
4
5
6
7
8
9
10
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
        http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

查看kubelet版本列表

1
yum list kubelet --showduplicates | sort -r

安装 kubeadm、kubelet

1
2
yum install -y kubelet-1.15.3-0 kubeadm-1.15.3-0
systemctl enable kubelet.service && systemctl start kubelet

初始化第一个kubernetes master节点

创建kubeadm配置的yaml文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
cat > kubeadm-config.yaml << EOF
apiServer:
  certSANs:
    - master01.sy.com
    - master02.sy.com
    - master03.sy.com
    - master.sy.com
    - 10.16.16.100
    - 10.16.16.19
    - 10.16.16.20
    - 10.16.16.28
    - 127.0.0.1
  extraArgs:
    authorization-mode: Node,RBAC
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "master.sy.com:8443"
controllerManager: {}
dns: 
  type: CoreDNS
etcd:
  local:    
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.15.3
networking: 
  dnsDomain: cluster.local  
  podSubnet: 192.160.0.0/16
  serviceSubnet: 192.168.0.0/17
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
EOF

初始化第一个master节点

1
kubeadm init --config kubeadm-config.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities 
and service account keys on each node and then running the following as root:

  kubeadm join master.sy.com:8443 --token ltjuyu.2otdxrrsy4ku6lri \
    --discovery-token-ca-cert-hash sha256:f6562358b50a516e7814136c73b57827626a75ae82389fbe0c70b6328700ee4a \
    --control-plane       

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join master.sy.com:8443 --token ltjuyu.2otdxrrsy4ku6lri \
    --discovery-token-ca-cert-hash sha256:f6562358b50a516e7814136c73b57827626a75ae82389fbe0c70b6328700ee4a

kubeconfig

1
2
3
[root@sy1 ~]# mkdir -p $HOME/.kube
[root@sy1 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@sy1 ~]# chown $(id -u):$(id -g) $HOME/.kube/config

安装网络插件

配置flannel插件的yaml文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
cat > kube-flannel.yaml << EOF
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
rules:
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "192.160.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-amd64
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      hostNetwork: true
      nodeSelector:
        beta.kubernetes.io/arch: amd64
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: registry.cn-shenzhen.aliyuncs.com/cp_m/flannel:v0.10.0-amd64
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: registry.cn-shenzhen.aliyuncs.com/cp_m/flannel:v0.10.0-amd64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: true
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
EOF

部署

1
kubectl apply -f kube-flannel.yaml

加入集群

Master加入集群构成高可用

复制文件，从master1上复制到2和3

1
2
3
4
5
6
7
8
9
ssh root@master02.sy.com mkdir -p /etc/kubernetes/pki/etcd
scp /etc/kubernetes/admin.conf root@master02.sy.com:/etc/kubernetes
scp /etc/kubernetes/pki/{ca.*,sa.*,front-proxy-ca.*} root@master02.sy.com:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca.* root@master02.sy.com:/etc/kubernetes/pki/etcd

ssh root@master03.sy.com mkdir -p /etc/kubernetes/pki/etcd
scp /etc/kubernetes/admin.conf root@master03.sy.com:/etc/kubernetes
scp /etc/kubernetes/pki/{ca.*,sa.*,front-proxy-ca.*} root@master03.sy.com:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca.* root@master03.sy.com:/etc/kubernetes/pki/etcd

master节点加入集群

1
kubeadm join master.sy.com:8443 --token ltjuyu.2otdxrrsy4ku6lri --discovery-token-ca-cert-hash sha256:f6562358b50a516e7814136c73b57827626a75ae82389fbe0c70b6328700ee4a --experimental-control-plane

结果如下

1
2
3
4
5
[root@sy1 ~]# kubectl get nodes
NAME              STATUS   ROLES    AGE     VERSION
master01.sy.com   Ready    master   28m     v1.15.3
master02.sy.com   Ready    master   3m12s   v1.15.3
master03.sy.com   Ready    master   24s     v1.15.3

node加入集群

1
2
kubeadm join master.sy.com:8443 --token ltjuyu.2otdxrrsy4ku6lri \
    --discovery-token-ca-cert-hash sha256:f6562358b50a516e7814136c73b57827626a75ae82389fbe0c70b6328700ee4a
Donate