k8s二进制1.14版本添加node节点

发表于 | 分类于 | 浏览 次

前言

新增俩台node节点加进k8s集群。

初始化

安装依赖包

以下操作均在所有机器操作

1
yum install -y epel-release conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget unzip net-tools fuse-devel fuse fuse-libs

关闭防火墙

1
2
3
4
systemctl stop firewalld
systemctl disable firewalld
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT

关闭 swap 分区

1
2
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

关闭 SELinux

1
2
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

加载内核并优化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
modprobe ip_vs_rr
modprobe br_netfilter
cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
cp kubernetes.conf  /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf

ntp

1
ntpdate ntp1.aliyun.com

创建相关目录

1
mkdir -p  /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert

操作

添加hosts

#在之前的master节点添加

1
2
[root@test-master-1 ~]# echo "10.16.8.86  test-node-16" >>/etc/hosts                  
[root@test-master-1 ~]# echo "10.16.8.80  test-node-17" >>/etc/hosts

新加的俩台节点加hosts

1
2
3
4
5
6
7
8
cat >>/etc/hosts<<EOF
10.16.8.10  test-master-1
10.16.8.11  test-master-2
10.16.8.12  test-master-3
10.16.8.13  test-node-15
10.16.8.86  test-node-16
10.16.8.80  test-node-17
EOF

分发秘钥等信息

master节点操作

1
2
ssh-copy-id -i ~/.ssh/id_rsa.pub test-node-16
ssh-copy-id -i ~/.ssh/id_rsa.pub test-node-17

推送CA证书

1
2
3
cd /etc/kubernetes/cert
scp ca.pem ca-config.json test-node-16:/etc/kubernetes/cert/
scp ca.pem ca-config.json test-node-17:/etc/kubernetes/cert/

flannal部署

拷贝二进制文件

1
2
3
cd /opt/k8s/work/flannel
scp flanneld mk-docker-opts.sh test-node-16:/opt/k8s/bin/
scp flanneld mk-docker-opts.sh test-node-17:/opt/k8s/bin/

拷贝flanneld密钥 

1
2
3
4
ssh test-node-16 "mkdir -p /etc/flanneld/cert"
ssh test-node-17 "mkdir -p /etc/flanneld/cert"
scp /etc/flanneld/cert/flanneld*.pem test-node-16:/etc/flanneld/cert
scp /etc/flanneld/cert/flanneld*.pem test-node-17:/etc/flanneld/cert

拷贝flannel启动文件

1
2
3
4
5
scp /etc/systemd/system/flanneld.service test-node-16:/etc/systemd/system/
scp /etc/systemd/system/flanneld.service test-node-17:/etc/systemd/system/
#启动flannel
ssh test-node-16 "systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld"
ssh test-node-17 "systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld"

Kubernetes Node 节点安装Docker

1
2
3
4
5
6
7
8
9
10
yum install -y yum-utils \
  device-mapper-persistent-data \
  lvm2
  
  
yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo
    
yum install docker-ce-19.03.1-3.el7 -y

创建配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
mkdir -p /etc/docker/
cat > /etc/docker/daemon.json <<EOF
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "registry-mirrors": ["https://dockerhub.mirrors.nwafu.edu.cn/","https://docker.mirrors.ustc.edu.cn","https://hub-mirror.c.163.com"],
    "max-concurrent-downloads": 20,
    "live-restore": true,
    "max-concurrent-uploads": 10,
    "debug": true,
    "log-opts": {
      "max-size": "100m",
      "max-file": "5"
    }
}
EOF

修改docker启动配置文件

1
2
3
4
vim /usr/lib/systemd/system/docker.service

EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/containerd.sock
1
2
3
systemctl daemon-reload && systemctl enable docker && systemctl restart docker

systemctl status docker|grep Active

安装kubelet

创建kubelet bootstrap kubeconfig文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#master节点操作
另一台创建文件等和这个同理，这里只列举一个。

cd /opt/k8s/work
export BOOTSTRAP_TOKEN=$(kubeadm token create \
  --description kubelet-bootstrap-token \
  --groups system:bootstrappers:test-node-16 \
  --kubeconfig ~/.kube/config)
# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/cert/ca.pem \
  --embed-certs=true \
  --server=https://10.16.8.14:6443 \
  --kubeconfig=kubelet-bootstrap-test-node-16.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
  --token=${BOOTSTRAP_TOKEN} \
  --kubeconfig=kubelet-bootstrap-test-node-16.kubeconfig
# 设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kubelet-bootstrap \
  --kubeconfig=kubelet-bootstrap-test-node-16.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=kubelet-bootstrap-test-node-16.kubeconfig

分发kubeconfig

1
2
cd /opt/k8s/work
scp kubelet-bootstrap-test-node-16.kubeconfig test-node-16:/etc/kubernetes/kubelet-bootstrap.kubeconfig

查看kubeadm为各个节点创建的token 

1
kubeadm token list --kubeconfig ~/.kube/config

创建和分发kubelet参数配置

1
2
3
4
5
cd /opt/k8s/work
sed -e "s/##NODE_IP##/10.16.8.86/" kubelet-config.yaml.template > kubelet-config-10.16.8.86.yaml.template
sed -e "s/##NODE_IP##/10.16.8.80/" kubelet-config.yaml.template > kubelet-config-10.16.8.80.yaml.template
scp kubelet-config-10.16.8.86.yaml.template root@test-node-16:/etc/kubernetes/kubelet-config.yaml
scp kubelet-config-10.16.8.80.yaml.template root@test-node-17:/etc/kubernetes/kubelet-config.yaml

拷贝kubelet启动文件和命令

1
2
3
4
5
6
7
8
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
sed -e "s/##NODE_NAME##/test-node-16/" kubelet.service.template > kubelet-test-node-16.service
sed -e "s/##NODE_NAME##/test-node-17/" kubelet.service.template > kubelet-test-node-17.service
scp kubelet-test-node-16.service root@test-node-16:/etc/systemd/system/kubelet.service
scp kubelet-test-node-17.service root@test-node-17:/etc/systemd/system/kubelet.service
scp /opt/k8s/bin/kubelet test-node-16:/opt/k8s/bin/
scp /opt/k8s/bin/kubelet test-node-17:/opt/k8s/bin/

启动kubelet 

1
2
3
4
5
6
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
ssh root@test-node-16 "mkdir -p ${K8S_DIR}/kubelet/kubelet-plugins/volume/exec/"
ssh root@test-node-16 "systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet"
ssh root@test-node-17 "mkdir -p ${K8S_DIR}/kubelet/kubelet-plugins/volume/exec/"
ssh root@test-node-17 "systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet"

签注证书

1
kubectl get csr | grep Pending | awk '{print $1}' | xargs kubectl certificate approve

安装kube-proxy

推送kube-proxy二进制启动文件和kubeconfig文件

1
2
3
4
5
cd /opt/k8s/work/
scp kubernetes/server/bin/kube-proxy test-node-16:/opt/k8s/bin/
scp kubernetes/server/bin/kube-proxy test-node-17:/opt/k8s/bin/
scp kube-proxy.kubeconfig root@test-node-16:/etc/kubernetes/
scp kube-proxy.kubeconfig root@test-node-17:/etc/kubernetes/

创建和分发kube-proxy配置文件 

1
2
3
4
5
6
7
cd /opt/k8s/work/
sed -e "s/##NODE_NAME##/test-node-16/" -e "s/##NODE_IP##/10.16.8.86/" kube-proxy-config.yaml.template > kube-proxy-config-test-node-16.yaml.template
sed -e "s/##NODE_NAME##/test-node-17/" -e "s/##NODE_IP##/10.16.8.80/" kube-proxy-config.yaml.template > kube-proxy-config-test-node-17.yaml.template
scp kube-proxy-config-test-node-16.yaml.template root@test-node-16:/etc/kubernetes/kube-proxy-config.yaml
scp kube-proxy-config-test-node-17.yaml.template root@test-node-17:/etc/kubernetes/kube-proxy-config.yaml
scp kube-proxy.service root@test-node-16:/etc/systemd/system/
scp kube-proxy.service root@test-node-17:/etc/systemd/system/

启动kube-proxy服务 

1
2
3
4
5
6
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
ssh root@test-node-16 "mkdir -p ${K8S_DIR}/kube-proxy"
ssh root@test-node-16 "systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy"
ssh root@test-node-17 "mkdir -p ${K8S_DIR}/kube-proxy"
ssh root@test-node-17 "systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy"
Donate