二进制部署 k8s

kubernetes为什么要叫k8s、因为你要搭建8遍才有可能成功一次、所以才叫k8s(手动滑稽、狗头保命)

环境说明：
172.16.1.11 master01 m1
172.16.1.12 master02 m2
172.16.1.13 master03 m3
172.16.1.14 node01 n1
172.16.1.15 node02 n2

172.16.1.33 为vip 地址

预留ip方便日后扩容、要不然日后扩容还得重新生成证书
172.16.1.15
172.16.1.16
172.16.1.17

一 基本环境配置(所有节点操作)：
1 修改hosts解析
cat /etc/hosts
172.16.1.11 master01 m1
172.16.1.12 master02 m2
172.16.1.13 master03 m3
172.16.1.14 node01 n1
172.16.1.15 node02 n2

2 配置yum源。docker、k8s、base、eper
cat > Centos7-Base-163.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
EOF

cat > docker-ce << EOF
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg

[docker-ce-stable-debuginfo]
name=Docker CE Stable - Debuginfo $basearch
baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/stable
enabled=0
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg

[docker-ce-stable-source]
name=Docker CE Stable - Sources
baseurl=https://download.docker.com/linux/centos/$releasever/source/stable
enabled=0
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg

[docker-ce-test]
name=Docker CE Test - $basearch
baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/test
enabled=0
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg

[docker-ce-test-debuginfo]
name=Docker CE Test - Debuginfo $basearch
baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/test
enabled=0
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg

[docker-ce-test-source]
name=Docker CE Test - Sources
baseurl=https://download.docker.com/linux/centos/$releasever/source/test
enabled=0
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg

[docker-ce-nightly]
name=Docker CE Nightly - $basearch
baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/nightly
enabled=0
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg

[docker-ce-nightly-debuginfo]
name=Docker CE Nightly - Debuginfo $basearch
baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/nightly
enabled=0
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg

[docker-ce-nightly-source]
name=Docker CE Nightly - Sources
baseurl=https://download.docker.com/linux/centos/$releasever/source/nightly
enabled=0
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg
EOF

3 安装必备的工具
yum -y install wget jq psimsc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git

4 关闭防火墙、selinux、swap
setenforce 0
sed -i "s/SELINUX=.*/SELINUX=disabled/g" /etc/selinux/config
systemctl stop firewalld
sed -i.bak 's/^.*centos-swap/#&/g' /etc/fstab

5 配置时间服务器
yum -y install ntpdate
ntpdate ntp.aliyun.com
echo '*/5 * * * * /usr/sbin/ntpdate ntp.aliyun.com' >> /var/spool/cron/root

6 停止NetworkManager
systemctl stop NetworkManager && systemctl disable NetworkManager

7 内核参数优化
cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp.keepaliv.probes = 3
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp.max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp.max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.top_timestamps = 0
net.core.somaxconn = 16384
EOF

# 立即生效
sysctl --system

8 ssh免密所有节点(在master1上面执行)
ssh-keygen
ssh-copy-id 127.0.0.1

#分发秘钥到所有节点
for i in m2 m3 n1 n2;do
ssh $i "cd /root && mkdir .ssh"
scp /root/.ssh/* $i:/root/.ssh/
ssh $i "chmod 600 /root/.ssh/*"
done

9 升级内核(如果服务器太老旧不建议升级到太新，升级到4.18、4.19即可)。
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
#查看可用的内核包
yum --enablerepo=elrepo-kernel install kernel-ml
#安装最新版本内核
sudo awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
#查看可用内核
grub2-set-default 0
#设置默认内核，0是上面查询出来的内核
grub2-mkconfig -o /boot/grub2/grub.cfg
#生成grub文件
reboot
10 所有节点安装配置ipvsadm
yum -y install ipvsadm
vim /etc/modules-load.d/ipvs.conf
# 加入以下内容
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack_ipv4
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
systemctl enable --now systemd-modules-load.service

11 安装docker
yum install -y yum-utils device-mapper-persistent-data lvm2 docker-ce
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://8mh75mhz.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload ; systemctl restart docker;systemctl enable --now docker.service

二 master各个组件部署
1 阐述master、node各个需要部署的组件
master：
etcd
kube-apiserver
kube-scheduler
kubelet
kube-proxy
kube-controller-manager
网络插件flanneld
CoreDNS
node：
kubelet
kube-proxy
etcd
网络插件flanneld

2 下载实验所需安装包。
cd /opt && git clone https://gitee.com/zhang-zhi-bo/k8s.git

3 安装证书生成工具
mv /opt/k8s/cfssl* /usr/local/bin/
chmod +x /usr/local/bin/

4 etcd集群证书
4.1 创建根证书
mkdir -p /opt/cert/ca

cat > /opt/cert/ca/ca-config.json < {
"signing": {
"default": {
"expiry": "8760h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "8760h"
}
}
}
}
EOF

4.1.1 详解：
default是默认策略，指定证书默认有效期是1年
profiles是定义使用场景，这里只是kubernetes，其实可以定义多个场景，分别指定不同的过期时间,使用场景等参数,后续签名证书时使用某个profile;
signing: 表示该证书可用于签名其它证书,生成的ca.pem证书
server auth: 表示client 可以用该CA 对server 提供的证书进行校验;
client auth: 表示server 可以用该CA 对client 提供的证书进行验证。

4.2 创建根证书请求文件
cat > /opt/cert/ca/ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names":[{
"C": "CN",
"ST": "ShangHai",
"L": "ShangHai"
}]
}
EOF

4.3 生成根证书
cd /opt/cert/ca
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2020/08/28 23:51:50 [INFO] generating a new CA key and certificate from CSR
2020/08/28 23:51:50 [INFO] generate received request
2020/08/28 23:51:50 [INFO] received CSR
2020/08/28 23:51:50 [INFO] generating key: rsa-2048
2020/08/28 23:51:50 [INFO] encoded CSR
2020/08/28 23:51:50 [INFO] signed certificate with serial number 66427391707536599498414068348802775591392574059

[root@master01 etcd]#ll
-rw-r--r-- 1 root root 282 8月 28 23:41 ca-config.json
-rw-r--r-- 1 root root 1013 8月 28 23:51 ca.csr
-rw-r--r-- 1 root root 196 8月 28 23:41 ca-csr.json
-rw------- 1 root root 1675 8月 28 23:51 ca-key.pem
-rw-r--r-- 1 root root 1334 8月 28 23:51 ca.pem

5 部署etcd集群
5.1 集群规划
etcd01 172.16.1.11
etcd02 172.16.1.12
etcd03 172.16.1.13

5.2 创建etcd证书 (16、17、18为预留ip方便日后扩容、要不然日后扩容还得重新生成证书)
mkdir -p /opt/cert/etcd
cd /opt/cert/etcd

cat > etcd-csr.json << EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"172.16.1.11",
"172.16.1.12",
"172.16.1.13",
"172.16.1.14",
"172.16.1.15"，
"172.16.1.16"，
"172.16.1.17"，
"172.16.1.18"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "ShangHai",
"L": "ShangHai"
}
]
}
EOF

cfssl gencert -ca=../ca/ca.pem -ca-key=../ca/ca-key.pem -config=../ca/ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
2020/08/29 00:02:20 [INFO] generate received request
2020/08/29 00:02:20 [INFO] received CSR
2020/08/29 00:02:20 [INFO] generating key: rsa-2048
2020/08/29 00:02:20 [INFO] encoded CSR
2020/08/29 00:02:20 [INFO] signed certificate with serial number 71348009702526539124716993806163559962125770315
[root@master01 etcd]# ll
总用量 16
-rw-r--r-- 1 root root 1074 8月 29 00:02 etcd.csr
-rw-r--r-- 1 root root 352 8月 28 23:59 etcd-csr.json
-rw------- 1 root root 1675 8月 29 00:02 etcd-key.pem
-rw-r--r-- 1 root root 1460 8月 29 00:02 etcd.pem

5.3 分发证书
for ip in master01 master02 master03
do
ssh root@${ip} "mkdir -pv /etc/etcd/ssl"
scp ../ca/ca*.pem root@${ip}:/etc/etcd/ssl
scp ./etcd*.pem root@${ip}:/etc/etcd/ssl
done

for ip in master01 master02 master03; do
ssh root@${ip} "ls -l /etc/etcd/ssl";
done

5.4 部署etcd
tar -xf /opt/k8s/etcd-v3.3.24-linux-amd64.tar.gz
for i in master02 master01 master03
do
scp /opt/k8s/etcd-v3.3.24-linux-amd64/etcd* root@$i:/usr/local/bin/
hostname && ssh $i "etcd --version"
done

5.5 注册etcd
mkdir -pv /etc/kubernetes/conf/etcd
ETCD_NAME=`hostname`
INTERNAL_IP=`hostname -i`
INITIAL_CLUSTER=master01=https://172.16.1.11:2380,master02=https://172.16.1.12:2380,master03=https://172.16.1.13:2380

cat << EOF | sudo tee /usr/lib/systemd/system/etcd.service
[Unit]
Description=etcd
Documentation=https://github.com/coreos

[Service]
ExecStart=/usr/local/bin/etcd \\
--name ${ETCD_NAME} \\
--cert-file=/etc/etcd/ssl/etcd.pem \\
--key-file=/etc/etcd/ssl/etcd-key.pem \\
--peer-cert-file=/etc/etcd/ssl/etcd.pem \\
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \\
--trusted-ca-file=/etc/etcd/ssl/ca.pem \\
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \\
--peer-client-cert-auth \\
--client-cert-auth \\
--initial-advertise-peer-urls https://${INTERNAL_IP}:2380 \\
--listen-peer-urls https://${INTERNAL_IP}:2380 \\
--listen-client-urls https://${INTERNAL_IP}:2379,https://127.0.0.1:2379 \\
--advertise-client-urls https://${INTERNAL_IP}:2379 \\
--initial-cluster-token etcd-cluster \\
--initial-cluster ${INITIAL_CLUSTER} \\
--initial-cluster-state new \\
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

5.6 三台节点启动etcd
for i in m1 m2 m3;do
ssh $i "systemctl enable --now etcd"
done

5.7 测试
ETCDCTL_API=3 etcdctl \
--cacert=/etc/etcd/ssl/etcd.pem \
--cert=/etc/etcd/ssl/etcd.pem \
--key=/etc/etcd/ssl/etcd-key.pem \
--endpoints="https://172.16.1.11:2379,https://172.16.1.12:2379,https://172.16.1.13:2379" \
member list --write-out='table'

| 2cd4d60db6db4371 | started | master01 | https://172.16.1.11:2380 | https://172.16.1.11:2379 |
| 3f62a4d067dd828a | started | master03 | https://172.16.1.13:2380 | https://172.16.1.13:2379 |
| 6ac10dc079d59789 | started | master02 | https://172.16.1.12:2380 | https://172.16.1.12:2379 |

ETCDCTL_API=3 etcdctl \
--cacert=/etc/etcd/ssl/etcd.pem \
--cert=/etc/etcd/ssl/etcd.pem \
--key=/etc/etcd/ssl/etcd-key.pem \
--endpoints="https://172.16.1.11:2379,https://172.16.1.12:2379,https://172.16.1.13:2379" \
endpoint status --write-out='table'

| https://172.16.1.11:2379 | 2cd4d60db6db4371 | 3.3.24 | 3.9 MB | false | 103 | 325691 |
| https://172.16.1.12:2379 | 6ac10dc079d59789 | 3.3.24 | 3.9 MB | false | 103 | 325691 |
| https://172.16.1.13:2379 | 3f62a4d067dd828a | 3.3.24 | 3.9 MB | true | 103 | 325691 |
6 部署master集群证书
6.1创建集群证书
[root@master01 k8s]# mkdir /opt/cert/k8s
[root@master01 k8s]# cd /opt/cert/k8s
[root@master01 k8s]# pwd
/opt/cert/k8s
[root@master01 k8s]# cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF

[root@master01 k8s]# cat > ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "ShangHai",
"ST": "ShangHai"
}
]
}
EOF

[root@master01 k8s]# ll
-rw-r--r-- 1 root root 294 Sep 13 19:59 ca-config.json
-rw-r--r-- 1 root root 212 Sep 13 20:01 ca-csr.json

6.2 生成根证书
[root@master01 k8s]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2020/09/13 20:01:45 [INFO] generating a new CA key and certificate from CSR
2020/09/13 20:01:45 [INFO] generate received request
2020/09/13 20:01:45 [INFO] received CSR
2020/09/13 20:01:45 [INFO] generating key: rsa-2048
2020/09/13 20:01:46 [INFO] encoded CSR
2020/09/13 20:01:46 [INFO] signed certificate with serial number 588993429584840635805985813644877690042550093427

[root@master01 k8s]# ll
total 20
-rw-r--r-- 1 root root 294 Sep 13 19:59 ca-config.json
-rw-r--r-- 1 root root 960 Sep 13 20:01 ca.csr
-rw-r--r-- 1 root root 212 Sep 13 20:01 ca-csr.json
-rw------- 1 root root 1679 Sep 13 20:01 ca-key.pem
-rw-r--r-- 1 root root 1273 Sep 13 20:01 ca.pem

6.3 签发kube-apiserver证书
[root@master01 k8s]# cat > server-csr.json << EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"172.16.1.11",
"172.16.1.12",
"172.16.1.13",
"172.16.1.16",
"172.16.1.17",
"172.16.1.18",
"172.16.1.33",
"10.96.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "ShangHai",
"ST": "ShangHai"
}
]
}
EOF

[root@master01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
2020/08/29 12:29:41 [INFO] generate received request
2020/08/29 12:29:41 [INFO] received CSR
2020/08/29 12:29:41 [INFO] generating key: rsa-2048
2020/08/29 12:29:41 [INFO] encoded CSR
2020/08/29 12:29:41 [INFO] signed certificate with serial number 701177072439793091180552568331885323625122463841

6.4 签发kube-kube-controller-manager 证书
cat > kube-controller-manager-csr.json << EOF
{
"CN": "system:kube-controller-manager",
"hosts": [
"127.0.0.1",
"172.16.1.11",
"172.16.1.12",
"172.16.1.13",
"172.16.1.13",
"172.16.1.16",
"172.16.1.17",
"172.16.1.18",
"172.16.1.33"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-controller-manager",
"OU": "System"
}
]
}
EOF

[root@master01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
2020/08/29 12:40:21 [INFO] generate received request
2020/08/29 12:40:21 [INFO] received CSR
2020/08/29 12:40:21 [INFO] generating key: rsa-2048
2020/08/29 12:40:22 [INFO] encoded CSR
2020/08/29 12:40:22 [INFO] signed certificate with serial number 464924254532468215049650676040995556458619239240

6.5 签发kube-scheduler证书

cat > kube-scheduler-csr.json << EOF
{
"CN": "system:kube-scheduler",
"hosts": [
"127.0.0.1",
"172.16.1.11",
"172.16.1.12",
"172.16.1.13",
"172.16.1.16",
"172.16.1.17",
"172.16.1.18",
"172.16.1.33"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-scheduler",
"OU": "System"
}
]
}
EOF

[root@master01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
2020/08/29 12:42:29 [INFO] generate received request
2020/08/29 12:42:29 [INFO] received CSR
2020/08/29 12:42:29 [INFO] generating key: rsa-2048
2020/08/29 12:42:29 [INFO] encoded CSR
2020/08/29 12:42:29 [INFO] signed certificate with serial number 420546069405900774170348492061478728854870171400

6.6 签发kube-proxy证书
cat > kube-proxy-csr.json << EOF
{
"CN":"system:kube-proxy",
"hosts":[],
"key":{
"algo":"rsa",
"size":2048
},
"names":[
{
"C":"CN",
"L":"BeiJing",
"ST":"BeiJing",
"O":"system:kube-proxy",
"OU":"System"
}
]
}
EOF

[root@master01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
2020/08/29 12:45:11 [INFO] generate received request
2020/08/29 12:45:11 [INFO] received CSR
2020/08/29 12:45:11 [INFO] generating key: rsa-2048
2020/08/29 12:45:11 [INFO] encoded CSR
2020/08/29 12:45:11 [INFO] signed certificate with serial number 39717174368771783903269928946823692124470234079
2020/08/29 12:45:11 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management

6.7 签发管理用户证书
cat > admin-csr.json << EOF
{
"CN":"admin",
"key":{
"algo":"rsa",
"size":2048
},
"names":[
{
"C":"CN",
"L":"BeiJing",
"ST":"BeiJing",
"O":"system:masters",
"OU":"System"
}
]
}
EOF

[root@master01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
2020/08/29 12:50:46 [INFO] generate received request
2020/08/29 12:50:46 [INFO] received CSR
2020/08/29 12:50:46 [INFO] generating key: rsa-2048
2020/08/29 12:50:46 [INFO] encoded CSR
2020/08/29 12:50:46 [INFO] signed certificate with serial number 247283053743606613190381870364866954196747322330
2020/08/29 12:50:46 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management

6.8 颁发证书
mkdir -pv /etc/kubernetes/ssl
cp -p ./{ca*pem,server*pem,kube-controller-manager*pem,kube-scheduler*.pem,kube-proxy*pem,admin*.pem} /etc/kubernetes/ssl

for i in master02 master03; do
ssh root@$i "mkdir -pv /etc/kubernetes/ssl"
scp /etc/kubernetes/ssl/* root@$i:/etc/kubernetes/ssl
done

for i in master02 master03; do
ssh root@$i "mkdir -pv /etc/kubernetes/ssl"
scp /etc/kubernetes/ssl/* root@$i:/etc/kubernetes/ssl
done

6.9 分发组件
for i in m1 m2 m3;do
scp /opt/k8s/bin/kube* $i:/usr/local/bin/
ssh $i "chmod +x /usr/local/bin/kube*"
done

6.10 创建集群配置文件
6.10.1 创建配置文件临时存放目录
mkdir /opt/conf && cd /opt/conf

6.10.2 创建kube-controller-manager.kubeconfig文件
export KUBE_APISERVER="https://172.16.1.33:8443"

# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-controller-manager.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials "kube-controller-manager" \
--client-certificate=/etc/kubernetes/ssl/kube-controller-manager.pem \
--client-key=/etc/kubernetes/ssl/kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=kube-controller-manager.kubeconfig

# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="kube-controller-manager" \
--kubeconfig=kube-controller-manager.kubeconfig

# 配置默认上下文
kubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig

6.10.3 创建kube-scheduler.kubeconfig文件
export KUBE_APISERVER="https://172.16.1.33:8443"

# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-scheduler.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials "kube-scheduler" \
--client-certificate=/etc/kubernetes/ssl/kube-scheduler.pem \
--client-key=/etc/kubernetes/ssl/kube-scheduler-key.pem \
--embed-certs=true \
--kubeconfig=kube-scheduler.kubeconfig

# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="kube-scheduler" \
--kubeconfig=kube-scheduler.kubeconfig

# 配置默认上下文
kubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig

6.10.4 创建kube-proxy.kubeconfig文件
export KUBE_APISERVER="https://172.16.1.33:8443"

# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials "kube-proxy" \
--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig

# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="kube-proxy" \
--kubeconfig=kube-proxy.kubeconfig

# 配置默认上下文
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

6.10.5 创建admin.kubeconfig文件
export KUBE_APISERVER="https://172.16.1.33:8443"

# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=admin.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials "admin" \
--client-certificate=/etc/kubernetes/ssl/admin.pem \
--client-key=/etc/kubernetes/ssl/admin-key.pem \
--embed-certs=true \
--kubeconfig=admin.kubeconfig

# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="admin" \
--kubeconfig=admin.kubeconfig

# 配置默认上下文
kubectl config use-context default --kubeconfig=admin.kubeconfig

6.10.6 生成TLS bootstrapping所需token
# 必须要用自己机器创建的Token
TLS_BOOTSTRAPPING_TOKEN=`head -c 16 /dev/urandom | od -An -t x | tr -d ' '`

cat > token.csv << EOF
${TLS_BOOTSTRAPPING_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

[root@master01 k8s]# TLS_BOOTSTRAPPING_TOKEN=`head -c 16 /dev/urandom | od -An -t x | tr -d ' '`
[root@master01 k8s]# cat > token.csv << EOF
${TLS_BOOTSTRAPPING_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
[root@master01 k8s]# cat token.csv
1b076dcc88e04d64c3a9e7d7a1586fe5,kubelet-bootstrap,10001,"system:kubelet-bootstrap"

#创建TLS Bootstrapping集群配置文件

export KUBE_APISERVER="https://172.16.1.33:8443"

# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kubelet-bootstrap.kubeconfig

# 设置客户端认证参数,此处token必须用上叙token.csv中的token
kubectl config set-credentials "kubelet-bootstrap" \
--token=83abd8913ff0ee02cf4ea800a95f5994 \
--kubeconfig=kubelet-bootstrap.kubeconfig

# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="kubelet-bootstrap" \
--kubeconfig=kubelet-bootstrap.kubeconfig

# 配置默认上下文
kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig

6.11 分发配置文件
for i in master01 master02 master03;
do
ssh root@$i "mkdir -p /etc/kubernetes/cfg";
scp token.csv kube-scheduler.kubeconfig kube-controller-manager.kubeconfig admin.kubeconfig kube-proxy.kubeconfig kubelet-bootstrap.kubeconfig root@$i:/etc/kubernetes/cfg;
done

6.12 部署api-server
KUBE_APISERVER_IP=`hostname -i`

cat > /etc/kubernetes/cfg/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--advertise-address=${KUBE_APISERVER_IP} \\
--default-not-ready-toleration-seconds=360 \\
--default-unreachable-toleration-seconds=360 \\
--max-mutating-requests-inflight=2000 \\
--max-requests-inflight=4000 \\
--default-watch-cache-size=200 \\
--delete-collection-workers=2 \\
--bind-address=0.0.0.0 \\
--secure-port=6443 \\
--allow-privileged=true \\
--service-cluster-ip-range=10.96.0.0/16 \\
--service-node-port-range=10-52767 \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \\
--authorization-mode=RBAC,Node \\
--enable-bootstrap-token-auth=true \\
--token-auth-file=/etc/kubernetes/cfg/token.csv \\
--kubelet-client-certificate=/etc/kubernetes/ssl/server.pem \\
--kubelet-client-key=/etc/kubernetes/ssl/server-key.pem \\
--tls-cert-file=/etc/kubernetes/ssl/server.pem \\
--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \\
--client-ca-file=/etc/kubernetes/ssl/ca.pem \\
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/var/log/kubernetes/k8s-audit.log \\
--etcd-servers=https://172.16.1.11:2379,https://172.16.1.12:2379,https://172.16.1.13:2379 \\
--etcd-cafile=/etc/etcd/ssl/ca.pem \\
--etcd-certfile=/etc/etcd/ssl/etcd.pem \\
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem"
EOF

6.13 注册kube-apiserver服务
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-apiserver.conf
ExecStart=/usr/local/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF

#分发kube-apiserver配置文件
for i in master02 master03;
do
scp /usr/lib/systemd/system/kube-apiserver.service root@$i:/usr/lib/systemd/system/kube-apiserver.service
done

#启动

for i in m1 m2 m3; do
# 创建kubernetes日志目录
mkdir -p /var/log/kubernetes/
systemctl daemon-reload
systemctl enable --now kube-apiserver
done

#kube-apiserver 全部起来以后在执行
kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap

6.14 高可用部署api-server
6.14.1 安装haproxy、keepalived
yum install -y keepalived haproxy

6.14.2 配置haproxy服务
cat > /etc/haproxy/haproxy.cfg < global
maxconn 2000
ulimit-n 16384
log 127.0.0.1 local0 err
stats timeout 30s

defaults
log global
mode http
option httplog
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-request 15s
timeout http-keep-alive 15s

frontend monitor-in
bind *:33305
mode http
option httplog
monitor-uri /monitor

listen stats
bind *:8006
mode http
stats enable
stats hide-version
stats uri /stats
stats refresh 30s
stats realm Haproxy\ Statistics
stats auth admin:admin

frontend k8s-master
bind 0.0.0.0:8443
bind 127.0.0.1:8443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend k8s-master

backend k8s-master
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server master01 172.16.1.11:6443 check inter 2000 fall 2 rise 2 weight 100
server master02 172.16.1.12:6443 check inter 2000 fall 2 rise 2 weight 100
server master03 172.16.1.13:6443 check inter 2000 fall 2 rise 2 weight 100
EOF

6.14.3 分发配置文件
for i in master01 master02 master03;
do
ssh root@$i "mv /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg_bak"
scp haproxy.cfg root@$i:/etc/haproxy/haproxy.cfg
done

#打开策略链接
setsebool -P haproxy_connect_any=1

6.14.4 配置keepalived
mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak

cd /etc/keepalived

cat > /etc/keepalived/keepalived.conf < ! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_kubernetes {
script "/etc/keepalived/check_kubernetes.sh"
interval 2
weight -5
fall 3
rise 2
}
vrrp_instance VI_1 {
state MASTER
interface eth0
mcast_src_ip 172.16.1.11
virtual_router_id 51
priority 100
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
172.16.1.33
}
# track_script {
# chk_kubernetes
# }
}
EOF

#分发配置文件
for i in master02 master03;
do
ssh root@$i "mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak"
scp /etc/keepalived/keepalived.conf root@$i:/etc/keepalived/keepalived.conf
done

#配置master02 节点
sed -i 's#state MASTER#state BACKUP#g' /etc/keepalived/keepalived.conf
sed -i 's#172.16.1.11#172.16.1.12#g' /etc/keepalived/keepalived.conf
sed -i 's#priority 100#priority 90#g' /etc/keepalived/keepalived.conf

#配置master03 节点
sed -i 's#state MASTER#state BACKUP#g' /etc/keepalived/keepalived.conf
sed -i 's#172.16.1.11#172.16.1.13#g' /etc/keepalived/keepalived.conf
sed -i 's#priority 100#priority 98#g' /etc/keepalived/keepalived.conf

6.14.5 设置监控脚本
cat > /etc/keepalived/check_kubernetes.sh < #!/bin/bash

function chech_kubernetes() {
for ((i=0;i<5;i++));do
apiserver_pid_id=$(pgrep kube-apiserver)
if [[ ! -z $apiserver_pid_id ]];then
return
else
sleep 2
fi
apiserver_pid_id=0
done
}

# 1:running 0:stopped
check_kubernetes
if [[ $apiserver_pid_id -eq 0 ]];then
/usr/bin/systemctl stop keepalived
exit 1
else
exit 0
fi
EOF
chmod +x /etc/keepalived/check_kubernetes.sh

6.16.6 分发脚本
for i in m2 m3;do
scp /etc/keepalived/check_kubernetes.sh $i:/etc/keepalived/check_kubernetes.sh
ssh $i "chmod +x /etc/keepalived/check_kubernetes.sh"
done
6.17 部署kube-controller-manager服务
6.17.1 创建kube-controller-manager配置文件
cat > /etc/kubernetes/cfg/kube-controller-manager.conf << EOF
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--leader-elect=true \\
--cluster-name=kubernetes \\
--bind-address=127.0.0.1 \\
--allocate-node-cidrs=true \\
--cluster-cidr=10.244.0.0/12 \\
--service-cluster-ip-range=10.96.0.0/16 \\
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--root-ca-file=/etc/kubernetes/ssl/ca.pem \\
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--kubeconfig=/etc/kubernetes/cfg/kube-controller-manager.kubeconfig \\
--tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem \\
--tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem \\
--experimental-cluster-signing-duration=87600h0m0s \\
--controllers=*,bootstrapsigner,tokencleaner \\
--use-service-account-credentials=true \\
--node-monitor-grace-period=10s \\
--horizontal-pod-autoscaler-use-rest-clients=true"
EOF

6.17.2 注册kube-controller-manager服务
cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/usr/local/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

6.17.3 分发脚本
for i in master02 master03;
do
scp /etc/kubernetes/cfg/kube-controller-manager.conf root@$i:/etc/kubernetes/cfg
scp /usr/lib/systemd/system/kube-controller-manager.service root@$i:/usr/lib/systemd/system/kube-controller-manager.service
done

6.17.4 三台master启动
for i in m1 m2 m3;do
systemctl daemon-reload
systemctl enable --now kube-controller-manager
systemctl status kube-controller-manager
done

6.18 部署kube-scheduler服务
6.18.1 创建kube-scheduler配置文件
cat > /etc/kubernetes/cfg/kube-scheduler.conf << EOF
KUBE_SCHEDULER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--kubeconfig=/etc/kubernetes/cfg/kube-scheduler.kubeconfig \\
--leader-elect=true \\
--master=http://127.0.0.1:8080 \\
--bind-address=127.0.0.1 "
EOF

6.18.2 创建启动脚本
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-scheduler.conf
ExecStart=/usr/local/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

6.18.3 分发配置文件
for ip in master02 master03;
do
scp /usr/lib/systemd/system/kube-scheduler.service root@${ip}:/usr/lib/systemd/system
scp /etc/kubernetes/cfg/kube-scheduler.conf root@${ip}:/etc/kubernetes/cfg
done

6.18.4 三台master启动kube-scheduler
for i in m1 m2 m3;do
systemctl daemon-reload
systemctl enable --now kube-scheduler
done

6.19 查看集群状态
[root@master01 ~]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}

6.20 部署kubelet服务
6.20.1 创建kubelet配置
KUBE_HOSTNAME=`hostname`

cat > /etc/kubernetes/cfg/kubelet.conf << EOF
KUBELET_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--hostname-override=${KUBE_HOSTNAME} \\
--container-runtime=docker \\
--kubeconfig=/etc/kubernetes/cfg/kubelet.kubeconfig \\
--bootstrap-kubeconfig=/etc/kubernetes/cfg/kubelet-bootstrap.kubeconfig \\
--config=/etc/kubernetes/cfg/kubelet-config.yml \\
--cert-dir=/etc/kubernetes/ssl \\
--image-pull-progress-deadline=15m \\
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/k8sos/pause:3.2"
EOF

6.20.2 创建kubelet-config配置文件
cat > /etc/kubernetes/cfg/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 172.16.1.11
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 10.96.0.2
clusterDomain: cluster.local
failSwapOn: false
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/ssl/ca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110
EOF

6.20.3 创建启动脚本
cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kubelet.conf
ExecStart=/usr/local/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

6.20.4 分发配置文件
for ip in master01 master02 master03 ;do
scp /etc/kubernetes/cfg/{kubelet-config.yml,kubelet.conf} root@${ip}:/etc/ kubernetes/cfg
scp /usr/lib/systemd/system/kubelet.service root@${ip}:/usr/lib/systemd/system
done

#master02 处理
sed -i 's#master01#master02#g' /etc/kubernetes/cfg/kubelet.conf
sed -i 's#172.16.1.11#172.16.1.12#g' /etc/kubernetes/cfg/kubelet-config.yml

#master03 处理
sed -i 's#master01#master03#g' /etc/kubernetes/cfg/kubelet.conf
sed -i 's#172.16.1.11#172.16.1.13#g' /etc/kubernetes/cfg/kubelet-config.yml

6.20.5 启动kubelet
for i in m1 m2 m3;do
systemctl daemon-reload;systemctl enable --now kubelet;systemctl status kubelet.service
done

6.21 配置kube-proxy服务
6.21.1 创建kube-proxy配置文件
cat > /etc/kubernetes/cfg/kube-proxy.conf << EOF
KUBE_PROXY_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--config=/etc/kubernetes/cfg/kube-proxy-config.yml"
EOF

6.21.2 创建kube-proxy-config配置文件
cat > /etc/kubernetes/cfg/kube-proxy-config.yml << EOF
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 172.16.1.11
healthzBindAddress: 172.16.1.11:10256
metricsBindAddress: 172.16.1.11:10249
clientConnection:
burst: 200
kubeconfig: /etc/kubernetes/cfg/kube-proxy.kubeconfig
qps: 100
hostnameOverride: master01
clusterCIDR: 10.96.0.0/16
enableProfiling: true
mode: "ipvs"
kubeProxyIPTablesConfiguration:
masqueradeAll: false
kubeProxyIPVSConfiguration:
scheduler: rr
excludeCIDRs: []
EOF

6.21.3 创建启动脚本
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-proxy.conf
ExecStart=/usr/local/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

6.21.4 分发配置文件
for ip in master01 master02 master03 ;do
scp /etc/kubernetes/cfg/{kube-proxy-config.yml,kube-proxy.conf} root@${ip}:/etc/kubernetes/cfg/
scp /usr/lib/systemd/system/kube-proxy.service root@${ip}:/usr/lib/systemd/system/
done

#处理master02 配置文件
sed -i 's#172.16.1.11#172.16.1.12#g' /etc/kubernetes/cfg/kube-proxy-config.yml
sed -i 's#master01#master02#g' /etc/kubernetes/cfg/kube-proxy-config.yml

#处理master03 配置文件
sed -i 's#172.16.1.11#172.16.1.13#g' /etc/kubernetes/cfg/kube-proxy-config.yml
sed -i 's#master01#master03#g' /etc/kubernetes/cfg/kube-proxy-config.yml

6.21.5 查看配置文件
for ip in master01 master02 master03 ;do
echo ''; echo $ip; echo '';
ssh root@$ip "cat /etc/kubernetes/cfg/kube-proxy-config.yml";
done

6.21.6 启动
for i in m1 m2 m3; do
systemctl daemon-reload; systemctl enable --now kube-proxy; systemctl status kube-proxy
done

6.22 设置kubelet集群
6.22.1 查看kubelet加入集群请求（决定生死的时刻）
[root@master01 k8s]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
node-csr-51i8zZdDrIFh_zGjblcnJHVTVEZF03-MRLmxqW7ubuk 50m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending
node-csr-9DyYdqmYto4MW7IcGbTPqVePH9PHQN1nNefZEFcab7s 50m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending
node-csr-YzbkiJCgLrXM2whs0h00TDceGaBI3Ntly8Z7HGCYvFw 62m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending

6.22.2 批准加入
kubectl certificate approve `kubectl get csr | grep "Pending" | awk '{print $1}'`

6.22.3 查看加入集群的新节点 （三台master、上面配置的服务全部要起来如果查看不到检查下是不是有没起来的服务）
[root@master01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
master01 Ready 123m v1.18.8
master02 Ready 120m v1.18.8
master03 Ready 118m v1.18.8
node01 Ready 3s v1.18.8

6.22.4 配置集群角色
[root@master01 ~]# kubectl label nodes master01 node-role.kubernetes.io/master=master01
node/master01 labeled
[root@master01 ~]# kubectl label nodes master02 node-role.kubernetes.io/master=master02
node/master02 labeled
[root@master01 ~]# kubectl label nodes master03 node-role.kubernetes.io/master=master03
node/master03 labeled
node/node01 labeled
[root@master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
master01 Ready master 135m v1.18.8
master02 Ready master 131m v1.18.8
master03 Ready master 130m v1.18.8

6.22.5 打污点
[root@master01 ~]# kubectl taint nodes master01 node-role.kubernetes.io/master=master01:NoSchedule --overwrite
node/master01 modified
[root@master01 ~]# kubectl taint nodes master02 node-role.kubernetes.io/master=master02:NoSchedule --overwrite
node/master02 modified
[root@master01 ~]# kubectl taint nodes master03 node-role.kubernetes.io/master=master03:NoSchedule --overwrite
node/master03 modified

6.23 安装网络插件
6.23.1 分发脚本
tar -xf /opt/k8s/flannel-v0.13.1-rc1-linux-amd64.tar.gz
for i in master01 master02 master03; do
scp /opt/k8s/flanneld /opt/k8s/mk-docker-opts.sh root@$i:/usr/local/bin;
done

6.23.2 将flanneld配置写入集群数据库
etcdctl \
--ca-file=/etc/etcd/ssl/ca.pem \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--endpoints="https://172.16.1.11:2379,https://172.16.1.12:2379,https://172.16.1.13:2379" \
mk /coreos.com/network/config '{"Network":"10.244.0.0/12", "SubnetLen": 21, "Backend": {"Type": "vxlan", "DirectRouting": true}}'

#检查配置
etcdctl \
--ca-file=/etc/etcd/ssl/ca.pem \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--endpoints="https://172.16.1.11:2379,https://172.16.1.12:2379,https://172.16.1.13:2379" \
get /coreos.com/network/config

#同步etcd配置文件
for i in node02 node01; do
ssh root@$i "mkdir -pv /etc/etcd/ssl"
scp /etc/etcd/ssl/* root@$i:/etc/etcd/ssl/
done

[root@master01 /opt]# for i in node01 node02; do
ssh root@$i mkdir -pv /etc/etcd/ssl
scp /etc/etcd/ssl/* root@$i:/etc/etcd/ssl/
done

6.23.3 注册flanneld
cat > /usr/lib/systemd/system/flanneld.service << EOF
[Unit]
Description=Flanneld address
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service

[Service]
Type=notify
ExecStart=/usr/local/bin/flanneld \\
-etcd-cafile=/etc/etcd/ssl/ca.pem \\
-etcd-certfile=/etc/etcd/ssl/etcd.pem \\
-etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\
-etcd-endpoints=https://172.16.1.11:2379,https://172.16.1.12:2379,https://172.16.1.13:2379 \\
-etcd-prefix=/coreos.com/network \\
-ip-masq
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=always
RestartSec=5
StartLimitInterval=0
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF

6.23.4 分发配置文件
for i in master02 master03;do scp /usr/lib/systemd/system/flanneld.service root@$i:/usr/lib/systemd/system; done

6.23.5 修改docker启动模式
sed -i '/ExecStart/s/\(.*\)/#\1/' /usr/lib/systemd/system/docker.service
sed -i '/ExecReload/a ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/containerd.sock' /usr/lib/systemd/system/docker.service
sed -i '/ExecReload/a EnvironmentFile=-/run/flannel/subnet.env' /usr/lib/systemd/system/docker.service

[root@master01 ~]# sed -i '/ExecStart/s/\(.*\)/#\1/' /usr/lib/systemd/system/docker.service
[root@master01 ~]# sed -i '/ExecReload/a ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/containerd.sock' /usr/lib/systemd/system/docker.service
[root@master01 ~]# sed -i '/ExecReload/a EnvironmentFile=-/run/flannel/subnet.env' /usr/lib/systemd/system/docker.service

6.23.6 分发docker脚本
for ip in master02 master03;do scp /usr/lib/systemd/system/docker.service root@${ip}:/usr/lib/systemd/system; done

6.23.7 启动Flanneld服务
[root@master01 ~]# for i in master01 master02 master03 node01 node02;do
echo ">>> $i"
ssh root@$i "systemctl daemon-reload"
ssh root@$i "systemctl start flanneld"
ssh root@$i "systemctl restart docker"
done
>>> master01
>>> master02
>>> master03
>>> node01
>>> node02

6.24 部署CoreDNS
6.24.1 下载文件
git clone https://github.com/coredns/deployment.git && cd deployment/kubernetes/

6.24.2 绑定匿名用户权限
[root@master01 ~]# kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=kubernetes
clusterrolebinding.rbac.authorization.k8s.io/cluster-system-anonymous created

6.24.3 修改coredns并允许
sed -i 's#coredns/coredns#code.aliyun.com/zhangzhishiu/k8ss#g' coredns.yaml.sed

./deploy.sh -i 10.96.0.2 -s | kubectl apply -f -

kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-85b4878f78-5xr2z 1/1 Running 0 2m31s
6.24.4 绑定用户的超管权限
[root@master01 ~]# kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=kubernetes
clusterrolebinding.rbac.authorization.k8s.io/cluster-system-anonymous created

6.24.5 测试集群
[root@master01 ~]# kubectl run test -it --rm --image=busybox:1.28.3
If you don't see a command prompt, try pressing enter.
/ # nslookup docs.docs.svc.cluster.local
Server: 10.96.0.2
Address 1: 10.96.0.2 kube-dns.kube-system.svc.cluster.local

Name: docs.docs.svc.cluster.local
Address 1: 10.241.96.2 10-241-96-2.docs.docs.svc.cluster.local
Address 2: 10.242.0.2 10-242-0-2.docs.docs.svc.cluster.local

三 部署集群node节点
1 分发工具
[root@master01 bin]# for i in node02 node01; do scp kubelet kube-proxy root@$i:/usr/local/bin/; done
kubelet 100% 108MB 120.2MB/s 00:00
kube-proxy 100% 37MB 98.1MB/s 00:00
kubelet 100% 108MB 117.4MB/s 00:00

for i in node02 node01; do echo $i; ssh root@$i "ls -lh /usr/local/bin"; done

2 颁发node节点证书
for i in node01 node02; do
ssh root@$i "mkdir -pv /etc/kubernetes/ssl"
scp -pr ./{ca*.pem,admin*pem,kube-proxy*pem} root@$i:/etc/kubernetes/ssl
done

3 配置kubelet 服务
for ip in node01 node02;do
ssh root@${ip} "mkdir -pv /var/log/kubernetes"
ssh root@${ip} "mkdir -pv /etc/kubernetes/cfg/"
scp /etc/kubernetes/cfg/{kubelet-config.yml,kubelet.conf} root@${ip}:/etc/kubernetes/cfg
scp /usr/lib/systemd/system/kubelet.service root@${ip}:/usr/lib/systemd/system
done

4 处理配置文件
# 修改node01配置
sed -i 's#master-01#node-01#g' /etc/kubernetes/cfg/kubelet.conf
sed -i 's#172.16.1.11#172.16.1.14#g' /etc/kubernetes/cfg/kubelet-config.yml

# 修改node02配置
sed -i 's#master-01#node-02#g' /etc/kubernetes/cfg/kubelet.conf
sed -i 's#172.16.1.11#172.16.1.15#g' /etc/kubernetes/cfg/kubelet-config.yml

5 开启kubelet
for i in n1 n2 ;do
systemctl daemon-reload;systemctl enable --now kubelet;systemctl status kubelet.service
done

6 配置kube-proxy服务
for ip in node01 node02;do
scp /etc/kubernetes/cfg/{kube-proxy-config.yml,kube-proxy.conf} root@${ip}:/etc/kubernetes/cfg/
scp /usr/lib/systemd/system/kube-proxy.service root@${ip}:/usr/lib/systemd/system/
done

7 修改配置文件
# 修改node01节点

sed -i 's#172.16.1.11#172.16.1.15#g' /etc/kubernetes/cfg/kube-proxy-config.yml
sed -i 's#master-01#node-02#g' /etc/kubernetes/cfg/kube-proxy-config.yml

# 修改node02节点
sed -i 's#172.16.1.11#172.16.1.15#g' /etc/kubernetes/cfg/kube-proxy-config.yml
sed -i 's#master-01#node-02#g' /etc/kubernetes/cfg/kube-proxy-config.yml

8 设置开机自弃
for i in n1 n2 ; do
systemctl daemon-reload; systemctl enable --now kube-proxy; systemctl status kube-proxy
done

9 分发网络插件
for i in node01 node02; do scp flanneld mk-docker-opts.sh root@$i:/usr/local/bin; done

10 同步etcd配置文件
for i in node01 node02;do
ssh root@$i "mkdir -pv /etc/etcd/ssl"
scp -p /etc/etcd/ssl/*.pem root@$i:/etc/etcd/ssl
done

11 分发flannel服务脚本
for i in node01 node02;do scp /usr/lib/systemd/system/flanneld.service root@$i:/usr/lib/systemd/system; done

12 分发docker脚本
[root@master01 ~]# for ip in node01 node02;do scp /usr/lib/systemd/system/docker.service root@${ip}:/usr/lib/systemd/system; done
docker.service 100% 1830 6.0MB/s 00:00
docker.service 100% 1830 4.7MB/s 00:00

13 重启docker开启flanneld服务
[root@master01 ~]# for i in node01 node02;do
echo ">>> $i"
ssh root@$i "systemctl daemon-reload"
ssh root@$i "systemctl restart flanneld"
ssh root@$i "systemctl restart docker"
done
>>> master01
>>> master02
>>> master03
>>> node01
>>> node02
14 查看kubelet 加入集群请求
[root@master01 k8s]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
node-csr-51i8zZdDrIFh_zGjblcnJHVTVEZF03-MRLmxqW7ubuk 50m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending
node-csr-9DyYdqmYto4MW7IcGbTPqVePH9PHQN1nNefZEFcab7s 50m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending
node-csr-YzbkiJCgLrXM2whs0h00TDceGaBI3Ntly8Z7HGCYvFw 62m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending

15 批准加入
kubectl certificate approve `kubectl get csr | grep "Pending" | awk '{print $1}'`

16 查看节点
[root@master01 k8s]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
master01 Ready master 10d v1.18.8 192.168.12.71 CentOS Linux 7 (Core) 4.4.233-1.el7.elrepo.x86_64 docker://19.3.13
master02 Ready master 10d v1.18.8 192.168.12.72 CentOS Linux 7 (Core) 4.4.233-1.el7.elrepo.x86_64 docker://19.3.13
master03 Ready master 10d v1.18.8 192.168.12.73 CentOS Linux 7 (Core) 4.4.233-1.el7.elrepo.x86_64 docker://19.3.13
node01 Ready node 10d v1.18.8 192.168.12.74 CentOS Linux 7 (Core) 4.4.233-1.el7.elrepo.x86_64 docker://19.3.13
node02 Ready node 10d v1.18.8 192.168.12.75 CentOS Linux 7 (Core) 4.4.233-1.el7.elrepo.x86_64 docker://19.3.13

17 验证集群
[root@master01 ~]# kubectl create deployment nginx --image=nginx
deployment.apps/nginx created
[root@master01 ~]# kubectl expose deployment nginx --port=80 --type=NodePort
service/nginx exposed

总结：
master节点每台服务器需要部署以下服务、缺一不可:
etcd
kube-apiserver
kube-scheduler
kubelet
kube-proxy
kube-controller-manager
网络插件flanneld
CoreDNS
node节点需要部署以下服务缺一不可：
kubelet
kube-proxy
etcd
网络插件flanneld

其实每个组件部署方式大同小异、我们过来看，从服务启动脚本一步一步反着推

kube-proxy：

#查看服务启动脚本
cat /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-proxy.conf
ExecStart=/usr/local/bin/kube-proxy $KUBE_PROXY_OPTS
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

#查看服务启动参数、里面有每个服务需要用到那些配置文件、或者证书等，基本上都记录的有，具体每个选项什么意思感兴趣可以自行百度
cat /etc/kubernetes/cfg/kube-proxy.conf
KUBE_PROXY_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/var/log/kubernetes \
--config=/etc/kubernetes/cfg/kube-proxy-config.yml"

kubelet:

cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
After=docker.service

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kubelet.conf
ExecStart=/usr/local/bin/kubelet $KUBELET_OPTS
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

cat /etc/kubernetes/cfg/kubelet.conf
KUBELET_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/var/log/kubernetes \
--hostname-override=master01 \
--container-runtime=docker \
--kubeconfig=/etc/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/etc/kubernetes/cfg/kubelet-bootstrap.kubeconfig \
--config=/etc/kubernetes/cfg/kubelet-config.yml \
--cert-dir=/etc/kubernetes/ssl \
--image-pull-progress-deadline=15m \
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/k8sos/pause:3.2"

kube-apiserver:
cat /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-apiserver.conf ########
ExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

#下面可以看到指定了一些证书，都是服务所需的，
cat /etc/kubernetes/cfg/kube-apiserver.conf
KUBE_APISERVER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/var/log/kubernetes \
--advertise-address=172.16.1.11 \
--default-not-ready-toleration-seconds=360 \
--default-unreachable-toleration-seconds=360 \
--max-mutating-requests-inflight=2000 \
--max-requests-inflight=4000 \
--default-watch-cache-size=200 \
--delete-collection-workers=2 \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--allow-privileged=true \
--service-cluster-ip-range=10.96.0.0/16 \
--service-node-port-range=10-52767 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth=true \
--token-auth-file=/etc/kubernetes/cfg/token.csv \
--kubelet-client-certificate=/etc/kubernetes/ssl/server.pem \
--kubelet-client-key=/etc/kubernetes/ssl/server-key.pem \
--tls-cert-file=/etc/kubernetes/ssl/server.pem \
--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kubernetes/k8s-audit.log \
--etcd-servers=https://172.16.1.11:2379,https://172.16.1.12:2379,https://172.16.1.13:2379 \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem"