k8s 批量安装脚本

这里部署的是三个 master 节点和一个 node 节点，etcd 复用 master 节点。
有三个脚本分别是 env.sh ha.sh install.sh

注意事项：
一、firewalld 和 selinux 没写在脚本里面，这个要根据自己情况关闭，不是千篇一律。
二、执行所有脚本之前修改里面的 ip ，k8s 镜像源和仓库，k8s 版本号。
三、执行脚本顺序
1、env.sh
2、ha.sh
3、install.sh
四、手动加入其他控制节点和工作节点之后安装 cni 插件。
五、给脚本附加执行权限

cat > env.sh << EOF
#!/bin/bash
##############################################
#
# k8s 各节点安装常用的软件、要修改的各项前置配置
#
##############################################

# Author: Johnny
# Email: xxx@163.com
# Date: 05/25/2022
# Filename: env.sh


# 定义变量(根据自己情况添加或减少节点 IP)
ip_list1="192.168.200.4 192.168.200.5 192.168.200.6"         # 添加除了第一台主节点以外的 IP。


# 添加个节点 IP 到 hosts 文件
cat >> /etc/hosts << EOF          #  修改 ip 和主机名
192.168.200.3 master3
192.168.200.4 master4
192.168.200.5 master5
192.168.200.6 node6
EOF

for n in $ip_list1
do
   scp /etc/hosts $n:/etc/
   echo -e "\033[1;32m 解析完成\033[0m"
done
   
   

# 关闭各节点 swap 分区
sed -i '/swap/s/^/#/g' /etc/fstab

for n in $ip_list1
do
   ssh root@$n sed -i '/swap/s/^/#/g' /etc/fstab
   echo -e "\033[1;32m swap 分区已关闭\033[0m"
done
   


# 各节点安装 ipset
yum -y install ipvsadm ipset sysstat conntrack libseccomp 

for n in $ip_list1
do
   ssh root@$n yum -y install ipvsadm ipset sysstat conntrack libseccomp
   echo -e "\033[1;32m 部署完成\033[0m" 
done


# 各节点启用 IPVS 模块
touch /etc/sysconfig/modules/ipvs.modules
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/sh
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

for n in $ip_list1
do
   scp /etc/sysconfig/modules/ipvs.modules $n:/etc/sysconfig/modules/ 
   ssh root@$n chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
   echo -e "\033[1;32m IPVS 模块已开启\033[0m"
done


# 将各节点的 IPv4 流量传递到 iptables 的链
modprobe br_netfilter
touch /etc/sysctl.d/k8s.conf
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
EOF
sysctl -p /etc/sysctl.d/k8s.conf

for n in $ip_list1
do
   scp /etc/sysctl.d/k8s.conf $n:/etc/sysctl.d/
   ssh root@$n modprobe br_netfilter && sysctl -p /etc/sysctl.d/k8s.conf
   echo -e "\033[1;32m IPV4 已设置完成\033[0m"
done
   


# 同步各节点的时间
timedatectl set-timezone Asia/Shanghai && chronyc -a makestep

for a in $ip_list1
do
   ssh root@$a timedatectl set-timezone Asia/Shanghai && chronyc -a makestep
   echo -e "\033[1;32m 时间同步完成\033[0m"
done
   


# 各节点安装 docker
cd /etc/yum.repos.d/ && wget https://download.docker.com/linux/centos/docker-ce.repo
yum -y install docker-ce-18.06.0.ce-3.el7
systemctl enable docker && systemctl start docker

for n in $ip_list1
do   
   scp /etc/yum.repos.d/docker-ce.repo $n:/etc/yum.repos.d/ 
   ssh root@$n yum -y install docker-ce-18.06.0.ce-3.el7
   ssh root@$n systemctl enable docker
   ssh root@$n systemctl start docker
   echo -e "\033[1;32m docker 已安装\033[0m"
done
   


# 配置各节点的 docker 驱动
touch /etc/docker/daemon.json
cat > /etc/docker/daemon.json << EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ]
}
EOF
systemctl restart docker

for n in $ip_list1 
do 
   scp /etc/docker/daemon.json $n:/etc/docker/
   ssh root@$n systemctl restart docker
   ssh root@$n docker info | grep Cgroup
   echo -e "\033[1;32m docker 加速器配置完成\033[0m"
done
   


# 配置各节点 k8s 源和镜像仓库
touch /etc/yum.repos.d/kubernetes.repo
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=0    
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

for n in $ip_list1
do
   scp /etc/yum.repos.d/kubernetes.repo $n:/etc/yum.repos.d/
   echo -e "\033[1;32m k8s 镜像源和仓库配置完成\033[0m"
done
   


# 各节点安装 kubeadm kubelet kubectl
yum -y install kubeadm-1.18.2 kubelet-1.18.2 kubectl-1.18.2                  # 选择自己要安装的版本
systemctl enable kubelet && systemctl daemon-reload

for n in $ip_list1
do
   ssh root@$n yum -y install kubeadm-1.18.2 kubelet-1.18.2 kubectl-1.18.2
   ssh root@$n systemctl enable kubelet && systemctl daemon-reload
   echo -e "\033[1;32m k8s 客户端和管理工具部署完成\033[0m"
done
EOF

cat > ha.sh << EOF
#!/bin/bash
##############################################
#
# k8s 高可用和负载均衡
#
##############################################

# Author: Johnny
# Email: xxx@163.com
# Date: 05/25/2022
# Filename: ha.sh


# 安装 keepalived 和 haproxy 并修改配置文件参数
yum -y install keepalived haproxy  
cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL

# 添加如下内容
   script_user root
   enable_script_security
}

vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"         # 检测脚本路径
    interval 3
    weight -2 
    fall 10
    rise 2
}

vrrp_instance VI_1 {
    state MASTER            # MASTER
    interface ens33         # 本机网卡名
    virtual_router_id 51
    priority 100             # 权重100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.200.16      # 虚拟IP
    }
    track_script {
        check_haproxy       # 模块
    }
}
EOF

cat > /etc/haproxy/haproxy.cfg << EOF
#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend  kubernetes-apiserver
    mode                        tcp
    bind                        *:16443
    option                      tcplog
    default_backend             kubernetes-apiserver

#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
listen stats
    bind            *:1080
    stats auth      admin:awesomePassword
    stats refresh   5s
    stats realm     HAProxy\ Statistics
    stats uri       /admin?stats

#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiserver
    mode        tcp
    balance     roundrobin
    server  master1 192.168.200.3:6443 check
    server  master2 192.168.200.4:6443 check
    server  master3 192.168.200.5:6443 check
EOF


touch /etc/keepalived/check_haproxy.sh
cat > /etc/keepalived/check_haproxy.sh << EOF
#!/bin/sh
# HAPROXY down
A=`ps -C haproxy --no-header | wc -l`
if [ $A -eq 0 ]
then
systmectl start haproxy
if [ ps -C haproxy --no-header | wc -l -eq 0 ]
then
killall -9 haproxy
echo "HAPROXY down" | mail -s "haproxy"
sleep 3600
fi 
fi
EOF

for file in /etc/keepalived/check_haproxy.sh
do
   chmod +x $file
   echo -e "\033[1;32m 脚本已附加权限\033[0m" 
done
   
systemctl enable keepalived >/dev/null 2>&1 
systemctl start keepalived 
systemctl enable haproxy >/dev/null 2>&1 
systemctl start haproxy 


# 定义变量
ip_list="192.168.200.4 192.168.200.5"     # 填写需要部署高可用的节点 ip，第一台主节点 ip 不用写。

for a in $ip_list
do
   ssh root@$a yum -y install keepalived haproxy 
   scp /etc/keepalived/keepalived.conf $a:/etc/keepalived/ >/dev/null 2>&1
   scp /etc/haproxy/haproxy.cfg $a:/etc/haproxy >/dev/null 2>&1
   scp /etc/keepalived/check_haproxy.sh $a:/etc/keepalived/ >/dev/null 2>&1
   ssh root@$a sed -i "s/MASTER/BACKUP/g" `grep MASTER -rl /etc/keepalived/`
   ssh root@$a systemctl enable keepalived >/dev/null 2>&1 
   ssh root@$a systemctl start keepalived 
   ssh root@$a systemctl enable haproxy >/dev/null 2>&1
   ssh root@$a systemctl start haproxy
   echo -e "\033[1;32m 节点主备名称修改完成\033[0m" 
done


list1=192.168.200.4                  

for b in $list1
do
  ssh root@$b sed -i 's/100/99/g' /etc/keepalived/keepalived.conf
  ssh root@$b systemctl restart keepalived 
  ssh root@$b systemctl status keepalived 
  ssh root@$b systemctl status haproxy
  echo -e "\033[1;32m 权重修改完成\033[0m"  
done


list2=192.168.200.5

for c in $list2
do
  ssh root@$c sed -i 's/100/98/g' /etc/keepalived/keepalived.conf
  ssh root@$c systemctl restart keepalived 
  ssh root@$c systemctl status keepalived
  ssh root@$c systemctl status haproxy
  echo -e "\033[1;32m 权重修改完成\033[0m"
done
EOF

cat > install.sh << EOF
#!/bin/bash
##############################################
#
# k8s 初始化集群并加入其他节点
#
##############################################

# Author: Johnny
# Email: xxx@163.com
# Date: 05/25/2022
# Filename: install.sh

# 定义变量
ip_list2="192.168.200.4 192.168.200.5"             # 其他主节点 ip


# 修改 kubeadm-config.yaml 默认配置文件
kubeadm config print init-defaults > kubeadm-config.yaml
cat > kubeadm-config.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.200.3     
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: master3        
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.200.16:16443"    高可用 ip + 端口号
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io    
kind: ClusterConfiguration
kubernetesVersion: v1.18.2        # k8s 版本号
networking:
  dnsDomain: cluster.local
  podSubnet: "10.244.0.0/16"
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:                       ###    这两行在1.20版本要去掉
  SupportIPVSProxyMode: true        ###
mode: ipvs
EOF


# 下载 k8s 镜像
kubeadm config images pull --config kubeadm-config.yaml

for m in $ip_list2  
do
   scp kubeadm-config.yaml $m:/root/ 
   ssh root@$m kubeadm config images pull --config kubeadm-config.yaml
   echo -e "\033[1;32m 镜像下载完成\033[0m"
done


# 初始化集群
kubeadm init --config kubeadm-config.yaml --ignore-preflight-errors=all 


# 将主节点证书复制到其他节点
for m in $ip_list2
do
   ssh root@$m mkdir -p /etc/kubernetes/pki/etcd
   scp /etc/kubernetes/pki/ca.* $m:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/sa.* $m:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/front-proxy-ca.* $m:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/etcd/ca.* $m:/etc/kubernetes/pki/etcd/
   scp /etc/kubernetes/admin.conf $m:/etc/kubernetes/
   echo -e "\033[1;32m 其他控制节点证书传送完成\033[0m"
done


for b in 192.168.200.6            # node 节点 ip
do
   scp /etc/kubernetes/admin.conf $b:/etc/kubernetes/
   echo -e "\033[1;32m 其他工作节点证书传送完成\033[0m"
done
EOF