三、所有master节点安装keepalive和haproxy

1、安装keepelived和haproxy

yum install -y keepelived haproxy

 2、配置keeplived文件

cat < /etc/keepalived/keepalived.conf
! /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script check_apiserver {
  script "/etc/keepalived/check_apiserver.sh"   
  interval 3
  weight -2
  fall 10
  rise 2
}

vrrp_instance VI_1 {
    state  ${STATE} 
    interface ${INTERFACE}
    virtual_router_id  ${ROUTER_ID}
    priority ${PRIORITY}
    authentication {
        auth_type PASS
        auth_pass ${AUTH_PASS}
    }
    virtual_ipaddress {
        ${APISERVER_VIP}
    }
    track_script {
        check_apiserver
    }
}
EOF

##在上面的文件中替换自己相应的内容：
/etc/keepalived/check_apiserver.sh   定义检查脚本路径
${STATE}                             如果是主节点 则为MASTER 其他则为 BACKUP。我这里选择k8s-master1为MASTER；k8s-master2 、k8s-master3为BACKUP；
${INTERFACE}                         是网络接口，即服务器网卡的，我的服务器均为eth0；
${ROUTER_ID}                         这个值只要在keepalived集群中保持一致即可，我使用的是默认值51；
${PRIORITY}                          优先级，在master上比在备份服务器上高就行了。我的master设为100，备份服务50；
${AUTH_PASS}                         这个值只要在keepalived集群中保持一致即可；
${APISERVER_VIP}                     就是VIP的地址，我的为：192.168.100.253

 3、配置keeplived健康检查脚本

#!/bin/sh

errorExit() {
    echo "*** $*" 1>&2
    exit 1
}

APISERVER_VIP=192.168.100.253
APISERVER_DEST_PORT=8443

curl --silent --max-time 2 --insecure https://localhost:${APISERVER_DEST_PORT}/ -o /dev/null || errorExit "Error GET https://localhost:${APISERVER_DEST_PORT}/"
if ip addr | grep -q ${APISERVER_VIP}; then
    curl --silent --max-time 2 --insecure https://${APISERVER_VIP}:${APISERVER_DEST_PORT}/ -o /dev/null || errorExit "Error GET https://${APISERVER_VIP}:${APISERVER_DEST_PORT}/"
fi

${APISERVER_VIP} 就是VIP的地址，192.168.100.253；
${APISERVER_DEST_PORT} 这个是定义API Server交互的负载均衡端口，其实就是HAProxy绑定前端负载均衡的端口号，因为HAProxy和k8s一起部署，这里做一个区分，我使用了8443

 4、配置haproxy

cat < /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# apiserver frontend which proxys to the masters
#---------------------------------------------------------------------
frontend apiserver
    bind *:8443   ##注意这里的端口，其实就是haproxy的端口，后面加入集群的时候需要用到
    mode tcp
    option tcplog
    default_backend apiserver

#---------------------------------------------------------------------
# round robin balancing for apiserver
#---------------------------------------------------------------------
backend apiserver
    option httpchk GET /healthz
    http-check expect status 200
    mode tcp
    option ssl-hello-chk
    balance     roundrobin
        server k8s-master1 192.168.100.220:6443 check
        server k8s-master2 192.168.100.221:6443 check
        server k8s-master3 192.168.100.222:6443 check
EOF

5、启动服务并配置开机自启

systemctl enable haproxy --now
systemctl enable keepalived --now

四、部署Kubernetes

##配置k8s源
cat > /etc/yum.repos.d/kubernetes.repo <> /etc/crictl.yaml
runtime-endpoint: unix:///var/run/containerd/containerd.sock
image-endpoint: unix:///var/run/containerd/containerd.sock
timeout: 10 
debug: false
EOF

五、初始化master集群

##生成初始化配置文件
kubeadm config print init-defaults > kubeadm-init.yaml

##修改配置文件
cat > kubeadm-init.yaml << EOF

piVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.100.220  #当前主机ip
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: master
  taints:
  - effect: "NoSchedule"
    key: "node-role.kubernetes.io/master"
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: k8svip:8443  ##重点！！ 这里是我们自定义的域名，端口是haproxy的端口，单master可注释掉这个参数
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.24.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

EOF


## 查看所需镜像列表
kubeadm config images list --config kubeadm-init.yaml

## 预拉取镜像
kubeadm config images pull --config kubeadm-init.yaml

##镜像拉取完后我们查看本地镜像是否下载成功
crictl images

#初始化开始
kubeadm init --config=kubeadm-init.yaml | tee kubeadm-init.log
注意：如果是多master执行到  预拉取镜像 这一步就好了，其他master节点不需要初始化

保存好如下内容 

#该内容是其他master加入需要执行的命令
  kubeadm join k8svip:8443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:364c1b1c7fbbd6a680ea5616670a608531da25c5f62b4c5ed2bdce3321ccb83e \
        --control-plane 
#该内容是node节点加入需要执行的命令
  kubeadm join k8svip:8443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:364c1b1c7fbbd6a680ea5616670a608531da25c5f62b4c5ed2bdce3321ccb83e 

配置api所需要文件

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

六、master2、master3节点加入集群

##复制密钥及相关文件

cat << EOF >> /root/cpkey.sh

# ！/bin/bash
CONTROL_PLANE_IPS="192.168.100.221 192.168.100.222" #masterIP

for host in ${CONTROL_PLANE_IPS}; do
ssh root@${host} mkdir -p /etc/kubernetes/pki/etcd
scp /etc/kubernetes/admin.conf root@${host}:/etc/kubernetes
scp /etc/kubernetes/pki/{ca.*,sa.*,front-proxy-ca.*} root@${host}:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca.* root@${host}:/etc/kubernetes/pki/etcd
done

EOF

##在master2、master3 执行在master1上init后输出的join命令,需要带上参数 `--control-plane` 表示把master控制节点加入集群
kubeadm join k8svip:8443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:364c1b1c7fbbd6a680ea5616670a608531da25c5f62b4c5ed2bdce3321ccb83e \
        --control-plane 
        
        
##执行完后输出
This node has joined the cluster and a new control plane instance was created:

* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
* A new etcd member was added to the local/stacked etcd cluster.

To start administering your cluster from this node, you need to run the following as a regular user:

        mkdir -p $HOME/.kube
        sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
        sudo chown $(id -u):$(id -g) $HOME/.kube/config

七、安装网络插件

curl https://docs.projectcalico.org/manifests/calico.yaml -O
# 修改镜像
sed -i 's#docker.io/calico/cni:v3.22.2#registry.cn-shanghai.aliyuncs.com/wanfei/cni:v3.22.2#' calico.yaml
sed -i 's#docker.io/calico/pod2daemon-flexvol:v3.22.2#registry.cn-shanghai.aliyuncs.com/wanfei/pod2daemon-flexvol:v3.22.2#' calico.yaml
sed -i 's#docker.io/calico/node:v3.22.2#registry.cn-shanghai.aliyuncs.com/wanfei/node:v3.22.2#' calico.yaml
sed -i 's#docker.io/calico/kube-controllers:v3.22.2#registry.cn-shanghai.aliyuncs.com/wanfei/kube-controllers:v3.22.2#' calico.yaml
kubectl apply -f calico.yaml

##如果遇到插件报错：network: stat /var/lib/calico/nodename: no such file or directory: check that the calico/n
rm -rf /etc/cni/net.d/*
rm -rf /var/lib/cni/calico
rm -rf /var/lib/calico
systemctl  restart kubelet
#然后删除相关资源重建

八、kubectl加入tab强化字典

#bash环境
yum -y install bash-completion
echo 'source  <(kubectl  completion  bash)' >> ~/.bashrc
##重新加载shell

#zsh环境
source <(kubectl completion zsh)

#设置别名
echo 'alias k=kubectl' >>~/.bashrc
echo 'complete -F __start_kubectl k' >>~/.bashrc