安装与重装k8s 1.15.0
准备
系统配置
1 # 两台CentOS 7.6主机修改主机名 2 # 对于管理面节点 3 echo “node1” > /etc/hostname 4 hostnamectl set-hostname node1 5 # 对于数据面节点 6 echo "node2" > /etc/hostname 7 hostnamectl set-hostname "node2" 8 9 # 安装nslookup和ifconfig 10 # nslookup 11 yum install -y bind-utils 12 # ifconfig 13 yum install -y net-tools.x86_64 14 15 # 禁用防火墙 16 systemctl stop firewalld 17 systemctl disable firewalld 18 19 # 禁用SELINUX 20 setenforce 0 21 vi /etc/selinux/config 22 SELINUX=disabled 23 24 # 创建/etc/sysctl.d/k8s.conf文件，添加如下内容 25 net.bridge.bridge-nf-call-ip6tables = 1 26 net.bridge.bridge-nf-call-iptables = 1 27 net.ipv4.ip_forward = 1 28 # 使修改生效 29 modprobe br_netfilter 30 sysctl -p /etc/sysctl.d/k8s.conf
配置kube-proxy开启ipvs的前置条件
1 # 所有节点上执行以下脚本 2 cat > /etc/sysconfig/modules/ipvs.modules <<EOF 3 #!/bin/bash 4 modprobe -- ip_vs 5 modprobe -- ip_vs_rr 6 modprobe -- ip_vs_wrr 7 modprobe -- ip_vs_sh 8 modprobe -- nf_conntrack_ipv4 9 EOF 10 chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4 11 yum install -y ipset 12 yum install -y ipvsadm
使用lsmod | grep -e ip_vs -e nf_conntrack_ipv4命令查看是否已经正确加载所需的内核模块。
安装Docker并修改docker cgroup driver为systemd
1 yum install -y yum-utils \ 2 device-mapper-persistent-data \ 3 lvm2 4 yum install -y wget 5 6 cd /etc/yum.repos.d/ 7 wget https://download.docker.com/linux/centos/docker-ce.repo 8 9 yum install -y --setopt=obsoletes=0 docker-ce-18.09.7-3.el7 10 11 systemctl start docker 12 systemctl enable docker 13 14 # 修改各个节点上docker的cgroup driver为systemd，默认是cgroupfs。 15 # 创建或修改/etc/docker/daemon.json 16 { 17 "exec-opts": ["native.cgroupdriver=systemd"] 18 } 19 # 重启Docker 20 systemctl restart docker 21 # 查看cgroup driver 22 docker info | grep Cgroup
使用kubeadm部署k8s并纳管节点
安装kubelet、kubeadm和kubectl
1 # 配置kubeadm源 2 vim /etc/yum.repos.d/kubernetes.repo 3 4 [kubernetes] 5 name=Kubernetes 6 baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64 7 enabled=1 8 gpgcheck=0 9 repo_gpgcheck=0 10 gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg 11 http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg 12 13 # 安装kubelet、kubeadm 和 kubectl 1.15.0版本 14 yum makecache fast 15 yum install -y kubelet-1.15.0 kubeadm-1.15.0 kubectl-1.15.0
关闭系统的Swap
1 swapoff -a
修改/etc/fstab文件，注释掉SWAP的自动挂载
使用free -m确认swap已经关闭
swappiness参数调整，修改/etc/sysctl.d/k8s.conf添加下面一行
1 vm.swappiness=0
执行sysctl -p /etc/sysctl.d/k8s.conf使修改生效。
使用kubeadm init初始化集群
1 # 在各节点开机启动kubelet服务 2 systemctl enable kubelet.service 3 4 kubeadm.yaml 5 apiVersion: kubeadm.k8s.io/v1beta2 6 kind: InitConfiguration 7 localAPIEndpoint: 8 advertiseAddress: 192.168.124.25 9 bindPort: 6443 10 nodeRegistration: 11 taints: 12 - effect: PreferNoSchedule 13 key: node-role.kubernetes.io/master 14 --- 15 apiVersion: kubeadm.k8s.io/v1beta2 16 kind: ClusterConfiguration 17 kubernetesVersion: v1.15.0 18 networking: 19 podSubnet: 10.244.0.0/16 20 21 # 拉取镜像 22 # k8s.gcr.io/kube-apiserver:v1.15.0 23 # k8s.gcr.io/kube-controller-manager:v1.15.0 24 # k8s.gcr.io/kube-scheduler:v1.15.0 25 # k8s.gcr.io/kube-proxy:v1.15.0 26 # k8s.gcr.io/pause:3.1 27 # k8s.gcr.io/etcd:3.3.10 28 # k8s.gcr.io/coredns:1.3.1 29 30 # 执行如下脚本 31 #!/bin/bash 32 images=(kube-apiserver:v1.15.0 kube-controller-manager:v1.15.0 kube-scheduler:v1.15.0 33 kube-proxy:v1.15.0 pause:3.1 etcd:3.3.10 coredns:1.3.1) 34 for imageName in ${images[@]} ; do 35 docker pull registry.aliyuncs.com/google_containers/$imageName 36 docker tag registry.aliyuncs.com/google_containers/$imageName k8s.gcr.io/$imageName 37 docker rmi registry.aliyuncs.com/google_containers/$imageName 38 done 39 40 # 在master上执行下面的命令 41 kubeadm init --config kubeadm.yaml --ignore-preflight-errors=Swap 42 43 # master配置常规用户如何使用kubectl访问集群 44 mkdir -p $HOME/.kube 45 sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 46 sudo chown $(id -u):$(id -g) $HOME/.kube/config 47 # 配置原因是，k8s集群默认需要加密方式访问。把k8s集群的安全配置文件保存到当前用户的.kube目录下，kubectl默认会使用这个目录下的授权信息访问k8s集群。
安装容器网络flannel
1 docker pull quay-mirror.qiniu.com/coreos/flannel:v0.11.0-amd64 2 docker tag quay-mirror.qiniu.com/coreos/flannel:v0.11.0-amd64 quay.io/coreos/flannel:v0.11.0-amd64 3 docker rmi quay-mirror.qiniu.com/coreos/flannel:v0.11.0-amd64 4 5 mkdir -p ~/k8s/ 6 cd ~/k8s 7 curl -O https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml 8 kubectl create -f kube-flannel.yml
纳管节点
1 kubeadm join 192.168.124.25:6443 --token wjasin.yxpd3l1stau9wp10 \ 2 --discovery-token-ca-cert-hash sha256:8beba8c0f153d9e96772b0ed493a0bc0fccaef3a0782bd6dedd5b029b3a10ea5 \ 3 --ignore-preflight-errors=Swap 4 5 # 从master上获取join token 6 kubeadm token create --print-join-command
kube-proxy开启ipvs
修改ConfigMap的kube-system/kube-proxy中的config.conf，mode: “ipvs”
1 kubectl edit cm kube-proxy -n kube-system
把mode改成ipvs。
之后重启各个节点上的kube-proxy pod
1 kubectl get pod -n kube-system | grep kube-proxy | awk '{system("kubectl delete pod "$1" -n kube-system")}'
数据面节点上下载k8s.gcr.io/pause:3.1和kube-proxy:v1.15.3
1 docker pull registry.aliyuncs.com/google_containers/pause:3.1 2 docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1 3 docker rmi registry.aliyuncs.com/google_containers/pause:3.1 4 5 docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0 6 docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0 k8s.gcr.io/kube-proxy:v1.15.0 7 docker rmi registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0
查看所有的Pod
kubectl get pod --all-namespaces -o wide
发现两个节点各自有一个flannel网络插件和kube-proxy对应的Pod。
Master节点设置成不可调度（不把业务Pod调度到管理面）
默认情况下，Master节点打的污点是node-role.kubernetes.io/master:PreferNoSchedule，表示尽量不要调度，实在没有地方调度的情况下，才考虑可以调度过来。
1 # 去掉该污点 2 [root@node1 deploy]# kubectl taint nodes node1 node-role.kubernetes.io/master- 3 node/node1 untainted 4 # 打上不可调度的污点 5 [root@node1 deploy]# kubectl taint nodes node1 node-role.kubernetes.io/master=:NoSchedule 6 node/node1 tainted
因虚拟机IP地址改变而重装k8s和纳管
卸载k8s
k8s_uninstall.sh
管理面节点
1 kubeadm reset -f 2 modprobe -r ipip 3 rm -rf ~/.kube/ 4 rm -rf /etc/kubernetes/ 5 rm -rf /etc/systemd/system/kubelet.service.d 6 rm -rf /etc/systemd/system/kubelet.service 7 #rm -rf /usr/bin/kube* 8 rm -rf /etc/cni 9 rm -rf /opt/cni 10 rm -rf /var/lib/etcd 11 rm -rf /var/etcd
数据面节点
1 kubeadm reset -f 2 systemctl stop kubelet 3 systemctl stop docker 4 rm -rf /var/lib/cni/ 5 rm -rf /var/lib/kubelet/* 6 rm -rf /etc/cni/ 7 ifconfig cni0 down 8 ifconfig flannel.1 down 9 ifconfig docker0 down 10 ip link delete cni0 11 ip link delete flannel.1 12 systemctl start docker 13 systemctl start kubelet
重装k8s
k8s_reinstall.sh
1 # 修改管理面的主机名 2 echo "master" > /etc/hostname 3 hostnamectl set-hostname master 4 5 # 修改kubeadm.yaml和/etc/hosts里面的ip 6 hostIp=$(ifconfig ens33 | grep "inet " | awk '{print $2}') 7 sed -i "s/advertiseAddress:.*/advertiseAddress: ${hostIp}/" ../yaml/kubeadm.yaml 8 9 # 初始化集群 10 kubeadm init --config ../yaml/kubeadm.yaml --ignore-preflight-errors=Swap 11 12 # 添加kubectl执行权限 13 mkdir -p $HOME/.kube 14 cp /etc/kubernetes/admin.conf $HOME/.kube/config 15 chown $(id -u):$(id -g) $HOME/.kube/config 16 17 # 安装容器网络 18 kubectl create -f ../yaml/kube-flannel.yml
纳管节点
add_node.sh
1 # 修改hostname 2 hostIp=$(ifconfig ens33 | grep "inet " | awk '{print $2}') 3 echo "${hostIp}" > /etc/hostname 4 hostnamectl set-hostname ${hostIp} 5 6 # 纳管当前节点 7 # kubeadm join master_ip:6443 --token ... 8 kubeadm join 192.168.1.7:6443 --token vtee5p.mln1qrk82ml2t7xe \ 9 --discovery-token-ca-cert-hash sha256:8dad8ae7ce1bbe79a704b7f23213bf55f5116da7fd4d35e0ee2ec7ae91338316