k8s 1.22版本更换容器运行时，把 docker 换成 Containerd
详解：
1、k8s 版本1.20以后会弃用 docker，但是也可以用。但最好还是替换成解决方案中的。
2、Containerd 是一个行业标准的容器运行时，强调简单性、健壮性和可移植性。
3、Kubernetes 通过 CRI 接口的形式将 Containerd 用作 Kubernetes 集群的容器运行时。
架构图
一、部署 containerd
1、创建配置文件
[root@master1 ~]# cat /etc/modules-load.d/containerd.conf
overlay
br_netfilter
2、创建完配置文件执行以下命令
modprobe overlay
modprobe br_netfilter
3、立即生效
sysctl --system
4、下载 docker-ce 源
wget http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
或者
yum-config-manager \
--add-repo \
http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
5、安装 containerd 服务并加入开机启动
yum install -y containerd.io
systemctl enable containerd && systemctl start containerd
二、配置 containerd
1、创建路径
mkdir -p /etc/containerd
2、获取默认配置文件
containerd config default | sudo tee /etc/containerd/config.toml
3、修改配置文件，新增 "SystemdCgroup = true"，使用 systemd 作为 cgroup 驱动程序
[root@master1 ~]# cat /etc/containerd/config.toml
version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0
[grpc]
address = "/run/containerd/containerd.sock"
tcp_address = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
[ttrpc]
address = ""
uid = 0
gid = 0
[debug]
address = ""
uid = 0
gid = 0
level = ""
[metrics]
address = ""
grpc_histogram = false
[cgroup]
path = ""
[timeouts]
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"
[plugins]
[plugins."io.containerd.gc.v1.scheduler"]
pause_threshold = 0.02
deletion_threshold = 0
mutation_threshold = 100
schedule_delay = "0s"
startup_delay = "100ms"
[plugins."io.containerd.grpc.v1.cri"]
disable_tcp_service = true
stream_server_address = "127.0.0.1"
stream_server_port = "0"
stream_idle_timeout = "4h0m0s"
enable_selinux = false
selinux_category_range = 1024
sandbox_image = "k8s.gcr.io/pause:3.2" # 更换镜像加速
stats_collect_period = 10
systemd_cgroup = false
enable_tls_streaming = false
max_container_log_line_size = 16384
disable_cgroup = false
disable_apparmor = false
restrict_oom_score_adj = false
max_concurrent_downloads = 3
disable_proc_mount = false
unset_seccomp_profile = ""
tolerate_missing_hugetlb_controller = true
disable_hugetlb_controller = true
ignore_image_defined_volumes = false
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "overlayfs"
default_runtime_name = "runc"
no_pivot = false
disable_snapshot_annotations = true
discard_unpacked_layers = false
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
runtime_type = ""
runtime_engine = ""
runtime_root = ""
privileged_without_host_devices = false
base_runtime_spec = ""
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
runtime_type = ""
runtime_engine = ""
runtime_root = ""
privileged_without_host_devices = false
base_runtime_spec = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
runtime_engine = ""
runtime_root = ""
privileged_without_host_devices = false
base_runtime_spec = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true ## 增加这个一行
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
max_conf_num = 1
conf_template = ""
[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://dockerhub.mirrors.nwafu.edu.cn"] # 镜像仓库加速根据自己情况添加更改
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
endpoint = ["https://registry.aliyuncs.com/k8sxio"] # 镜像仓库加速根据自己情况添加更改
[plugins."io.containerd.grpc.v1.cri".image_decryption]
key_model = ""
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins."io.containerd.internal.v1.opt"]
path = "/opt/containerd"
[plugins."io.containerd.internal.v1.restart"]
interval = "10s"
[plugins."io.containerd.metadata.v1.bolt"]
content_sharing_policy = "shared"
[plugins."io.containerd.monitor.v1.cgroups"]
no_prometheus = false
[plugins."io.containerd.runtime.v1.linux"]
shim = "containerd-shim"
runtime = "runc"
runtime_root = ""
no_shim = false
shim_debug = false
[plugins."io.containerd.runtime.v2.task"]
platforms = ["linux/amd64"]
[plugins."io.containerd.service.v1.diff-service"]
default = ["walking"]
[plugins."io.containerd.snapshotter.v1.devmapper"]
root_path = ""
pool_name = ""
base_image_size = ""
async_remove = false
4、查看 containerd 运行状态（以下状态视为正常）
[root@master1 ~]# systemctl status containerd
● containerd.service - containerd container runtime
Loaded: loaded (/usr/lib/systemd/system/containerd.service; enabled; vendor preset: disabled)
Active: active (running) since Sun 2022-03-06 08:09:00 CST; 1h 43min ago
Docs: https://containerd.io
Process: 931 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
Main PID: 941 (containerd)
Tasks: 11
Memory: 61.4M
CGroup: /system.slice/containerd.service
└─941 /usr/bin/containerd
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.887356305+08:00" level=info msg="Start recovering state"
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.887756475+08:00" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
Mar 06 08:09:00 master1 systemd[1]: Started containerd container runtime.
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.890318315+08:00" level=info msg=serving... address=/run/containerd/containerd.sock.ttrpc
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.890398891+08:00" level=info msg=serving... address=/run/containerd/containerd.sock
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.892060037+08:00" level=info msg="containerd successfully booted in 0.074829s"
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.977695574+08:00" level=info msg="Start event monitor"
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.977773239+08:00" level=info msg="Start snapshots syncer"
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.977799738+08:00" level=info msg="Start cni network conf syncer"
Mar 06 08:09:00 master1 containerd[941]: time="2022-03-06T08:09:00.977808051+08:00" level=info msg="Start streaming server"
5、更改套接字
[root@master1 ~]# cat /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
6、修改 k8s kubeadm-config.yaml 文件
[root@master1 ~]# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.200.3 # 本机IP
bindPort: 6443
nodeRegistration:
criSocket: /run/containerd/containerd.sock # 此处千万不要忘记修改，如果不修改等于没有替换。（此处已经更改完了）
name: master1 # 本主机名
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.200.16:16443" # 虚拟IP和haproxy端口
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers # 镜像仓库源要根据自己实际情况修改
kind: ClusterConfiguration
kubernetesVersion: v1.22.0 # k8s版本
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
SupportIPVSProxyMode: true
mode: ipvs
7、配置kubelet文件
1>、增加以下参数/etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock
或者
增加以下参数/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --runtime-request-timeout=15m
8、查看是否替换成功
[root@master1 ~]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
master1 Ready control-plane,master 159m v1.22.0 192.168.245.3 CentOS Linux 7 (Core) 3.10.0-1160.el7.x86_64 containerd://1.4.13 # 出现containerd代表成功，如果还是docker就是失败
master2 Ready control-plane,master 148m v1.22.0 192.168.245.4 CentOS Linux 7 (Core) 3.10.0-1160.el7.x86_64 containerd://1.4.13
master3 Ready control-plane,master 146m v1.22.0 192.168.245.5 CentOS Linux 7 (Core) 3.10.0-1160.el7.x86_64 containerd://1.4.13
node1 Ready 145m v1.22.0 192.168.245.6 CentOS Linux 7 (Core) 3.10.0-1160.el7.x86_64 containerd://1.4.13
9、查看 kubelet 指定的 cri socket
[root@master1 ~]# ps -ef | grep kubelet | grep containerd.sock
root 53926 1 1 19:15 ? 00:00:30 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock --pod-infra-container-image=k8s.gcr.io/pause:3.5
--container-runtime-endpoint=/run/containerd/containerd.sock：代表正在运行
10、验证crictl是否能用
[root@master1 ~]# crictl -v
crictl version v1.19.0
[root@master1 ~]# crictl
NAME:
crictl - client for CRI
USAGE:
crictl [global options] command [command options] [arguments...]
VERSION:
v1.19.0
COMMANDS:
attach Attach to a running container
create Create a new container
exec Run a command in a running container
version Display runtime version information
images, image, img List images
inspect Display the status of one or more containers
inspecti Return the status of one or more images
imagefsinfo Return image filesystem info
inspectp Display the status of one or more pods
logs Fetch the logs of a container
port-forward Forward local port to a pod
ps List containers
pull Pull an image from a registry
run Run a new container inside a sandbox
runp Run a new pod
rm Remove one or more containers
rmi Remove one or more images
rmp Remove one or more pods
pods List pods
start Start one or more created containers
info Display information of the container runtime
stop Stop one or more running containers
stopp Stop one or more running pods
update Update one or more running containers
config Get and set crictl client configuration options
stats List container(s) resource usage statistics
completion Output shell completion code
help, h Shows a list of commands or help for one command
GLOBAL OPTIONS:
--config value, -c value Location of the client config file. If not specified and the default does not exist, the program's directory is searched as well (default: "/etc/crictl.yaml") [$CRI_CONFIG_FILE]
--debug, -D Enable debug mode (default: false)
--image-endpoint value, -i value Endpoint of CRI image manager service (default: uses 'runtime-endpoint' setting) [$IMAGE_SERVICE_ENDPOINT]
--runtime-endpoint value, -r value Endpoint of CRI container runtime service (default: uses in order the first successful one of [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock]). Default is now deprecated and the endpoint should be set instead. [$CONTAINER_RUNTIME_ENDPOINT]
--timeout value, -t value Timeout of connecting to the server in seconds (e.g. 2s, 20s.). 0 or less is set to default (default: 2s)
--help, -h show help (default: false)
--version, -v print the version (default: false)
[root@master1 ~]# crictl images
IMAGE TAG IMAGE ID SIZE
docker.io/rancher/mirrored-flannelcni-flannel-cni-plugin v1.0.1 ac40ce6257406 3.82MB
docker.io/rancher/mirrored-flannelcni-flannel v0.17.0 9247abf086779 19.9MB
k8s.gcr.io/coredns/coredns v1.8.4 8d147537fb7d1 13.7MB
k8s.gcr.io/etcd 3.5.0-0 0048118155842 99.9MB
k8s.gcr.io/kube-apiserver v1.22.0 838d692cbe28e 31.3MB
k8s.gcr.io/kube-controller-manager v1.22.0 5344f96781f4d 29.8MB
k8s.gcr.io/kube-proxy v1.22.0 bbad1636b30d8 35.9MB
k8s.gcr.io/kube-scheduler v1.22.0 3db3d153007f7 15MB
k8s.gcr.io/pause 3.5 ed210e3e4a5ba 301kB
registry.aliyuncs.com/k8sxio/pause 3.2 80d28bedfe5de 300kB
k8s.gcr.io/pause 3.2 80d28bedfe5de 300kB
[root@master1 ~]# crictl ps
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID
d1c63df9da9e6 5344f96781f4d 3 hours ago Running kube-controller-manager 1 8cf1ba07a38f2
b1b85664edcdc 3db3d153007f7 3 hours ago Running kube-scheduler 1 3b2c2774a0c46
e00654629d33b 0048118155842 3 hours ago Running etcd 0 d3840af7aad56
60eb4aaf28148 838d692cbe28e 3 hours ago Running kube-apiserver 0 f72baa3d032e4
[root@master1 ~]# crictl ps -a
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID
1798aa1d0bcba bbad1636b30d8 6 seconds ago Exited kube-proxy 31 a70ecef631329
443c374fcf156 9247abf086779 About a minute ago Exited kube-flannel 29 9360f9327df30
cf65c666f30b6 9247abf086779 2 hours ago Exited install-cni 0 9360f9327df30
5e6fe0a74738f ac40ce6257406 2 hours ago Exited install-cni-plugin 0 9360f9327df30
d1c63df9da9e6 5344f96781f4d 3 hours ago Running kube-controller-manager 1 8cf1ba07a38f2
b1b85664edcdc 3db3d153007f7 3 hours ago Running kube-scheduler 1 3b2c2774a0c46
8e4a50bdbbf0a 3db3d153007f7 3 hours ago Exited kube-scheduler 0 3b2c2774a0c46
e00654629d33b 0048118155842 3 hours ago Running etcd 0 d3840af7aad56
d868b2fd6c84a 5344f96781f4d 3 hours ago Exited kube-controller-manager 0 8cf1ba07a38f2
60eb4aaf28148 838d692cbe28e 3 hours ago Running kube-apiserver 0 f72baa3d032e4
11、下载或者删除镜像
[root@master1 ~]# crictl pull nginx
Image is up to date for sha256:605c77e624ddb75e6110f997c58876baa13f8754486b461117934b24a9dc3a85
[root@master1 ~]# crictl rmi 60
Deleted: docker.io/library/nginx:latest