黑群晖DS3615xs支持docker吗 黑群晖d2550

#CentOS [root@k8s-master01 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0 DEVICE=eth0 NAME=eth0 BOOTPROTO=none ONBOOT=yes IPADDR=172.31.3.101 PREFIX=21 GATEWAY=172.31.0.2 DNS1=223.5.5.5 DNS2=180.76.76.76 #Ubuntu root@k8s-master01:~# cat /etc/netplan/01-netcfg.yaml network: version: 2 renderer: networkd ethernets: eth0: addresses: [172.31.3.101/21] gateway4: 172.31.0.2 nameservers: addresses: [223.5.5.5, 180.76.76.76]

hostnamectl set-hostname k8s-master01.example.local hostnamectl set-hostname k8s-master02.example.local hostnamectl set-hostname k8s-master03.example.local hostnamectl set-hostname k8s-ha01.example.local hostnamectl set-hostname k8s-ha02.example.local hostnamectl set-hostname k8s-harbor01.example.local hostnamectl set-hostname k8s-harbor02.example.local hostnamectl set-hostname k8s-etcd01.example.local hostnamectl set-hostname k8s-etcd02.example.local hostnamectl set-hostname k8s-etcd03.example.local hostnamectl set-hostname k8s-node01.example.local hostnamectl set-hostname k8s-node02.example.local hostnamectl set-hostname k8s-node03.example.local

rm -f /etc/yum.repos.d/*.repo cat > /etc/yum.repos.d/base.repo <<EOF [base] name=base baseurl=https://mirrors.cloud.tencent.com/centos/\$releasever/os/\$basearch/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever [extras] name=extras baseurl=https://mirrors.cloud.tencent.com/centos/\$releasever/extras/\$basearch/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever [updates] name=updates baseurl=https://mirrors.cloud.tencent.com/centos/\$releasever/updates/\$basearch/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever [centosplus] name=centosplus baseurl=https://mirrors.cloud.tencent.com/centos/\$releasever/centosplus/\$basearch/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever [epel] name=epel baseurl=https://mirrors.cloud.tencent.com/epel/\$releasever/\$basearch/ gpgcheck=1 gpgkey=https://mirrors.cloud.tencent.com/epel/RPM-GPG-KEY-EPEL-\$releasever EOF

rm -f /etc/yum.repos.d/*.repo cat > /etc/yum.repos.d/base.repo <<EOF [BaseOS] name=BaseOS baseurl=https://mirrors.aliyun.com/rockylinux/\$releasever/BaseOS/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial [AppStream] name=AppStream baseurl=https://mirrors.aliyun.com/rockylinux/\$releasever/AppStream/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial [extras] name=extras baseurl=https://mirrors.aliyun.com/rockylinux/\$releasever/extras/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial [plus] name=plus baseurl=https://mirrors.aliyun.com/rockylinux/\$releasever/plus/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial [epel] name=epel baseurl=https://mirrors.aliyun.com/epel/\$releasever/Everything/\$basearch/ gpgcheck=1 gpgkey=https://mirrors.aliyun.com/epel/RPM-GPG-KEY-EPEL-\$releasever EOF

rm -f /etc/yum.repos.d/*.repo cat > /etc/yum.repos.d/base.repo <<EOF [BaseOS] name=BaseOS baseurl=https://mirrors.cloud.tencent.com/centos/\$releasever-stream/BaseOS/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial [AppStream] name=AppStream baseurl=https://mirrors.cloud.tencent.com/centos/\$releasever-stream/AppStream/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial [extras] name=extras baseurl=https://mirrors.cloud.tencent.com/centos/\$releasever-stream/extras/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial [centosplus] name=centosplus baseurl=https://mirrors.cloud.tencent.com/centos/\$releasever-stream/centosplus/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial [epel] name=epel baseurl=https://mirrors.cloud.tencent.com/epel/\$releasever/Everything/\$basearch/ gpgcheck=1 gpgkey=https://mirrors.cloud.tencent.com/epel/RPM-GPG-KEY-EPEL-\$releasever EOF

cat > /etc/apt/sources.list <<EOF deb http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse deb-src http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse deb http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs)-security main restricted universe multiverse deb-src http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs)-security main restricted universe multiverse deb http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs)-updates main restricted universe multiverse deb-src http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs)-updates main restricted universe multiverse deb http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs)-proposed main restricted universe multiverse deb-src http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs)-proposed main restricted universe multiverse deb http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs)-backports main restricted universe multiverse deb-src http://mirrors.cloud.tencent.com/ubuntu/ $(lsb_release -cs)-backports main restricted universe multiverse EOF apt update

#CentOS安装 yum -y install vim tree lrzsz wget jq psmisc net-tools telnet yum-utils device-mapper-persistent-data lvm2 git #Rocky除了安装上面工具，还需要安装rsync yum -y install rsync #Ubuntu安装 apt -y install tree lrzsz jq

[root@k8s-master01 ~]# cat ssh_key.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2021-12-20 #FileName: ssh_key.sh #Description: ssh_key for CentOS 7/8 & Ubuntu 18.04/24.04 & Rocky 8 #Copyright (C): 2021 All rights reserved #********************************************************************************************* COLOR="echo -e \\033[01;31m" END='\033[0m' NET_NAME=`ip addr |awk -F"[: ]" '/^2: e.*/{print $3}'` IP=`ip addr show ${NET_NAME}| awk -F" +|/" '/global/{print $3}'` export SSHPASS=123456 HOSTS=" 172.31.3.102 172.31.3.103 172.31.3.104 172.31.3.105 172.31.3.106 172.31.3.107 172.31.3.108 172.31.3.109 172.31.3.110 172.31.3.111 172.31.3.112 172.31.3.113" os(){ OS_ID=`sed -rn '/^NAME=/s@.*="([[:alpha:]]+).*"$@\1@p' /etc/os-release` } ssh_key_push(){ rm -f ~/.ssh/id_rsa* ssh-keygen -f /root/.ssh/id_rsa -P '' &> /dev/null if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ] &> /dev/null;then rpm -q sshpass &> /dev/null || { ${COLOR}"安装sshpass软件包"${END};yum -y install sshpass &> /dev/null; } else dpkg -S sshpass &> /dev/null || { ${COLOR}"安装sshpass软件包"${END};apt -y install sshpass &> /dev/null; } fi sshpass -e ssh-copy-id -o StrictHostKeyChecking=no ${IP} &> /dev/null [ $? -eq 0 ] && echo ${IP} is finished || echo ${IP} is false for i in ${HOSTS};do sshpass -e scp -o StrictHostKeyChecking=no -r /root/.ssh root@${i}: &> /dev/null [ $? -eq 0 ] && echo ${i} is finished || echo ${i} is false done for i in ${HOSTS};do scp /root/.ssh/known_hosts ${i}:.ssh/ &> /dev/null [ $? -eq 0 ] && echo ${i} is finished || echo ${i} is false done } main(){ os ssh_key_push } main [root@k8s-master01 ~]# bash ssh_key.sh 安装sshpass软件包 172.31.3.101 is finished 172.31.3.102 is finished 172.31.3.103 is finished 172.31.3.104 is finished 172.31.3.105 is finished 172.31.3.106 is finished 172.31.3.107 is finished 172.31.3.108 is finished 172.31.3.109 is finished 172.31.3.110 is finished 172.31.3.111 is finished 172.31.3.112 is finished 172.31.3.113 is finished 172.31.3.102 is finished 172.31.3.103 is finished 172.31.3.104 is finished 172.31.3.105 is finished 172.31.3.106 is finished 172.31.3.107 is finished 172.31.3.108 is finished 172.31.3.109 is finished 172.31.3.110 is finished 172.31.3.111 is finished 172.31.3.112 is finished 172.31.3.113 is finished

cat >> /etc/hosts <<EOF 172.31.3.101 k8s-master01.example.local k8s-master01 172.31.3.102 k8s-master02.example.local k8s-master02 172.31.3.103 k8s-master03.example.local k8s-master03 172.31.3.104 k8s-ha01.example.local k8s-ha01 172.31.3.105 k8s-ha02.example.local k8s-ha02 172.31.3.106 k8s-harbor01.example.local k8s-harbor01 172.31.3.107 k8s-harbor02.example.local k8s-harbor02 172.31.3.108 k8s-etcd01.example.local k8s-etcd01 172.31.3.109 k8s-etcd02.example.local k8s-etcd02 172.31.3.110 k8s-etcd03.example.local k8s-etcd03 172.31.3.111 k8s-node01.example.local k8s-node01 172.31.3.112 k8s-node02.example.local k8s-node02 172.31.3.113 k8s-node03.example.local k8s-node03 172.31.3.188 kubeapi.raymonds.cc kubeapi 172.31.3.188 harbor.raymonds.cc harbor EOF for i in {102..113};do scp /etc/hosts 172.31.3.$i:/etc/ ;done

#CentOS systemctl disable --now firewalld #CentOS 7 systemctl disable --now NetworkManager #Ubuntu systemctl disable --now ufw

#CentOS setenforce 0 sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config #Ubuntu Ubuntu没有安装SELinux，不用设置

sed -ri 's/.*swap.*/#&/' /etc/fstab swapoff -a #Ubuntu 20.04，执行下面命令 sed -ri 's/.*swap.*/#&/' /etc/fstab SD_NAME=`lsblk|awk -F"[ └─]" '/SWAP/{printf $3}'` systemctl mask dev-${SD_NAME}.swap swapoff -a

[root@k8s-ha01 ~]# cat install_chrony_server.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2021-11-22 #FileName: install_chrony_server.sh #Description: install_chrony_server for CentOS 7/8 & Ubuntu 18.04/20.04 & Rocky 8 #Copyright (C): 2021 All rights reserved #********************************************************************************************* COLOR="echo -e \\033[01;31m" END='\033[0m' os(){ OS_ID=`sed -rn '/^NAME=/s@.*="([[:alpha:]]+).*"$@\1@p' /etc/os-release` } install_chrony(){ ${COLOR}"安装chrony软件包..."${END} if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ] &> /dev/null;then yum -y install chrony &> /dev/null sed -i -e '/^pool.*/d' -e '/^server.*/d' -e '/^# Please consider .*/a\server ntp.aliyun.com iburst\nserver time1.cloud.tencent.com iburst\nserver ntp.tuna.tsinghua.edu.cn iburst' -e 's@^#allow.*@allow 0.0.0.0/0@' -e 's@^#local.*@local stratum 10@' /etc/chrony.conf systemctl enable --now chronyd &> /dev/null systemctl is-active chronyd &> /dev/null || { ${COLOR}"chrony 启动失败,退出!"${END} ; exit; } ${COLOR}"chrony安装完成"${END} else apt -y install chrony &> /dev/null sed -i -e '/^pool.*/d' -e '/^# See http:.*/a\server ntp.aliyun.com iburst\nserver time1.cloud.tencent.com iburst\nserver ntp.tuna.tsinghua.edu.cn iburst' /etc/chrony/chrony.conf echo "allow 0.0.0.0/0" >> /etc/chrony/chrony.conf echo "local stratum 10" >> /etc/chrony/chrony.conf systemctl enable --now chronyd &> /dev/null systemctl is-active chronyd &> /dev/null || { ${COLOR}"chrony 启动失败,退出!"${END} ; exit; } ${COLOR}"chrony安装完成"${END} fi } main(){ os install_chrony } main [root@k8s-ha01 ~]# bash install_chrony_server.sh chrony安装完成 [root@k8s-ha02 ~]# bash install_chrony_server.sh chrony安装完成 [root@k8s-ha01 ~]# chronyc sources -nv 210 Number of sources = 3 MS Name/IP address Stratum Poll Reach LastRx Last sample =============================================================================== ^* 203.107.6.88 2 6 17 39 -1507us[-8009us] +/- 37ms ^- 139.199.215.251 2 6 17 39 +10ms[ +10ms] +/- 48ms ^? 101.6.6.172 0 7 0 - +0ns[ +0ns] +/- 0ns [root@k8s-ha02 ~]# chronyc sources -nv 210 Number of sources = 3 MS Name/IP address Stratum Poll Reach LastRx Last sample =============================================================================== ^* 203.107.6.88 2 6 17 40 +90us[-1017ms] +/- 32ms ^+ 139.199.215.251 2 6 33 37 +13ms[ +13ms] +/- 25ms ^? 101.6.6.172 0 7 0 - +0ns[ +0ns] +/- 0ns

[root@k8s-master01 ~]# cat install_chrony_client.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2021-11-22 #FileName: install_chrony_client.sh #Description: install_chrony_client for CentOS 7/8 & Ubuntu 18.04/20.04 & Rocky 8 #Copyright (C): 2021 All rights reserved #********************************************************************************************* COLOR="echo -e \\033[01;31m" END='\033[0m' SERVER1=172.31.3.104 SERVER2=172.31.3.105 os(){ OS_ID=`sed -rn '/^NAME=/s@.*="([[:alpha:]]+).*"$@\1@p' /etc/os-release` } install_chrony(){ ${COLOR}"安装chrony软件包..."${END} if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ] &> /dev/null;then yum -y install chrony &> /dev/null sed -i -e '/^pool.*/d' -e '/^server.*/d' -e '/^# Please consider .*/a\server '${SERVER1}' iburst\nserver '${SERVER2}' iburst' /etc/chrony.conf systemctl enable --now chronyd &> /dev/null systemctl is-active chronyd &> /dev/null || { ${COLOR}"chrony 启动失败,退出!"${END} ; exit; } ${COLOR}"chrony安装完成"${END} else apt -y install chrony &> /dev/null sed -i -e '/^pool.*/d' -e '/^# See http:.*/a\server '${SERVER1}' iburst\nserver '${SERVER2}' iburst' /etc/chrony/chrony.conf systemctl enable --now chronyd &> /dev/null systemctl is-active chronyd &> /dev/null || { ${COLOR}"chrony 启动失败,退出!"${END} ; exit; } systemctl restart chronyd ${COLOR}"chrony安装完成"${END} fi } main(){ os install_chrony } main [root@k8s-master01 ~]# for i in k8s-master02 k8s-master03 k8s-harbor01 k8s-harbor02 k8s-etcd01 k8s-etcd02 k8s-etcd03 k8s-node01 k8s-node02 k8s-node03;do scp -o StrictHostKeyChecking=no install_chrony_client.sh $i:/root/ ; done [root@k8s-master01 ~]# bash install_chrony_client.sh [root@k8s-master02 ~]# bash install_chrony_client.sh [root@k8s-master03 ~]# bash install_chrony_client.sh [root@k8s-harbor01:~]# bash install_chrony_client.sh [root@k8s-harbor02:~]# bash install_chrony_client.sh [root@k8s-etcd01:~]# bash install_chrony_client.sh [root@k8s-etcd02:~]# bash install_chrony_client.sh [root@k8s-etcd03:~]# bash install_chrony_client.sh [root@k8s-node01:~]# bash install_chrony_client.sh [root@k8s-node02:~]# bash install_chrony_client.sh [root@k8s-node03:~]# bash install_chrony_client.sh [root@k8s-master01 ~]# chronyc sources -nv MS Name/IP address Stratum Poll Reach LastRx Last sample =============================================================================== ^* k8s-ha01.example.local 3 6 37 51 +13us[-1091us] +/- 38ms ^+ k8s-ha02.example.local 3 6 37 51 -14us[-1117us] +/- 36ms

ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime echo 'Asia/Shanghai' >/etc/timezone #Ubuntu还要设置下面内容 cat >> /etc/default/locale <<-EOF LC_TIME=en_DK.UTF-8 EOF

ulimit -SHn 65535 cat >>/etc/security/limits.conf <<EOF * soft nofile 65536 * hard nofile 131072 * soft nproc 65535 * hard nproc 655350 * soft memlock unlimited * hard memlock unlimited EOF

[root@k8s-master01 ~]# wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm [root@k8s-master01 ~]# wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm

[root@k8s-master01 ~]# for i in k8s-master02 k8s-master03 k8s-node01 k8s-node02;do scp kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm $i:/root/ ; done

cd /root && yum localinstall -y kernel-ml*

grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"

grubby --default-kernel [root@k8s-master01 ~]# grubby --default-kernel /boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64

reboot uname -a [root@k8s-master01 ~]# uname -a Linux k8s-master01 4.19.12-1.el7.elrepo.x86_64 #1 SMP Fri Dec 21 11:06:36 EST 2018 x86_64 x86_64 x86_64 GNU/Linux

#CentOS yum -y install ipvsadm ipset sysstat conntrack libseccomp #Ubuntu apt -y install ipvsadm ipset sysstat conntrack libseccomp-dev

modprobe -- ip_vs modprobe -- ip_vs_rr modprobe -- ip_vs_wrr modprobe -- ip_vs_sh modprobe -- nf_conntrack #内核小于4.18，把这行改成nf_conntrack_ipv4 cat >> /etc/modules-load.d/ipvs.conf <<EOF ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp ip_vs_sh nf_conntrack #内核小于4.18，把这行改成nf_conntrack_ipv4 ip_tables ip_set xt_set ipt_set ipt_rpfilter ipt_REJECT ipip EOF 然后执行systemctl restart systemd-modules-load.service即可

cat > /etc/sysctl.d/k8s.conf <<EOF net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 fs.may_detach_mounts = 1 vm.overcommit_memory=1 vm.panic_on_oom=0 fs.inotify.max_user_watches=89100 fs.file-max=52706963 fs.nr_open=52706963 net.netfilter.nf_conntrack_max=2310720 net.ipv4.tcp_keepalive_time = 600 net.ipv4.tcp_keepalive_probes = 3 net.ipv4.tcp_keepalive_intvl =15 net.ipv4.tcp_max_tw_buckets = 36000 net.ipv4.tcp_tw_reuse = 1 net.ipv4.tcp_max_orphans = 327680 net.ipv4.tcp_orphan_retries = 3 net.ipv4.tcp_syncookies = 1 net.ipv4.tcp_max_syn_backlog = 16384 net.ipv4.ip_conntrack_max = 65536 net.ipv4.tcp_max_syn_backlog = 16384 net.ipv4.tcp_timestamps = 0 net.core.somaxconn = 16384 EOF sysctl --system

net.ipv4.ip_forward = 1 #其值为0,说明禁止进行IP转发；如果是1,则说明IP转发功能已经打开。 net.bridge.bridge-nf-call-iptables = 1 #二层的网桥在转发包时也会被iptables的FORWARD规则所过滤，这样有时会出现L3层的iptables rules去过滤L2的帧的问题 net.bridge.bridge-nf-call-ip6tables = 1 #是否在ip6tables链中过滤IPv6包 fs.may_detach_mounts = 1 #当系统有容器运行时，需要设置为1 vm.overcommit_memory=1 #0， 表示内核将检查是否有足够的可用内存供应用进程使用；如果有足够的可用内存，内存申请允许；否则，内存申请失败，并把错误返回给应用进程。 #1， 表示内核允许分配所有的物理内存，而不管当前的内存状态如何。 #2， 表示内核允许分配超过所有物理内存和交换空间总和的内存 vm.panic_on_oom=0 #OOM就是out of memory的缩写，遇到内存耗尽、无法分配的状况。kernel面对OOM的时候，咱们也不能慌乱，要根据OOM参数来进行相应的处理。 #值为0：内存不足时，启动 OOM killer。 #值为1：内存不足时，有可能会触发 kernel panic（系统重启），也有可能启动 OOM killer。 #值为2：内存不足时，表示强制触发 kernel panic，内核崩溃GG（系统重启）。 fs.inotify.max_user_watches=89100 #表示同一用户同时可以添加的watch数目（watch一般是针对目录，决定了同时同一用户可以监控的目录数量） fs.file-max=52706963 #所有进程最大的文件数 fs.nr_open=52706963 #单个进程可分配的最大文件数 net.netfilter.nf_conntrack_max=2310720 #连接跟踪表的大小，建议根据内存计算该值CONNTRACK_MAX = RAMSIZE (in bytes) / 16384 / (x / 32)，并满足nf_conntrack_max=4*nf_conntrack_buckets，默认262144 net.ipv4.tcp_keepalive_time = 600 #KeepAlive的空闲时长，或者说每次正常发送心跳的周期，默认值为7200s（2小时） net.ipv4.tcp_keepalive_probes = 3 #在tcp_keepalive_time之后，没有接收到对方确认，继续发送保活探测包次数，默认值为9（次） net.ipv4.tcp_keepalive_intvl =15 #KeepAlive探测包的发送间隔，默认值为75s net.ipv4.tcp_max_tw_buckets = 36000 #Nginx 之类的中间代理一定要关注这个值，因为它对你的系统起到一个保护的作用，一旦端口全部被占用，服务就异常了。 tcp_max_tw_buckets 能帮你降低这种情况的发生概率，争取补救时间。 net.ipv4.tcp_tw_reuse = 1 #只对客户端起作用，开启后客户端在1s内回收 net.ipv4.tcp_max_orphans = 327680 #这个值表示系统所能处理不属于任何进程的socket数量，当我们需要快速建立大量连接时，就需要关注下这个值了。 net.ipv4.tcp_orphan_retries = 3 #出现大量fin-wait-1 #首先，fin发送之后，有可能会丢弃，那么发送多少次这样的fin包呢？fin包的重传，也会采用退避方式，在2.6.358内核中采用的是指数退避，2s，4s，最后的重试次数是由tcp_orphan_retries来限制的。 net.ipv4.tcp_syncookies = 1 #tcp_syncookies是一个开关，是否打开SYN Cookie功能，该功能可以防止部分SYN攻击。tcp_synack_retries和tcp_syn_retries定义SYN的重试次数。 net.ipv4.tcp_max_syn_backlog = 16384 #进入SYN包的最大请求队列.默认1024.对重负载服务器,增加该值显然有好处. net.ipv4.ip_conntrack_max = 65536 #表明系统将对最大跟踪的TCP连接数限制默认为65536 net.ipv4.tcp_max_syn_backlog = 16384 #指定所能接受SYN同步包的最大客户端数量，即半连接上限； net.ipv4.tcp_timestamps = 0 #在使用 iptables 做 nat 时，发现内网机器 ping 某个域名 ping 的通，而使用 curl 测试不通, 原来是 net.ipv4.tcp_timestamps 设置了为 1 ，即启用时间戳 net.core.somaxconn = 16384 #Linux中的一个kernel参数，表示socket监听（listen）的backlog上限。什么是backlog呢？backlog就是socket的监听队列，当一个请求（request）尚未被处理或建立时，他会进入backlog。而socket server可以一次性处理backlog中的所有请求，处理后的请求不再位于监听队列中。当server处理请求较慢，以至于监听队列被填满后，新来的请求会被拒绝。

reboot lsmod | grep --color=auto -e ip_vs -e nf_conntrack [root@k8s-master01 ~]# lsmod | grep --color=auto -e ip_vs -e nf_conntrack ip_vs_ftp 16384 0 nf_nat 32768 1 ip_vs_ftp ip_vs_sed 16384 0 ip_vs_nq 16384 0 ip_vs_fo 16384 0 ip_vs_sh 16384 0 ip_vs_dh 16384 0 ip_vs_lblcr 16384 0 ip_vs_lblc 16384 0 ip_vs_wrr 16384 0 ip_vs_rr 16384 0 ip_vs_wlc 16384 0 ip_vs_lc 16384 0 ip_vs 151552 24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp nf_conntrack 143360 2 nf_nat,ip_vs nf_defrag_ipv6 20480 1 nf_conntrack nf_defrag_ipv4 16384 1 nf_conntrack libcrc32c 16384 4 nf_conntrack,nf_nat,xfs,ip_vs

[root@k8s-ha01 ~]# cat install_haproxy.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2022-04-14 #FileName: install_haproxy.sh #Description: The test script #Copyright (C): 2021 All rights reserved #********************************************************************************************* SRC_DIR=/usr/local/src COLOR="echo -e \\033[01;31m" END='\033[0m' CPUS=`lscpu |awk '/^CPU\(s\)/{print $2}'` #lua下载地址：http://www.lua.org/ftp/lua-5.4.4.tar.gz LUA_FILE=lua-5.4.4.tar.gz #haproxy下载地址：https://www.haproxy.org/download/2.6/src/haproxy-2.6.4.tar.gz HAPROXY_FILE=haproxy-2.6.4.tar.gz HAPROXY_INSTALL_DIR=/apps/haproxy STATS_AUTH_USER=admin STATS_AUTH_PASSWORD=123456 VIP=172.31.3.188 MASTER01=172.31.3.101 MASTER02=172.31.3.102 MASTER03=172.31.3.103 HARBOR01=172.31.3.106 HARBOR02=172.31.3.107 os(){ OS_ID=`sed -rn '/^NAME=/s@.*="([[:alpha:]]+).*"$@\1@p' /etc/os-release` } check_file (){ cd ${SRC_DIR} ${COLOR}'检查Haproxy相关源码包'${END} if [ ! -e ${LUA_FILE} ];then ${COLOR}"缺少${LUA_FILE}文件,请把文件放到${SRC_DIR}目录下"${END} exit elif [ ! -e ${HAPROXY_FILE} ];then ${COLOR}"缺少${HAPROXY_FILE}文件,请把文件放到${SRC_DIR}目录下"${END} exit else ${COLOR}"相关文件已准备好"${END} fi } install_haproxy(){ [ -d ${HAPROXY_INSTALL_DIR} ] && { ${COLOR}"Haproxy已存在，安装失败"${END};exit; } ${COLOR}"开始安装Haproxy"${END} ${COLOR}"开始安装Haproxy依赖包"${END} if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ] &> /dev/null;then yum -y install gcc make gcc-c++ glibc glibc-devel pcre pcre-devel openssl openssl-devel systemd-devel libtermcap-devel ncurses-devel libevent-devel readline-devel &> /dev/null else apt update &> /dev/null;apt -y install gcc make openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev libreadline-dev libsystemd-dev &> /dev/null fi tar xf ${LUA_FILE} LUA_DIR=`echo ${LUA_FILE} | sed -nr 's/^(.*[0-9]).([[:lower:]]).*/\1/p'` cd ${LUA_DIR} make all test cd ${SRC_DIR} tar xf ${HAPROXY_FILE} HAPROXY_DIR=`echo ${HAPROXY_FILE} | sed -nr 's/^(.*[0-9]).([[:lower:]]).*/\1/p'` cd ${HAPROXY_DIR} make -j ${CPUS} ARCH=x86_64 TARGET=linux-glibc USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_CPU_AFFINITY=1 USE_LUA=1 LUA_INC=${SRC_DIR}/${LUA_DIR}/src/ LUA_LIB=${SRC_DIR}/${LUA_DIR}/src/ PREFIX=${HAPROXY_INSTALL_DIR} make install PREFIX=${HAPROXY_INSTALL_DIR} [ $? -eq 0 ] && $COLOR"Haproxy编译安装成功"$END || { $COLOR"Haproxy编译安装失败,退出!"$END;exit; } cat > /lib/systemd/system/haproxy.service <<-EOF [Unit] Description=HAProxy Load Balancer After=syslog.target network.target [Service] ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid ExecReload=/bin/kill -USR2 $MAINPID [Install] WantedBy=multi-user.target EOF [ -L /usr/sbin/haproxy ] || ln -s ../..${HAPROXY_INSTALL_DIR}/sbin/haproxy /usr/sbin/ &> /dev/null [ -d /etc/haproxy ] || mkdir /etc/haproxy &> /dev/null [ -d /var/lib/haproxy/ ] || mkdir -p /var/lib/haproxy/ &> /dev/null cat > /etc/haproxy/haproxy.cfg <<-EOF global maxconn 100000 chroot ${HAPROXY_INSTALL_DIR} stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin uid 99 gid 99 daemon pidfile /var/lib/haproxy/haproxy.pid log 127.0.0.1 local3 info defaults option http-keep-alive option forwardfor maxconn 100000 mode http timeout connect 300000ms timeout client 300000ms timeout server 300000ms listen stats mode http bind 0.0.0.0:9999 stats enable log global stats uri /haproxy-status stats auth ${STATS_AUTH_USER}:${STATS_AUTH_PASSWORD} listen kubernetes-6443 bind ${VIP}:6443 mode tcp log global server ${MASTER01} ${MASTER01}:6443 check inter 3s fall 2 rise 5 server ${MASTER02} ${MASTER02}:6443 check inter 3s fall 2 rise 5 server ${MASTER03} ${MASTER03}:6443 check inter 3s fall 2 rise 5 listen harbor-80 bind ${VIP}:80 mode http log global balance source server ${HARBOR01} ${HARBOR01}:80 check inter 3s fall 2 rise 5 server ${HARBOR02} ${HARBOR02}:80 check inter 3s fall 2 rise 5 EOF cat >> /etc/sysctl.conf <<-EOF net.ipv4.ip_nonlocal_bind = 1 EOF sysctl -p &> /dev/null echo "PATH=${HAPROXY_INSTALL_DIR}/sbin:${PATH}" > /etc/profile.d/haproxy.sh systemctl daemon-reload systemctl enable --now haproxy &> /dev/null systemctl is-active haproxy &> /dev/null && ${COLOR}"Haproxy 服务启动成功!"${END} || { ${COLOR}"Haproxy 启动失败,退出!"${END} ; exit; } ${COLOR}"Haproxy安装完成"${END} } main(){ os check_file install_haproxy } main [root@k8s-ha01 ~]# bash install_haproxy.sh [root@k8s-ha02 ~]# bash install_haproxy.sh

[root@k8s-ha02 ~]# cat check_haproxy.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2022-01-09 #FileName: check_haproxy.sh #Description: The test script #Copyright (C): 2022 All rights reserved #********************************************************************************************* err=0 for k in $(seq 1 3);do check_code=$(pgrep haproxy) if [[ $check_code == "" ]]; then err=$(expr $err + 1) sleep 1 continue else err=0 break fi done if [[ $err != "0" ]]; then echo "systemctl stop keepalived" /usr/bin/systemctl stop keepalived exit 1 else exit 0 fi

[root@k8s-ha01 ~]# cat install_keepalived_master.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2021-12-29 #FileName: install_keepalived_master.sh #Description: install_keepalived for CentOS 7/8 & Ubuntu 18.04/20.04 & Rocky 8 #Copyright (C): 2021 All rights reserved #********************************************************************************************* SRC_DIR=/usr/local/src COLOR="echo -e \\033[01;31m" END='\033[0m' KEEPALIVED_URL=https://keepalived.org/software/ KEEPALIVED_FILE=keepalived-2.2.7.tar.gz KEEPALIVED_INSTALL_DIR=/apps/keepalived CPUS=`lscpu |awk '/^CPU\(s\)/{print $2}'` NET_NAME=`ip addr |awk -F"[: ]" '/^2: e.*/{print $3}'` STATE=MASTER PRIORITY=100 VIP=172.31.3.188 os(){ OS_ID=`sed -rn '/^NAME=/s@.*="([[:alpha:]]+).*"$@\1@p' /etc/os-release` OS_RELEASE_VERSION=`sed -rn '/^VERSION_ID=/s@.*="?([0-9]+)\.?.*"?@\1@p' /etc/os-release` } check_file (){ cd ${SRC_DIR} if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ] &> /dev/null;then rpm -q wget &> /dev/null || yum -y install wget &> /dev/null fi if [ ! -e ${KEEPALIVED_FILE} ];then ${COLOR}"缺少${KEEPALIVED_FILE}文件,如果是离线包,请放到${SRC_DIR}目录下"${END} ${COLOR}'开始下载Keepalived源码包'${END} wget ${KEEPALIVED_URL}${KEEPALIVED_FILE} || { ${COLOR}"Keepalived源码包下载失败"${END}; exit; } elif [ ! -e check_haproxy.sh ];then ${COLOR}"缺少check_haproxy.sh文件,请把文件放到${SRC_DIR}目录下"${END} exit else ${COLOR}"相关文件已准备好"${END} fi } install_keepalived(){ [ -d ${KEEPALIVED_INSTALL_DIR} ] && { ${COLOR}"Keepalived已存在，安装失败"${END};exit; } ${COLOR}"开始安装Keepalived"${END} ${COLOR}"开始安装Keepalived依赖包"${END} if [ ${OS_ID} == "Rocky" -a ${OS_RELEASE_VERSION} == 8 ];then URL=mirrors.sjtug.sjtu.edu.cn if [ ! `grep -R "\[PowerTools\]" /etc/yum.repos.d/` ];then cat > /etc/yum.repos.d/PowerTools.repo <<-EOF [PowerTools] name=PowerTools baseurl=https://${URL}/rocky/\$releasever/PowerTools/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial EOF fi fi if [ ${OS_ID} == "CentOS" -a ${OS_RELEASE_VERSION} == 8 ];then URL=mirrors.cloud.tencent.com if [ ! `grep -R "\[PowerTools\]" /etc/yum.repos.d/` ];then cat > /etc/yum.repos.d/PowerTools.repo <<-EOF [PowerTools] name=PowerTools baseurl=https://${URL}/centos/\$releasever/PowerTools/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial EOF fi fi if [[ ${OS_RELEASE_VERSION} == 8 ]] &> /dev/null;then yum -y install make gcc ipvsadm autoconf automake openssl-devel libnl3-devel iptables-devel ipset-devel file-devel net-snmp-devel glib2-devel pcre2-devel libnftnl-devel libmnl-devel systemd-devel &> /dev/null elif [[ ${OS_RELEASE_VERSION} == 7 ]] &> /dev/null;then yum -y install make gcc libnfnetlink-devel libnfnetlink ipvsadm libnl libnl-devel libnl3 libnl3-devel lm_sensors-libs net-snmp-agent-libs net-snmp-libs openssh-server openssh-clients openssl openssl-devel automake iproute &> /dev/null elif [[ ${OS_RELEASE_VERSION} == 20 ]] &> /dev/null;then apt update &> /dev/null;apt -y install make gcc ipvsadm build-essential pkg-config automake autoconf libipset-dev libnl-3-dev libnl-genl-3-dev libssl-dev libxtables-dev libip4tc-dev libip6tc-dev libipset-dev libmagic-dev libsnmp-dev libglib2.0-dev libpcre2-dev libnftnl-dev libmnl-dev libsystemd-dev else apt update &> /dev/null;apt -y install make gcc ipvsadm build-essential pkg-config automake autoconf iptables-dev libipset-dev libnl-3-dev libnl-genl-3-dev libssl-dev libxtables-dev libip4tc-dev libip6tc-dev libipset-dev libmagic-dev libsnmp-dev libglib2.0-dev libpcre2-dev libnftnl-dev libmnl-dev libsystemd-dev &> /dev/null fi tar xf ${KEEPALIVED_FILE} KEEPALIVED_DIR=`echo ${KEEPALIVED_FILE} | sed -nr 's/^(.*[0-9]).([[:lower:]]).*/\1/p'` cd ${KEEPALIVED_DIR} ./configure --prefix=${KEEPALIVED_INSTALL_DIR} --disable-fwmark make -j $CPUS && make install [ $? -eq 0 ] && ${COLOR}"Keepalived编译安装成功"${END} || { ${COLOR}"Keepalived编译安装失败,退出!"${END};exit; } [ -d /etc/keepalived ] || mkdir -p /etc/keepalived &> /dev/null cat > /etc/keepalived/keepalived.conf <<EOF ! Configuration File for keepalived global_defs { router_id LVS_DEVEL script_user root enable_script_security } vrrp_script check_haoroxy { script "/etc/keepalived/check_haproxy.sh" interval 5 weight -5 fall 2 rise 1 } vrrp_instance VI_1 { state ${STATE} interface ${NET_NAME} virtual_router_id 51 priority ${PRIORITY} advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { ${VIP} dev ${NET_NAME} label ${NET_NAME}:1 } track_script { check_haproxy } } EOF cp ./keepalived/keepalived.service /lib/systemd/system/ cd ${SRC_DIR} mv check_haproxy.sh /etc/keepalived/check_haproxy.sh chmod +x /etc/keepalived/check_haproxy.sh echo "PATH=${KEEPALIVED_INSTALL_DIR}/sbin:${PATH}" > /etc/profile.d/keepalived.sh systemctl daemon-reload systemctl enable --now keepalived &> /dev/null systemctl is-active keepalived &> /dev/null && ${COLOR}"Keepalived 服务启动成功!"${END} || { ${COLOR}"Keepalived 启动失败,退出!"${END} ; exit; } ${COLOR}"Keepalived安装完成"${END} } main(){ os check_file install_keepalived } main [root@k8s-ha01 ~]# bash install_keepalived_master.sh [root@k8s-ha01 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:05:9b:2a brd ff:ff:ff:ff:ff:ff inet 172.31.3.104/21 brd 172.31.7.255 scope global eth0 valid_lft forever preferred_lft forever inet 172.31.3.188/32 scope global eth0:1 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe05:9b2a/64 scope link valid_lft forever preferred_lft forever

[root@k8s-ha02 ~]# cat install_keepalived_backup.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2021-12-29 #FileName: install_keepalived_backup.sh #Description: install_keepalived for CentOS 7/8 & Ubuntu 18.04/20.04 & Rocky 8 #Copyright (C): 2021 All rights reserved #********************************************************************************************* SRC_DIR=/usr/local/src COLOR="echo -e \\033[01;31m" END='\033[0m' KEEPALIVED_URL=https://keepalived.org/software/ KEEPALIVED_FILE=keepalived-2.2.7.tar.gz KEEPALIVED_INSTALL_DIR=/apps/keepalived CPUS=`lscpu |awk '/^CPU\(s\)/{print $2}'` NET_NAME=`ip addr |awk -F"[: ]" '/^2: e.*/{print $3}'` STATE=BACKUP PRIORITY=90 VIP=172.31.3.188 os(){ OS_ID=`sed -rn '/^NAME=/s@.*="([[:alpha:]]+).*"$@\1@p' /etc/os-release` OS_RELEASE_VERSION=`sed -rn '/^VERSION_ID=/s@.*="?([0-9]+)\.?.*"?@\1@p' /etc/os-release` } check_file (){ cd ${SRC_DIR} if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ] &> /dev/null;then rpm -q wget &> /dev/null || yum -y install wget &> /dev/null fi if [ ! -e ${KEEPALIVED_FILE} ];then ${COLOR}"缺少${KEEPALIVED_FILE}文件,如果是离线包,请放到${SRC_DIR}目录下"${END} ${COLOR}'开始下载Keepalived源码包'${END} wget ${KEEPALIVED_URL}${KEEPALIVED_FILE} || { ${COLOR}"Keepalived源码包下载失败"${END}; exit; } elif [ ! -e check_haproxy.sh ];then ${COLOR}"缺少check_haproxy.sh文件,请把文件放到${SRC_DIR}目录下"${END} exit else ${COLOR}"相关文件已准备好"${END} fi } install_keepalived(){ [ -d ${KEEPALIVED_INSTALL_DIR} ] && { ${COLOR}"Keepalived已存在，安装失败"${END};exit; } ${COLOR}"开始安装Keepalived"${END} ${COLOR}"开始安装Keepalived依赖包"${END} if [ ${OS_ID} == "Rocky" -a ${OS_RELEASE_VERSION} == 8 ];then URL=mirrors.sjtug.sjtu.edu.cn if [ ! `grep -R "\[PowerTools\]" /etc/yum.repos.d/` ];then cat > /etc/yum.repos.d/PowerTools.repo <<-EOF [PowerTools] name=PowerTools baseurl=https://${URL}/rocky/\$releasever/PowerTools/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial EOF fi fi if [ ${OS_ID} == "CentOS" -a ${OS_RELEASE_VERSION} == 8 ];then URL=mirrors.cloud.tencent.com if [ ! `grep -R "\[PowerTools\]" /etc/yum.repos.d/` ];then cat > /etc/yum.repos.d/PowerTools.repo <<-EOF [PowerTools] name=PowerTools baseurl=https://${URL}/centos/\$releasever/PowerTools/\$basearch/os/ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial EOF fi fi if [[ ${OS_RELEASE_VERSION} == 8 ]] &> /dev/null;then yum -y install make gcc ipvsadm autoconf automake openssl-devel libnl3-devel iptables-devel ipset-devel file-devel net-snmp-devel glib2-devel pcre2-devel libnftnl-devel libmnl-devel systemd-devel &> /dev/null elif [[ ${OS_RELEASE_VERSION} == 7 ]] &> /dev/null;then yum -y install make gcc libnfnetlink-devel libnfnetlink ipvsadm libnl libnl-devel libnl3 libnl3-devel lm_sensors-libs net-snmp-agent-libs net-snmp-libs openssh-server openssh-clients openssl openssl-devel automake iproute &> /dev/null elif [[ ${OS_RELEASE_VERSION} == 20 ]] &> /dev/null;then apt update &> /dev/null;apt -y install make gcc ipvsadm build-essential pkg-config automake autoconf libipset-dev libnl-3-dev libnl-genl-3-dev libssl-dev libxtables-dev libip4tc-dev libip6tc-dev libipset-dev libmagic-dev libsnmp-dev libglib2.0-dev libpcre2-dev libnftnl-dev libmnl-dev libsystemd-dev else apt update &> /dev/null;apt -y install make gcc ipvsadm build-essential pkg-config automake autoconf iptables-dev libipset-dev libnl-3-dev libnl-genl-3-dev libssl-dev libxtables-dev libip4tc-dev libip6tc-dev libipset-dev libmagic-dev libsnmp-dev libglib2.0-dev libpcre2-dev libnftnl-dev libmnl-dev libsystemd-dev &> /dev/null fi tar xf ${KEEPALIVED_FILE} KEEPALIVED_DIR=`echo ${KEEPALIVED_FILE} | sed -nr 's/^(.*[0-9]).([[:lower:]]).*/\1/p'` cd ${KEEPALIVED_DIR} ./configure --prefix=${KEEPALIVED_INSTALL_DIR} --disable-fwmark make -j $CPUS && make install [ $? -eq 0 ] && ${COLOR}"Keepalived编译安装成功"${END} || { ${COLOR}"Keepalived编译安装失败,退出!"${END};exit; } [ -d /etc/keepalived ] || mkdir -p /etc/keepalived &> /dev/null cat > /etc/keepalived/keepalived.conf <<EOF ! Configuration File for keepalived global_defs { router_id LVS_DEVEL script_user root enable_script_security } vrrp_script check_haoroxy { script "/etc/keepalived/check_haproxy.sh" interval 5 weight -5 fall 2 rise 1 } vrrp_instance VI_1 { state ${STATE} interface ${NET_NAME} virtual_router_id 51 priority ${PRIORITY} advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { ${VIP} dev ${NET_NAME} label ${NET_NAME}:1 } track_script { check_haproxy } } EOF cp ./keepalived/keepalived.service /lib/systemd/system/ cd ${SRC_DIR} mv check_haproxy.sh /etc/keepalived/check_haproxy.sh chmod +x /etc/keepalived/check_haproxy.sh echo "PATH=${KEEPALIVED_INSTALL_DIR}/sbin:${PATH}" > /etc/profile.d/keepalived.sh systemctl daemon-reload systemctl enable --now keepalived &> /dev/null systemctl is-active keepalived &> /dev/null && ${COLOR}"Keepalived 服务启动成功!"${END} || { ${COLOR}"Keepalived 启动失败,退出!"${END} ; exit; } ${COLOR}"Keepalived安装完成"${END} } main(){ os check_file install_keepalived } main [root@k8s-ha02 ~]# bash install_keepalived_backup.sh

172.31.3.188 kubeapi.raymonds.cc

http://kubeapi.raymonds.cc:9999/haproxy-status

[root@k8s-harbor01 ~]# cat install_docker_binary_compose_harbor.sh #!/bin/bash # #************************************************************************************************** #Author: Raymond #Date: 2021-12-15 #FileName: install_docker_binary_compose_harbor.sh #Description: install_docker_binary_compose_harbor for CentOS 7/8 & Ubuntu 18.04/20.04 & Rocky 8 #Copyright (C): 2021 All rights reserved #************************************************************************************************** SRC_DIR=/usr/local/src COLOR="echo -e \\033[01;31m" END='\033[0m' URL='https://mirrors.cloud.tencent.com/docker-ce/linux/static/stable/x86_64/' DOCKER_FILE=docker-20.10.18.tgz #docker-compose下载地址:https://github.com/docker/compose/releases/download/v2.10.2/docker-compose-linux-x86_64 DOCKER_COMPOSE_FILE=docker-compose-linux-x86_64 #harbor下载地址:https://github.com/goharbor/harbor/releases/download/v2.6.0/harbor-offline-installer-v2.6.0.tgz HARBOR_FILE=harbor-offline-installer-v HARBOR_VERSION=2.6.0 TAR=.tgz HARBOR_INSTALL_DIR=/apps HARBOR_DOMAIN=harbor.raymonds.cc NET_NAME=`ip addr |awk -F"[: ]" '/^2: e.*/{print $3}'` IP=`ip addr show ${NET_NAME}| awk -F" +|/" '/global/{print $3}'` HARBOR_ADMIN_PASSWORD=123456 os(){ OS_ID=`sed -rn '/^NAME=/s@.*="([[:alpha:]]+).*"$@\1@p' /etc/os-release` OS_RELEASE_VERSION=`sed -rn '/^VERSION_ID=/s@.*="?([0-9]+)\.?.*"?@\1@p' /etc/os-release` } check_file (){ cd ${SRC_DIR} if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ] &> /dev/null;then rpm -q wget &> /dev/null || yum -y install wget &> /dev/null fi if [ ! -e ${DOCKER_FILE} ];then ${COLOR}"缺少${DOCKER_FILE}文件,如果是离线包,请把文件放到${SRC_DIR}目录下"${END} ${COLOR}'开始下载DOCKER二进制源码包'${END} wget ${URL}${DOCKER_FILE} || { ${COLOR}"DOCKER二进制安装包下载失败"${END}; exit; } elif [ ! -e ${DOCKER_COMPOSE_FILE} ];then ${COLOR}"缺少${DOCKER_COMPOSE_FILE}文件,请把文件放到${SRC_DIR}目录下"${END} exit elif [ ! -e ${HARBOR_FILE}${HARBOR_VERSION}${TAR} ];then ${COLOR}"缺少${HARBOR_FILE}${HARBOR_VERSION}${TAR}文件,请把文件放到${SRC_DIR}目录下"${END} exit else ${COLOR}"相关文件已准备好"${END} fi } install_docker(){ tar xf ${DOCKER_FILE} mv docker/* /usr/bin/ cat > /lib/systemd/system/docker.service <<-EOF [Unit] Description=Docker Application Container Engine Documentation=https://docs.docker.com After=network-online.target firewalld.service Wants=network-online.target [Service] Type=notify # the default is not to use systemd for cgroups because the delegate issues still # exists and systemd currently does not support the cgroup feature set required # for containers run by docker ExecStart=/usr/bin/dockerd -H unix://var/run/docker.sock ExecReload=/bin/kill -s HUP \$MAINPID # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. LimitNOFILE=infinity LimitNPROC=infinity LimitCORE=infinity # Uncomment TasksMax if your systemd version supports it. # Only systemd 226 and above support this version. #TasksMax=infinity TimeoutStartSec=0 # set delegate yes so that systemd does not reset the cgroups of docker containers Delegate=yes # kill only the docker process, not all processes in the cgroup KillMode=process # restart the docker process if it exits prematurely Restart=on-failure StartLimitBurst=3 StartLimitInterval=60s [Install] WantedBy=multi-user.target EOF mkdir -p /etc/docker cat > /etc/docker/daemon.json <<-EOF { "registry-mirrors": [ "https://registry.docker-cn.com", "http://hub-mirror.c.163.com", "https://docker.mirrors.ustc.edu.cn" ], "insecure-registries": ["${HARBOR_DOMAIN}"], "exec-opts": ["native.cgroupdriver=systemd"], "max-concurrent-downloads": 10, "max-concurrent-uploads": 5, "log-opts": { "max-size": "300m", "max-file": "2" }, "live-restore": true } EOF echo 'alias rmi="docker images -qa|xargs docker rmi -f"' >> ~/.bashrc echo 'alias rmc="docker ps -qa|xargs docker rm -f"' >> ~/.bashrc systemctl daemon-reload systemctl enable --now docker &> /dev/null systemctl is-active docker &> /dev/null && ${COLOR}"Docker 服务启动成功"${END} || { ${COLOR}"Docker 启动失败"${END};exit; } docker version && ${COLOR}"Docker 安装成功"${END} || ${COLOR}"Docker 安装失败"${END} } install_docker_compose(){ ${COLOR}"开始安装 Docker compose....."${END} sleep 1 mv ${SRC_DIR}/${DOCKER_COMPOSE_FILE} /usr/bin/docker-compose chmod +x /usr/bin/docker-compose docker-compose --version && ${COLOR}"Docker Compose 安装完成"${END} || ${COLOR}"Docker compose 安装失败"${END} } install_harbor(){ ${COLOR}"开始安装 Harbor....."${END} sleep 1 [ -d ${HARBOR_INSTALL_DIR} ] || mkdir ${HARBOR_INSTALL_DIR} tar xf ${SRC_DIR}/${HARBOR_FILE}${HARBOR_VERSION}${TAR} -C ${HARBOR_INSTALL_DIR}/ mv ${HARBOR_INSTALL_DIR}/harbor/harbor.yml.tmpl ${HARBOR_INSTALL_DIR}/harbor/harbor.yml sed -ri.bak -e 's/^(hostname:) .*/\1 '${IP}'/' -e 's/^(harbor_admin_password:) .*/\1 '${HARBOR_ADMIN_PASSWORD}'/' -e 's/^(https:)/#\1/' -e 's/ (port: 443)/# \1/' -e 's@ (certificate: .*)@# \1@' -e 's@ (private_key: .*)@# \1@' ${HARBOR_INSTALL_DIR}/harbor/harbor.yml if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ] &> /dev/null;then yum -y install python3 &> /dev/null || { ${COLOR}"安装软件包失败，请检查网络配置"${END}; exit; } else apt -y install python3 &> /dev/null || { ${COLOR}"安装软件包失败，请检查网络配置"${END}; exit; } fi ${HARBOR_INSTALL_DIR}/harbor/install.sh && ${COLOR}"Harbor 安装完成"${END} || ${COLOR}"Harbor 安装失败"${END} cat > /lib/systemd/system/harbor.service <<-EOF [Unit] Description=Harbor After=docker.service systemd-networkd.service systemd-resolved.service Requires=docker.service Documentation=http://github.com/vmware/harbor [Service] Type=simple Restart=on-failure RestartSec=5 ExecStart=/usr/bin/docker-compose -f /apps/harbor/docker-compose.yml up ExecStop=/usr/bin/docker-compose -f /apps/harbor/docker-compose.yml down [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable harbor &>/dev/null && ${COLOR}"Harbor已配置为开机自动启动"${END} } set_swap_limit(){ if [ ${OS_ID} == "Ubuntu" ];then ${COLOR}'设置Docker的"WARNING: No swap limit support"警告'${END} sed -ri '/^GRUB_CMDLINE_LINUX=/s@"$@ swapaccount=1"@' /etc/default/grub update-grub &> /dev/null ${COLOR}"10秒后，机器会自动重启"${END} sleep 10 reboot fi } main(){ os check_file [ -f /usr/bin/docker ] && ${COLOR}"Docker已安装"${END} || install_docker docker-compose --version &> /dev/null && ${COLOR}"Docker Compose已安装"${END} || install_docker_compose systemctl is-active harbor &> /dev/null && ${COLOR}"Harbor已安装"${END} || install_harbor grep -q "swapaccount=1" /etc/default/grub && ${COLOR}'"WARNING: No swap limit support"警告,已设置'${END} || set_swap_limit } main [root@k8s-harbor01 ~]# bash install_docker_compose_harbor.sh [root@k8s-harbor02 ~]# bash install_docker_compose_harbor.sh

[root@k8s-etcd01 ~]# wget https://github.com/etcd-io/etcd/releases/download/v3.5.4/etcd-v3.5.4-linux-amd64.tar.gz

[root@k8s-etcd01 ~]# tar -xf etcd-v3.5.4-linux-amd64.tar.gz --strip-compnotallow=1 -C /usr/local/bin etcd-v3.5.4-linux-amd64/etcd{,ctl}

[root@k8s-etcd01 ~]# etcdctl version etcdctl version: 3.5.4 API version: 3.5

[root@k8s-etcd01 ~]# for NODE in k8s-etcd02 k8s-etcd03; do echo $NODE; scp -o StrictHostKeyChecking=no /usr/local/bin/etcd* $NODE:/usr/local/bin/; done

[root@k8s-etcd01 ~]# mkdir -p /opt/cni/bin [root@k8s-etcd02 ~]# mkdir -p /opt/cni/bin [root@k8s-etcd03 ~]# mkdir -p /opt/cni/bin

[root@k8s-etcd01 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssl_1.6.1_linux_amd64 [root@k8s-etcd01 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssljson_1.6.1_linux_amd64 [root@k8s-etcd01 ~]# mv cfssl_1.6.1_linux_amd64 /usr/local/bin/cfssl [root@k8s-etcd01 ~]# mv cfssljson_1.6.1_linux_amd64 /usr/local/bin/cfssljson [root@k8s-etcd01 ~]# chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

[root@k8s-etcd01 ~]# mkdir /etc/etcd/ssl -p [root@k8s-etcd02 ~]# mkdir /etc/etcd/ssl -p [root@k8s-etcd03 ~]# mkdir /etc/etcd/ssl -p

[root@k8s-etcd01 ~]# mkdir pki [root@k8s-etcd01 ~]# cd pki/ [root@k8s-etcd01 pki]# cat > etcd-ca-csr.json <<EOF { "CN": "etcd", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "etcd", "OU": "Etcd Security" } ], "ca": { "expiry": "876000h" } } EOF # 生成etcd CA证书和CA证书的key [root@k8s-etcd01 pki]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca #执行结果 2022/09/25 16:22:04 [INFO] generating a new CA key and certificate from CSR 2022/09/25 16:22:04 [INFO] generate received request 2022/09/25 16:22:04 [INFO] received CSR 2022/09/25 16:22:04 [INFO] generating key: rsa-2048 2022/09/25 16:22:04 [INFO] encoded CSR 2022/09/25 16:22:04 [INFO] signed certificate with serial number 555179846778058823596990590382129342677441232285 [root@k8s-etcd01 pki]# ll /etc/etcd/ssl/etcd-ca* -rw-r--r-- 1 root root 1050 Sep 25 16:22 /etc/etcd/ssl/etcd-ca.csr -rw------- 1 root root 1679 Sep 25 16:22 /etc/etcd/ssl/etcd-ca-key.pem -rw-r--r-- 1 root root 1318 Sep 25 16:22 /etc/etcd/ssl/etcd-ca.pem [root@k8s-etcd01 pki]# cat > ca-config.json <<EOF { "signing": { "default": { "expiry": "876000h" }, "profiles": { "kubernetes": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "876000h" } } } } EOF [root@k8s-etcd01 pki]# cat > etcd-csr.json <<EOF { "CN": "etcd", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "etcd", "OU": "Etcd Security" } ] } EOF [root@k8s-etcd01 pki]# cfssl gencert \ -ca=/etc/etcd/ssl/etcd-ca.pem \ -ca-key=/etc/etcd/ssl/etcd-ca-key.pem \ -config=ca-config.json \ -hostname=127.0.0.1,k8s-etcd01,k8s-etcd02,k8s-etcd03,172.31.3.108,172.31.3.109,172.31.3.110 \ -profile=kubernetes \ etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd #执行结果 2022/09/25 16:22:51 [INFO] generate received request 2022/09/25 16:22:51 [INFO] received CSR 2022/09/25 16:22:51 [INFO] generating key: rsa-2048 2022/09/25 16:22:51 [INFO] encoded CSR 2022/09/25 16:22:51 [INFO] signed certificate with serial number 620811890976823698243438547142971255657636352843 [root@k8s-etcd01 pki]# ll /etc/etcd/ssl/etcd* ... -rw-r--r-- 1 root root 1123 Sep 25 16:22 /etc/etcd/ssl/etcd.csr -rw------- 1 root root 1675 Sep 25 16:22 /etc/etcd/ssl/etcd-key.pem -rw-r--r-- 1 root root 1456 Sep 25 16:22 /etc/etcd/ssl/etcd.pem

[root@k8s-etcd01 pki]# for NODE in k8s-etcd02 k8s-etcd03; do ssh -o StrictHostKeyChecking=no $NODE "mkdir -p /etc/etcd/ssl" for FILE in etcd-ca-key.pem etcd-ca.pem etcd-key.pem etcd.pem; do scp -o StrictHostKeyChecking=no /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE} done done

[root@k8s-etcd01 pki]# cat > /etc/etcd/etcd.config.yml <<EOF name: 'k8s-etcd01' data-dir: /var/lib/etcd wal-dir: /var/lib/etcd/wal snapshot-count: 5000 heartbeat-interval: 100 election-timeout: 1000 quota-backend-bytes: 0 listen-peer-urls: 'https://172.31.3.108:2380' listen-client-urls: 'https://172.31.3.108:2379,http://127.0.0.1:2379' max-snapshots: 3 max-wals: 5 cors: initial-advertise-peer-urls: 'https://172.31.3.108:2380' advertise-client-urls: 'https://172.31.3.108:2379' discovery: discovery-fallback: 'proxy' discovery-proxy: discovery-srv: initial-cluster: 'k8s-etcd01=https://172.31.3.108:2380,k8s-etcd02=https://172.31.3.109:2380,k8s-etcd03=https://172.31.3.110:2380' initial-cluster-token: 'etcd-k8s-cluster' initial-cluster-state: 'new' strict-reconfig-check: false enable-v2: true enable-pprof: true proxy: 'off' proxy-failure-wait: 5000 proxy-refresh-interval: 30000 proxy-dial-timeout: 1000 proxy-write-timeout: 5000 proxy-read-timeout: 0 client-transport-security: cert-file: '/etc/kubernetes/pki/etcd/etcd.pem' key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem' client-cert-auth: true trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem' auto-tls: true peer-transport-security: cert-file: '/etc/kubernetes/pki/etcd/etcd.pem' key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem' peer-client-cert-auth: true trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem' auto-tls: true debug: false log-package-levels: log-outputs: [default] force-new-cluster: false EOF

[root@k8s-etcd02 ~]# cat > /etc/etcd/etcd.config.yml <<EOF name: 'k8s-etcd02' data-dir: /var/lib/etcd wal-dir: /var/lib/etcd/wal snapshot-count: 5000 heartbeat-interval: 100 election-timeout: 1000 quota-backend-bytes: 0 listen-peer-urls: 'https://172.31.3.109:2380' listen-client-urls: 'https://172.31.3.109:2379,http://127.0.0.1:2379' max-snapshots: 3 max-wals: 5 cors: initial-advertise-peer-urls: 'https://172.31.3.109:2380' advertise-client-urls: 'https://172.31.3.109:2379' discovery: discovery-fallback: 'proxy' discovery-proxy: discovery-srv: initial-cluster: 'k8s-etcd01=https://172.31.3.108:2380,k8s-etcd02=https://172.31.3.109:2380,k8s-etcd03=https://172.31.3.110:2380' initial-cluster-token: 'etcd-k8s-cluster' initial-cluster-state: 'new' strict-reconfig-check: false enable-v2: true enable-pprof: true proxy: 'off' proxy-failure-wait: 5000 proxy-refresh-interval: 30000 proxy-dial-timeout: 1000 proxy-write-timeout: 5000 proxy-read-timeout: 0 client-transport-security: cert-file: '/etc/kubernetes/pki/etcd/etcd.pem' key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem' client-cert-auth: true trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem' auto-tls: true peer-transport-security: cert-file: '/etc/kubernetes/pki/etcd/etcd.pem' key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem' peer-client-cert-auth: true trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem' auto-tls: true debug: false log-package-levels: log-outputs: [default] force-new-cluster: false EOF

[root@k8s-etcd03 ~]# cat > /etc/etcd/etcd.config.yml <<EOF name: 'k8s-etcd03' data-dir: /var/lib/etcd wal-dir: /var/lib/etcd/wal snapshot-count: 5000 heartbeat-interval: 100 election-timeout: 1000 quota-backend-bytes: 0 listen-peer-urls: 'https://172.31.3.110:2380' listen-client-urls: 'https://172.31.3.110:2379,http://127.0.0.1:2379' max-snapshots: 3 max-wals: 5 cors: initial-advertise-peer-urls: 'https://172.31.3.110:2380' advertise-client-urls: 'https://172.31.3.110:2379' discovery: discovery-fallback: 'proxy' discovery-proxy: discovery-srv: initial-cluster: 'k8s-etcd01=https://172.31.3.108:2380,k8s-etcd02=https://172.31.3.109:2380,k8s-etcd03=https://172.31.3.110:2380' initial-cluster-token: 'etcd-k8s-cluster' initial-cluster-state: 'new' strict-reconfig-check: false enable-v2: true enable-pprof: true proxy: 'off' proxy-failure-wait: 5000 proxy-refresh-interval: 30000 proxy-dial-timeout: 1000 proxy-write-timeout: 5000 proxy-read-timeout: 0 client-transport-security: cert-file: '/etc/kubernetes/pki/etcd/etcd.pem' key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem' client-cert-auth: true trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem' auto-tls: true peer-transport-security: cert-file: '/etc/kubernetes/pki/etcd/etcd.pem' key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem' peer-client-cert-auth: true trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem' auto-tls: true debug: false log-package-levels: log-outputs: [default] force-new-cluster: false EOF

[root@k8s-etcd01 pki]# cat > /lib/systemd/system/etcd.service <<EOF [Unit] Description=Etcd Service Documentation=https://coreos.com/etcd/docs/latest/ After=network.target [Service] Type=notify ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml Restart=on-failure RestartSec=10 LimitNOFILE=65536 [Install] WantedBy=multi-user.target Alias=etcd3.service EOF [root@k8s-etcd01 pki]# for NODE in k8s-etcd02 k8s-etcd03; do scp /lib/systemd/system/etcd.service $NODE:/lib/systemd/system/;done

[root@k8s-etcd01 pki]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-etcd02 ~]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-etcd03 ~]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-etcd01 pki]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/ [root@k8s-etcd02 ~]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/ [root@k8s-etcd03 ~]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/ [root@k8s-etcd01 pki]# systemctl daemon-reload && systemctl enable --now etcd [root@k8s-etcd02 ~]# systemctl daemon-reload && systemctl enable --now etcd [root@k8s-etcd03 ~]# systemctl daemon-reload && systemctl enable --now etcd [root@k8s-etcd01 pki]# systemctl status etcd ● etcd.service - Etcd Service Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled) Active: active (running) since Sun 2022-09-25 16:24:46 CST; 8s ago Docs: https://coreos.com/etcd/docs/latest/ Main PID: 5085 (etcd) Tasks: 8 (limit: 11192) Memory: 26.5M CGroup: /system.slice/etcd.service └─5085 /usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml Sep 25 16:24:47 k8s-etcd01.example.local etcd[5085]: {"level":"info","ts":"2022-09-25T16:24:47.996+0800","caller":"rafthttp/stream.go:249","ms> Sep 25 16:24:47 k8s-etcd01.example.local etcd[5085]: {"level":"info","ts":"2022-09-25T16:24:47.996+0800","caller":"rafthttp/stream.go:274","ms> Sep 25 16:24:48 k8s-etcd01.example.local etcd[5085]: {"level":"info","ts":"2022-09-25T16:24:48.017+0800","caller":"etcdserver/server.go:775","> Sep 25 16:24:48 k8s-etcd01.example.local etcd[5085]: {"level":"warn","ts":"2022-09-25T16:24:48.034+0800","caller":"rafthttp/probing_status.go:> Sep 25 16:24:48 k8s-etcd01.example.local etcd[5085]: {"level":"warn","ts":"2022-09-25T16:24:48.034+0800","caller":"rafthttp/probing_status.go:> Sep 25 16:24:48 k8s-etcd01.example.local etcd[5085]: {"level":"info","ts":"2022-09-25T16:24:48.059+0800","caller":"rafthttp/stream.go:412","ms> Sep 25 16:24:48 k8s-etcd01.example.local etcd[5085]: {"level":"info","ts":"2022-09-25T16:24:48.067+0800","caller":"rafthttp/stream.go:412","ms> Sep 25 16:24:50 k8s-etcd01.example.local etcd[5085]: {"level":"info","ts":"2022-09-25T16:24:50.670+0800","caller":"etcdserver/server.go:2512",> Sep 25 16:24:50 k8s-etcd01.example.local etcd[5085]: {"level":"info","ts":"2022-09-25T16:24:50.672+0800","caller":"membership/cluster.go:576",> Sep 25 16:24:50 k8s-etcd01.example.local etcd[5085]: {"level":"info","ts":"2022-09-25T16:24:50.672+0800","caller":"etcdserver/server.go:2531",> [root@k8s-etcd02 ~]# systemctl status etcd [root@k8s-etcd03 ~]# systemctl status etcd

[root@k8s-etcd01 pki]# export ETCDCTL_API=3 [root@k8s-etcd01 pki]# etcdctl --endpoints="172.31.3.108:2379,172.31.3.109:2379,172.31.3.110:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem endpoint status --write-out=table +-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | +-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | 172.31.3.108:2379 | a9fef56ff96ed75c | 3.5.4 | 20 kB | true | false | 2 | 9 | 9 | | | 172.31.3.109:2379 | 8319ef09e8b3d277 | 3.5.4 | 20 kB | false | false | 2 | 9 | 9 | | | 172.31.3.110:2379 | 209a1f57c506dba2 | 3.5.4 | 20 kB | false | false | 2 | 9 | 9 | | +-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

#测试VIP [root@k8s-master01 pki]# ping 172.31.3.188 PING 172.31.3.188 (172.31.3.188) 56(84) bytes of data. 64 bytes from 172.31.3.188: icmp_seq=1 ttl=64 time=1.27 ms 64 bytes from 172.31.3.188: icmp_seq=2 ttl=64 time=0.585 ms ^C --- 172.31.3.188 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1002ms rtt min/avg/max/mdev = 0.585/0.931/1.277/0.346 ms [root@k8s-ha01 ~]# systemctl stop keepalived [root@k8s-ha01 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:05:9b:2a brd ff:ff:ff:ff:ff:ff inet 172.31.3.104/21 brd 172.31.7.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe05:9b2a/64 scope link valid_lft forever preferred_lft forever [root@k8s-ha02 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:5e:d8:f8 brd ff:ff:ff:ff:ff:ff inet 172.31.3.105/21 brd 172.31.7.255 scope global eth0 valid_lft forever preferred_lft forever inet 172.31.3.188/32 scope global eth0:1 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe5e:d8f8/64 scope link valid_lft forever preferred_lft forever [root@k8s-ha01 ~]# systemctl start keepalived [root@k8s-ha01 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:05:9b:2a brd ff:ff:ff:ff:ff:ff inet 172.31.3.104/21 brd 172.31.7.255 scope global eth0 valid_lft forever preferred_lft forever inet 172.31.3.188/32 scope global eth0:1 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe05:9b2a/64 scope link valid_lft forever preferred_lft forever [root@k8s-master01 pki]# telnet 172.31.3.188 6443 Trying 172.31.3.188... Connected to 172.31.3.188. Escape character is '^]'. Connection closed by foreign host.

[root@k8s-master01 ~]# cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf overlay br_netfilter EOF

[root@k8s-master01 ~]# modprobe -- overlay [root@k8s-master01 ~]# modprobe -- br_netfilter

[root@k8s-master01 ~]# cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 EOF

[root@k8s-master01 ~]# sysctl --system

https://github.com/containerd/containerd

[root@k8s-master01 ~]# tar tf containerd-1.6.8-linux-amd64.tar.gz bin/ bin/containerd-stress bin/containerd-shim-runc-v2 bin/containerd-shim bin/ctr bin/containerd bin/containerd-shim-runc-v1

[root@k8s-master01 ~]# tar tf cri-containerd-1.6.8-linux-amd64.tar.gz etc/crictl.yaml etc/systemd/ etc/systemd/system/ etc/systemd/system/containerd.service usr/ usr/local/ usr/local/sbin/ usr/local/sbin/runc usr/local/bin/ usr/local/bin/containerd-stress usr/local/bin/containerd-shim-runc-v2 usr/local/bin/containerd-shim usr/local/bin/ctr usr/local/bin/containerd usr/local/bin/critest usr/local/bin/ctd-decoder usr/local/bin/crictl usr/local/bin/containerd-shim-runc-v1 opt/containerd/ opt/containerd/cluster/ opt/containerd/cluster/gce/ opt/containerd/cluster/gce/env opt/containerd/cluster/gce/cloud-init/ opt/containerd/cluster/gce/cloud-init/master.yaml opt/containerd/cluster/gce/cloud-init/node.yaml opt/containerd/cluster/gce/cni.template opt/containerd/cluster/gce/configure.sh opt/containerd/cluster/version

[root@k8s-master01 ~]# tar tf cri-containerd-cni-1.6.8-linux-amd64.tar.gz etc/ etc/crictl.yaml etc/systemd/ etc/systemd/system/ etc/systemd/system/containerd.service etc/cni/ etc/cni/net.d/ etc/cni/net.d/10-containerd-net.conflist usr/ usr/local/ usr/local/sbin/ usr/local/sbin/runc usr/local/bin/ usr/local/bin/containerd-stress usr/local/bin/containerd-shim-runc-v2 usr/local/bin/containerd-shim usr/local/bin/ctr usr/local/bin/containerd usr/local/bin/critest usr/local/bin/ctd-decoder usr/local/bin/crictl usr/local/bin/containerd-shim-runc-v1 opt/ opt/containerd/ opt/containerd/cluster/ opt/containerd/cluster/gce/ opt/containerd/cluster/gce/env opt/containerd/cluster/gce/cloud-init/ opt/containerd/cluster/gce/cloud-init/master.yaml opt/containerd/cluster/gce/cloud-init/node.yaml opt/containerd/cluster/gce/cni.template opt/containerd/cluster/gce/configure.sh opt/containerd/cluster/version opt/cni/ opt/cni/bin/ opt/cni/bin/bandwidth opt/cni/bin/loopback opt/cni/bin/ipvlan opt/cni/bin/host-local opt/cni/bin/static opt/cni/bin/vlan opt/cni/bin/tuning opt/cni/bin/host-device opt/cni/bin/firewall opt/cni/bin/portmap opt/cni/bin/sbr opt/cni/bin/macvlan opt/cni/bin/bridge opt/cni/bin/dhcp opt/cni/bin/ptp opt/cni/bin/vrf

[root@k8s-master01 ~]# wget https://github.com/containerd/containerd/releases/download/v1.6.8/cri-containerd-1.6.8-linux-amd64.tar.gz #cri-containerd-1.6.8-linux-amd64.tar.gz 压缩包中已经按照官方二进制部署推荐的目录结构布局好。 里面包含了 systemd 配置文件，containerd 和ctr、crictl等部署文件。 将解压缩到系统的根目录 / 中: [root@k8s-master01 ~]# tar xf cri-containerd-1.6.8-linux-amd64.tar.gz -C /

[root@k8s-master01 ~]# mkdir -p /etc/containerd [root@k8s-master01 ~]# containerd config default | tee /etc/containerd/config.toml

[root@k8s-master01 ~]# vim /etc/containerd/config.toml ... [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] ... SystemdCgroup = true #SystemdCgroup改成true ... sandbox_image = "harbor.raymonds.cc/google_containers/pause:3.8" #sandbox_image的镜像改为私有仓库“harbor.raymonds.cc/google_containers/pause:3.8”，如果没有私有仓库改为阿里镜像源“registry.aliyuncs.com/google_containers/pause:3.8” #使用下面命令修改 sed -ri -e 's/(.*SystemdCgroup = ).*/\1true/' -e 's@(.*sandbox_image = ).*@\1\"harbor.raymonds.cc/google_containers/pause:3.8\"@' /etc/containerd/config.toml #如果没有harbor，请执行下面命令 sed -ri -e 's/(.*SystemdCgroup = ).*/\1true/' -e 's@(.*sandbox_image = ).*@\1\"registry.aliyuncs.com/google_containers/pause:3.8\"@' /etc/containerd/config.toml

[root@k8s-master01 ~]# vim /etc/containerd/config.toml ... [plugins."io.containerd.grpc.v1.cri".registry] ... #下面几行是配置私有仓库授权，如果没有私有仓库下面的不用设置 [plugins."io.containerd.grpc.v1.cri".registry.configs] [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.raymonds.cc".tls] insecure_skip_verify = true [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.raymonds.cc".auth] username = "admin" password = "123456" ... [plugins."io.containerd.grpc.v1.cri".registry.mirrors] #下面两行是配置镜像加速 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] endpoint = ["https://registry.docker-cn.com" ,"http://hub-mirror.c.163.com" ,"https://docker.mirrors.ustc.edu.cn"] #下面两行是配置私有仓库，如果没有私有仓库下面的不用设置 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.raymonds.cc"] endpoint = ["http://harbor.raymonds.cc"] ... #使用下面命令修改 sed -i -e '/.*registry.mirrors.*/a\ [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]\n endpoint = ["https://registry.docker-cn.com" ,"http://hub-mirror.c.163.com" ,"https://docker.mirrors.ustc.edu.cn"]\n [plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.raymonds.cc"]\n endpoint = ["http://harbor.raymonds.cc"]' -e '/.*registry.configs.*/a\ [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.raymonds.cc".tls]\n insecure_skip_verify = true\n [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.raymonds.cc".auth]\n username = "admin"\n password = "123456"' /etc/containerd/config.toml #如果没有harbor不需要设置私有仓库相关配置，只需要设置镜像加速，请使用下面命令执行 sed -i '/.*registry.mirrors.*/a\ [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]\n endpoint = ["https://registry.docker-cn.com" ,"http://hub-mirror.c.163.com" ,"https://docker.mirrors.ustc.edu.cn"]' /etc/containerd/config.toml

[root@k8s-master01 ~]# cat > /etc/crictl.yaml <<EOF runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock timeout: 10 debug: false EOF

[root@k8s-master01 ~]# systemctl daemon-reload && systemctl enable --now containerd

[root@k8s-master01 ~]# ctr version Client: Version: v1.6.8 Revision: 9cd3357b7fd7218e4aec3eae239db1f68a5a6ec6 Go version: go1.17.13 Server: Version: v1.6.8 Revision: 9cd3357b7fd7218e4aec3eae239db1f68a5a6ec6 UUID: 18d9c9c1-27cc-4883-be10-baf17a186aad [root@k8s-master01 ~]# crictl version Version: 0.1.0 RuntimeName: containerd RuntimeVersion: v1.6.8 RuntimeApiVersion: v1 [root@k8s-master01 ~]# crictl info ... }, ... "lastCNILoadStatus": "cni config load failed: no network config found in /etc/cni/net.d: cni plugin not initialized: failed to load cni config", "lastCNILoadStatus.default": "cni config load failed: no network config found in /etc/cni/net.d: cni plugin not initialized: failed to load cni config" } #这里cni插件报错，不用管，因为没有装containerd的CNI插件，kubernetes里不需要containerd的CNI插件，装上了还会冲突，后边安装flannel或calico的CNI插件

[root@k8s-master01 ~]# wget https://github.com/containerd/nerdctl/releases/download/v0.23.0/nerdctl-0.23.0-linux-amd64.tar.gz [root@k8s-master01 ~]# tar xf nerdctl-0.23.0-linux-amd64.tar.gz -C /usr/local/bin/ #配置nerdctl [root@k8s-master01 ~]# mkdir -p /etc/nerdctl/ [root@k8s-master01 ~]# cat > /etc/nerdctl/nerdctl.toml <<EOF namespace = "k8s.io" #设置nerdctl工具默认namespace insecure_registry = true #跳过安全镜像仓库检测 EOF

[root@k8s-master01 ~]# wget https://github.com/moby/buildkit/releases/download/v0.10.4/buildkit-v0.10.4.linux-amd64.tar.gz [root@k8s-master01 ~]# tar xf buildkit-v0.10.4.linux-amd64.tar.gz -C /usr/local/

[root@k8s-master01 ~]# cat > /usr/lib/systemd/system/buildkit.socket <<EOF [Unit] Description=BuildKit Documentation=https://github.com/moby/buildkit [Socket] ListenStream=%t/buildkit/buildkitd.sock SocketMode=0660 [Install] WantedBy=sockets.target EOF

[root@k8s-master01 ~]# cat > /usr/lib/systemd/system/buildkit.service << EOF [Unit] Description=BuildKit Requires=buildkit.socket After=buildkit.socket Documentation=https://github.com/moby/buildkit [Service] Type=notify ExecStart=/usr/local/bin/buildkitd --addr fd:// [Install] WantedBy=multi-user.target EOF

[root@k8s-master01 ~]# systemctl daemon-reload && systemctl enable --now buildkit root@k8s-master01:~# systemctl status buildkit ● buildkit.service - BuildKit Loaded: loaded (/lib/systemd/system/buildkit.service; enabled; vendor preset: enabled) Active: active (running) since Tue 2022-09-13 16:47:14 CST; 21s ago TriggeredBy: ● buildkit.socket Docs: https://github.com/moby/buildkit Main PID: 3303 (buildkitd) Tasks: 7 (limit: 4575) Memory: 14.5M CGroup: /system.slice/buildkit.service └─3303 /usr/local/bin/buildkitd --addr fd:// Sep 13 16:47:14 k8s-master01.example.local systemd[1]: Started BuildKit. Sep 13 16:47:14 k8s-master01.example.local buildkitd[3303]: time="2022-09-13T16:47:14+08:00" level=info msg="auto snapshotter: using overlayfs" Sep 13 16:47:14 k8s-master01.example.local buildkitd[3303]: time="2022-09-13T16:47:14+08:00" level=warning msg="using host network as the defa> Sep 13 16:47:14 k8s-master01.example.local buildkitd[3303]: time="2022-09-13T16:47:14+08:00" level=info msg="found worker \"sgqr1t2c81tj7ec7w3> Sep 13 16:47:14 k8s-master01.example.local buildkitd[3303]: time="2022-09-13T16:47:14+08:00" level=warning msg="using host network as the defa> Sep 13 16:47:14 k8s-master01.example.local buildkitd[3303]: time="2022-09-13T16:47:14+08:00" level=info msg="found worker \"w4fzprdjtuqtj3f3wd> Sep 13 16:47:14 k8s-master01.example.local buildkitd[3303]: time="2022-09-13T16:47:14+08:00" level=info msg="found 2 workers, default=\"sgqr1t> Sep 13 16:47:14 k8s-master01.example.local buildkitd[3303]: time="2022-09-13T16:47:14+08:00" level=warning msg="currently, only the default wo> Sep 13 16:47:14 k8s-master01.example.local buildkitd[3303]: time="2022-09-13T16:47:14+08:00" level=info msg="running server on /run/buildkit/b> [root@k8s-master01 ~]# nerdctl version Client: Version: v0.23.0 OS/Arch: linux/amd64 Git commit: 660680b7ddfde1d38a66ec1c7f08f8d89ab92c68 buildctl: Version: v0.10.4 GitCommit: a2ba6869363812a210fcc3ded6926757ab780b5f Server: containerd: Version: v1.6.8 GitCommit: 9cd3357b7fd7218e4aec3eae239db1f68a5a6ec6 runc: Version: 1.1.3 GitCommit: v1.1.3-0-g6724737f [root@k8s-master01 ~]# buildctl --version buildctl github.com/moby/buildkit v0.10.4 a2ba6869363812a210fcc3ded6926757ab780b5f [root@k8s-master01 ~]# nerdctl info Client: Namespace: default Debug Mode: false Server: Server Version: v1.6.8 Storage Driver: overlayfs Logging Driver: json-file Cgroup Driver: cgroupfs Cgroup Version: 1 Plugins: Log: fluentd journald json-file Storage: aufs native overlayfs Security Options: apparmor seccomp Profile: default Kernel Version: 5.4.0-107-generic Operating System: Ubuntu 20.04.4 LTS OSType: linux Architecture: x86_64 CPUs: 2 Total Memory: 3.81GiB Name: k8s-master01.example.local ID: ab901e55-fa37-496e-9920-ee6eff687687 WARNING: No swap limit support #系统警告信息 (没有开启 swap 资源限制 )

#SWAP报警提示，只有在ubuntu系统有，centos系统里没有不用设置 root@k8s-master01:~# sed -ri '/^GRUB_CMDLINE_LINUX=/s@"$@ swapaccount=1"@' /etc/default/grub root@k8s-master01:~# update-grub root@k8s-master01:~# reboot root@k8s-master01:~# nerdctl info Client: Namespace: default Debug Mode: false Server: Server Version: v1.6.8 Storage Driver: overlayfs Logging Driver: json-file Cgroup Driver: cgroupfs Cgroup Version: 1 Plugins: Log: fluentd journald json-file Storage: aufs native overlayfs Security Options: apparmor seccomp Profile: default Kernel Version: 5.4.0-125-generic Operating System: Ubuntu 20.04.4 LTS OSType: linux Architecture: x86_64 CPUs: 2 Total Memory: 3.81GiB Name: k8s-master01.example.local ID: ab901e55-fa37-496e-9920-ee6eff687687 #现在就没有SWAP报警提示

#CentOS [root@k8s-master01 ~]# yum -y install bash-completion #Ubuntu [root@k8s-master01 ~]# apt -y install bash-completion [root@k8s-master01 ~]# echo "source <(nerdctl completion bash)" >> ~/.bashrc

[root@k8s-master02 ~]# cat install_containerd_binary.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2022-09-13 #FileName: install_containerd_binary.sh #Description: install_containerd_binary for centos 7/8 & ubuntu 18.04/20.04 & Rocky 8 #Copyright (C): 2022 All rights reserved #********************************************************************************************* SRC_DIR=/usr/local/src COLOR="echo -e \\033[01;31m" END='\033[0m' #Containerd下载地址：https://github.com/containerd/containerd/releases/download/v1.6.8/cri-containerd-1.6.8-linux-amd64.tar.gz CONTAINERD_FILE=cri-containerd-1.6.8-linux-amd64.tar.gz PAUSE_VERSION=3.8 HARBOR_DOMAIN=harbor.raymonds.cc USERNAME=admin PASSWORD=123456 #Netdctl下载地址：https://github.com/containerd/nerdctl/releases/download/v0.23.0/nerdctl-0.23.0-linux-amd64.tar.gz NETDCTL_FILE=nerdctl-0.23.0-linux-amd64.tar.gz #Buildkit下载地址：https://github.com/moby/buildkit/releases/download/v0.10.4/buildkit-v0.10.4.linux-amd64.tar.gz BUILDKIT_FILE=buildkit-v0.10.4.linux-amd64.tar.gz os(){ OS_ID=`sed -rn '/^NAME=/s@.*="([[:alpha:]]+).*"$@\1@p' /etc/os-release` } check_file (){ cd ${SRC_DIR} if [ ! -e ${CONTAINERD_FILE} ];then ${COLOR}"缺少${CONTAINERD_FILE}文件,请把文件放到${SRC_DIR}目录下"${END} exit elif [ ! -e ${NETDCTL_FILE} ];then ${COLOR}"缺少${NETDCTL_FILE}文件,请把文件放到${SRC_DIR}目录下"${END} exit elif [ ! -e ${BUILDKIT_FILE} ];then ${COLOR}"缺少${BUILDKIT_FILE}文件,请把文件放到${SRC_DIR}目录下"${END} exit else ${COLOR}"相关文件已准备好"${END} fi } install_containerd(){ [ -f /usr/local/bin/containerd ] && { ${COLOR}"Containerd已存在，安装失败"${END};exit; } cat > /etc/modules-load.d/containerd.conf <<-EOF overlay br_netfilter EOF modprobe -- overlay modprobe -- br_netfilter cat > /etc/sysctl.d/99-kubernetes-cri.conf <<-EOF net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 EOF sysctl --system &> /dev/null ${COLOR}"开始安装Containerd..."${END} tar xf ${CONTAINERD_FILE} -C / mkdir -p /etc/containerd containerd config default | tee /etc/containerd/config.toml &> /dev/null sed -ri -e 's/(.*SystemdCgroup = ).*/\1true/' -e 's@(.*sandbox_image = ).*@\1\"'''${HARBOR_DOMAIN}'''/google_containers/pause:'''${PAUSE_VERSION}'''\"@' -e '/.*registry.mirrors.*/a\ [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]\n endpoint = ["https://registry.docker-cn.com" ,"http://hub-mirror.c.163.com" ,"https://docker.mirrors.ustc.edu.cn"]\n [plugins."io.containerd.grpc.v1.cri".registry.mirrors."'''${HARBOR_DOMAIN}'''"]\n endpoint = ["http://'''${HARBOR_DOMAIN}'''"]' -e '/.*registry.configs.*/a\ [plugins."io.containerd.grpc.v1.cri".registry.configs."'''${HARBOR_DOMAIN}'''".tls]\n insecure_skip_verify = true\n [plugins."io.containerd.grpc.v1.cri".registry.configs."'''${HARBOR_DOMAIN}'''".auth]\n username = "'''${USERNAME}'''"\n password = "'''${PASSWORD}'''"' /etc/containerd/config.toml cat > /etc/crictl.yaml <<-EOF runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock timeout: 10 debug: false EOF systemctl daemon-reload && systemctl enable --now containerd &> /dev/null systemctl is-active containerd &> /dev/null && ${COLOR}"Containerd 服务启动成功"${END} || { ${COLOR}"Containerd 启动失败"${END};exit; } ctr version && ${COLOR}"Containerd 安装成功"${END} || ${COLOR}"Containerd 安装失败"${END} } set_alias(){ echo 'alias rmi="nerdctl images -qa|xargs nerdctl rmi -f"' >> ~/.bashrc echo 'alias rmc="nerdctl ps -qa|xargs nerdctl rm -f"' >> ~/.bashrc } install_netdctl_buildkit(){ ${COLOR}"开始安装Netdctl..."${END} tar xf ${NETDCTL_FILE} -C /usr/local/bin/ mkdir -p /etc/nerdctl/ cat > /etc/nerdctl/nerdctl.toml <<-EOF namespace = "k8s.io" insecure_registry = true EOF ${COLOR}"开始安装Buildkit..."${END} tar xf ${BUILDKIT_FILE} -C /usr/local/ cat > /usr/lib/systemd/system/buildkit.socket <<-EOF [Unit] Description=BuildKit Documentation=https://github.com/moby/buildkit [Socket] ListenStream=%t/buildkit/buildkitd.sock SocketMode=0660 [Install] WantedBy=sockets.target EOF cat > /usr/lib/systemd/system/buildkit.service <<-EOF [Unit] Description=BuildKit Requires=buildkit.socket After=buildkit.socket Documentation=https://github.com/moby/buildkit [Service] Type=notify ExecStart=/usr/local/bin/buildkitd --addr fd:// [Install] WantedBy=multi-user.target EOF systemctl daemon-reload && systemctl enable --now buildkit &> /dev/null systemctl is-active buildkit &> /dev/null && ${COLOR}"Buildkit 服务启动成功"${END} || { ${COLOR}"Buildkit 启动失败"${END};exit; } buildctl --version && ${COLOR}"Buildkit 安装成功"${END} || ${COLOR}"Buildkit 安装失败"${END} } nerdctl_command_completion(){ if [ ${OS_ID} == "CentOS" -o ${OS_ID} == "Rocky" ];then yum -y install bash-completion else apt -y install bash-completion fi echo "source <(nerdctl completion bash)" >> ~/.bashrc . ~/.bashrc } set_swap_limit(){ if [ ${OS_ID} == "Ubuntu" ];then ${COLOR}'设置Docker的"WARNING: No swap limit support"警告'${END} sed -ri '/^GRUB_CMDLINE_LINUX=/s@"$@ swapaccount=1"@' /etc/default/grub update-grub &> /dev/null ${COLOR}"10秒后，机器会自动重启"${END} sleep 10 reboot fi } main(){ os check_file install_containerd set_alias install_netdctl_buildkit nerdctl_command_completion set_swap_limit } main [root@k8s-master02 ~]# bash install_containerd_binary.sh [root@k8s-master03 ~]# bash install_containerd_binary.sh [root@k8s-node01 ~]# bash install_containerd_binary.sh [root@k8s-node02 ~]# bash install_containerd_binary.sh [root@k8s-node03 ~]# bash install_containerd_binary.sh

[root@k8s-master01 ~]# mkdir /etc/etcd/ssl -p [root@k8s-master02 ~]# mkdir /etc/etcd/ssl -p [root@k8s-master03 ~]# mkdir /etc/etcd/ssl -p

[root@k8s-etcd01 pki]# for NODE in k8s-master01 k8s-master02 k8s-master03; do ssh -o StrictHostKeyChecking=no $NODE "mkdir -p /etc/etcd/ssl" for FILE in etcd-ca-key.pem etcd-ca.pem etcd-key.pem etcd.pem; do scp -o StrictHostKeyChecking=no /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE} done done

[root@k8s-master01 ~]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-master02 ~]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-master03 ~]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-master01 ~]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/ [root@k8s-master02 ~]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/ [root@k8s-master03 ~]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/

[root@k8s-master01 ~]# wget https://dl.k8s.io/v1.25.2/kubernetes-server-linux-amd64.tar.gz

[root@k8s-master01 ~]# tar -xf kubernetes-server-linux-amd64.tar.gz --strip-compnotallow=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}

[root@k8s-master01 ~]# kubelet --version Kubernetes v1.25.2

[root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03; do echo $NODE; scp -o StrictHostKeyChecking=no /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/; done

[root@k8s-master01 ~]# mkdir -p /opt/cni/bin [root@k8s-master02 ~]# mkdir -p /opt/cni/bin [root@k8s-master03 ~]# mkdir -p /opt/cni/bin

[root@k8s-master01 ~]# mkdir -p /etc/kubernetes/pki [root@k8s-master02 ~]# mkdir -p /etc/kubernetes/pki [root@k8s-master03 ~]# mkdir -p /etc/kubernetes/pki

[root@k8s-master01 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssl_1.6.1_linux_amd64 [root@k8s-master01 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssljson_1.6.1_linux_amd64 [root@k8s-master01 ~]# mv cfssl_1.6.1_linux_amd64 /usr/local/bin/cfssl [root@k8s-master01 ~]# mv cfssljson_1.6.1_linux_amd64 /usr/local/bin/cfssljson [root@k8s-master01 ~]# chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

[root@k8s-master01 ~]# mkdir pki [root@k8s-master01 ~]# cd pki/ [root@k8s-master01 pki]# cat > ca-csr.json <<EOF { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "Kubernetes", "OU": "Kubernetes-manual" } ], "ca": { "expiry": "876000h" } } EOF [root@k8s-master01 pki]# cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca #执行结果 2022/09/25 16:39:54 [INFO] generating a new CA key and certificate from CSR 2022/09/25 16:39:54 [INFO] generate received request 2022/09/25 16:39:54 [INFO] received CSR 2022/09/25 16:39:54 [INFO] generating key: rsa-2048 2022/09/25 16:39:54 [INFO] encoded CSR 2022/09/25 16:39:54 [INFO] signed certificate with serial number 144467388419643602118985040994832054816132660567 [root@k8s-master01 pki]# ll /etc/kubernetes/pki/ca* -rw-r--r-- 1 root root 1070 Sep 25 16:39 /etc/kubernetes/pki/ca.csr -rw------- 1 root root 1679 Sep 25 16:39 /etc/kubernetes/pki/ca-key.pem -rw-r--r-- 1 root root 1363 Sep 25 16:39 /etc/kubernetes/pki/ca.pem

[root@k8s-master01 pki]# cat > ca-config.json <<EOF { "signing": { "default": { "expiry": "876000h" }, "profiles": { "kubernetes": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "876000h" } } } } EOF [root@k8s-master01 pki]# cat > apiserver-csr.json <<EOF { "CN": "kube-apiserver", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "Kubernetes", "OU": "Kubernetes-manual" } ] } EOF [root@k8s-master01 pki]# cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -hostname=10.96.0.1,172.31.3.188,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,172.31.3.101,172.31.3.102,172.31.3.103 -profile=kubernetes apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver #执行结果 2022/09/25 16:40:41 [INFO] generate received request 2022/09/25 16:40:41 [INFO] received CSR 2022/09/25 16:40:41 [INFO] generating key: rsa-2048 2022/09/25 16:40:41 [INFO] encoded CSR 2022/09/25 16:40:41 [INFO] signed certificate with serial number 430981208956747841835012330139928202432283184832 [root@k8s-master01 pki]# ll /etc/kubernetes/pki/apiserver* -rw-r--r-- 1 root root 1297 Sep 25 16:40 /etc/kubernetes/pki/apiserver.csr -rw------- 1 root root 1675 Sep 25 16:40 /etc/kubernetes/pki/apiserver-key.pem -rw-r--r-- 1 root root 1692 Sep 25 16:40 /etc/kubernetes/pki/apiserver.pem

[root@k8s-master01 pki]# cat > front-proxy-ca-csr.json <<EOF { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 } } EOF [root@k8s-master01 pki]# cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca #执行结果 2022/09/25 16:41:21 [INFO] generating a new CA key and certificate from CSR 2022/09/25 16:41:21 [INFO] generate received request 2022/09/25 16:41:21 [INFO] received CSR 2022/09/25 16:41:21 [INFO] generating key: rsa-2048 2022/09/25 16:41:22 [INFO] encoded CSR 2022/09/25 16:41:22 [INFO] signed certificate with serial number 673709205875970548123940386290897802924816968201 [root@k8s-master01 pki]# ll /etc/kubernetes/pki/front-proxy-ca* -rw-r--r-- 1 root root 891 Sep 25 16:41 /etc/kubernetes/pki/front-proxy-ca.csr -rw------- 1 root root 1679 Sep 25 16:41 /etc/kubernetes/pki/front-proxy-ca-key.pem -rw-r--r-- 1 root root 1094 Sep 25 16:41 /etc/kubernetes/pki/front-proxy-ca.pem [root@k8s-master01 pki]# cat > front-proxy-client-csr.json <<EOF { "CN": "front-proxy-client", "key": { "algo": "rsa", "size": 2048 } } EOF [root@k8s-master01 pki]# cfssl gencert -ca=/etc/kubernetes/pki/front-proxy-ca.pem -ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem -config=ca-config.json -profile=kubernetes front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client #返回结果（忽略警告） 2022/09/25 16:42:01 [INFO] generate received request 2022/09/25 16:42:01 [INFO] received CSR 2022/09/25 16:42:01 [INFO] generating key: rsa-2048 2022/09/25 16:42:01 [INFO] encoded CSR 2022/09/25 16:42:01 [INFO] signed certificate with serial number 85560094871910737940366819468847340063056850525 2022/09/25 16:42:01 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements"). [root@k8s-master01 pki]# ll /etc/kubernetes/pki/front-proxy-client* -rw-r--r-- 1 root root 903 Sep 25 16:42 /etc/kubernetes/pki/front-proxy-client.csr -rw------- 1 root root 1675 Sep 25 16:42 /etc/kubernetes/pki/front-proxy-client-key.pem -rw-r--r-- 1 root root 1188 Sep 25 16:42 /etc/kubernetes/pki/front-proxy-client.pem

[root@k8s-master01 pki]# cat > manager-csr.json <<EOF { "CN": "system:kube-controller-manager", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "system:kube-controller-manager", "OU": "Kubernetes-manual" } ] } EOF [root@k8s-master01 pki]# cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -profile=kubernetes manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager #执行结果 2022/09/25 16:42:37 [INFO] generate received request 2022/09/25 16:42:37 [INFO] received CSR 2022/09/25 16:42:37 [INFO] generating key: rsa-2048 2022/09/25 16:42:37 [INFO] encoded CSR 2022/09/25 16:42:37 [INFO] signed certificate with serial number 730142395596434023542887289901373515374508177240 2022/09/25 16:42:37 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements"). [root@k8s-master01 pki]# ll /etc/kubernetes/pki/controller-manager* -rw-r--r-- 1 root root 1082 Sep 25 16:42 /etc/kubernetes/pki/controller-manager.csr -rw------- 1 root root 1675 Sep 25 16:42 /etc/kubernetes/pki/controller-manager-key.pem -rw-r--r-- 1 root root 1501 Sep 25 16:42 /etc/kubernetes/pki/controller-manager.pem # 注意，如果不是高可用集群，172.31.3.188:6443改为master01的地址 # set-cluster：设置一个集群项， [root@k8s-master01 pki]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://172.31.3.188:6443 --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig #执行结果 Cluster "kubernetes" set. # set-credentials 设置一个用户项 [root@k8s-master01 pki]# kubectl config set-credentials system:kube-controller-manager --client-certificate=/etc/kubernetes/pki/controller-manager.pem --client-key=/etc/kubernetes/pki/controller-manager-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig #执行结果 User "system:kube-controller-manager" set. # 设置一个环境项，一个上下文 [root@k8s-master01 pki]# kubectl config set-context system:kube-controller-manager@kubernetes --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig #执行结果 Context "system:kube-controller-manager@kubernetes" created. # 使用某个环境当做默认环境 [root@k8s-master01 pki]# kubectl config use-context system:kube-controller-manager@kubernetes --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig #执行结果 Switched to context "system:kube-controller-manager@kubernetes".

[root@k8s-master01 pki]# cat > scheduler-csr.json <<EOF { "CN": "system:kube-scheduler", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "system:kube-scheduler", "OU": "Kubernetes-manual" } ] } EOF [root@k8s-master01 pki]# cfssl gencert \ -ca=/etc/kubernetes/pki/ca.pem \ -ca-key=/etc/kubernetes/pki/ca-key.pem \ -config=ca-config.json \ -profile=kubernetes \ scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler #执行结果 2022/09/25 16:48:53 [INFO] generate received request 2022/09/25 16:48:53 [INFO] received CSR 2022/09/25 16:48:53 [INFO] generating key: rsa-2048 2022/09/25 16:48:53 [INFO] encoded CSR 2022/09/25 16:48:53 [INFO] signed certificate with serial number 228881838573494475296244558577950441626719203074 2022/09/25 16:48:53 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements"). [root@k8s-master01 pki]# ll /etc/kubernetes/pki/scheduler* -rw-r--r-- 1 root root 1058 Sep 25 16:48 /etc/kubernetes/pki/scheduler.csr -rw------- 1 root root 1679 Sep 25 16:48 /etc/kubernetes/pki/scheduler-key.pem -rw-r--r-- 1 root root 1476 Sep 25 16:48 /etc/kubernetes/pki/scheduler.pem # 注意，如果不是高可用集群，172.31.3.188:6443改为master01的地址 [root@k8s-master01 pki]# kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/pki/ca.pem \ --embed-certs=true \ --server=https://172.31.3.188:6443 \ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig #执行结果 Cluster "kubernetes" set. [root@k8s-master01 pki]# kubectl config set-credentials system:kube-scheduler \ --client-certificate=/etc/kubernetes/pki/scheduler.pem \ --client-key=/etc/kubernetes/pki/scheduler-key.pem \ --embed-certs=true \ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig #执行结果 User "system:kube-scheduler" set. [root@k8s-master01 pki]# kubectl config set-context system:kube-scheduler@kubernetes \ --cluster=kubernetes \ --user=system:kube-scheduler \ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig #执行结果 Context "system:kube-scheduler@kubernetes" created. [root@k8s-master01 pki]# kubectl config use-context system:kube-scheduler@kubernetes \ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig #执行结果 Switched to context "system:kube-scheduler@kubernetes".

[root@k8s-master01 pki]# cat > admin-csr.json <<EOF { "CN": "admin", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "system:masters", "OU": "Kubernetes-manual" } ] } EOF [root@k8s-master01 pki]# cfssl gencert \ -ca=/etc/kubernetes/pki/ca.pem \ -ca-key=/etc/kubernetes/pki/ca-key.pem \ -config=ca-config.json \ -profile=kubernetes \ admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin #执行结果 2022/09/25 16:50:04 [INFO] generate received request 2022/09/25 16:50:04 [INFO] received CSR 2022/09/25 16:50:04 [INFO] generating key: rsa-2048 2022/09/25 16:50:04 [INFO] encoded CSR 2022/09/25 16:50:04 [INFO] signed certificate with serial number 42182691318354263130529298881094841114488241757 2022/09/25 16:50:04 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements"). [root@k8s-master01 pki]# ll /etc/kubernetes/pki/admin* -rw-r--r-- 1 root root 1025 Sep 25 16:50 /etc/kubernetes/pki/admin.csr -rw------- 1 root root 1675 Sep 25 16:50 /etc/kubernetes/pki/admin-key.pem -rw-r--r-- 1 root root 1444 Sep 25 16:50 /etc/kubernetes/pki/admin.pem # 注意，如果不是高可用集群，172.31.3.188:8443改为master01的地址 [root@k8s-master01 pki]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://172.31.3.188:6443 --kubeconfig=/etc/kubernetes/admin.kubeconfig #执行结果 Cluster "kubernetes" set. [root@k8s-master01 pki]# kubectl config set-credentials kubernetes-admin --client-certificate=/etc/kubernetes/pki/admin.pem --client-key=/etc/kubernetes/pki/admin-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/admin.kubeconfig #执行结果 User "kubernetes-admin" set. [root@k8s-master01 pki]# kubectl config set-context kubernetes-admin@kubernetes --cluster=kubernetes --user=kubernetes-admin --kubeconfig=/etc/kubernetes/admin.kubeconfig #执行结果 Context "kubernetes-admin@kubernetes" created. [root@k8s-master01 pki]# kubectl config use-context kubernetes-admin@kubernetes --kubeconfig=/etc/kubernetes/admin.kubeconfig #执行结果 Switched to context "kubernetes-admin@kubernetes".

[root@k8s-master01 pki]# openssl genrsa -out /etc/kubernetes/pki/sa.key 2048 #返回结果 Generating RSA private key, 2048 bit long modulus ..............................+++ ....................................+++ e is 65537 (0x10001) [root@k8s-master01 pki]# openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub #执行结果 writing RSA key [root@k8s-master01 pki]# ll /etc/kubernetes/pki/sa* -rw------- 1 root root 1679 Sep 25 16:51 /etc/kubernetes/pki/sa.key -rw-r--r-- 1 root root 451 Sep 25 16:51 /etc/kubernetes/pki/sa.pub

[root@k8s-master01 pki]# for NODE in k8s-master02 k8s-master03; do for FILE in $(ls /etc/kubernetes/pki | grep -v etcd); do scp /etc/kubernetes/pki/${FILE} $NODE:/etc/kubernetes/pki/${FILE}; done; for FILE in admin.kubeconfig controller-manager.kubeconfig scheduler.kubeconfig; do scp /etc/kubernetes/${FILE} $NODE:/etc/kubernetes/${FILE}; done; done

[root@k8s-master01 pki]# ll -R /etc/kubernetes/pki/ /etc/kubernetes/pki/: total 92 -rw-r--r-- 1 root root 1025 Sep 25 16:50 admin.csr -rw------- 1 root root 1675 Sep 25 16:50 admin-key.pem -rw-r--r-- 1 root root 1444 Sep 25 16:50 admin.pem -rw-r--r-- 1 root root 1297 Sep 25 16:40 apiserver.csr -rw------- 1 root root 1675 Sep 25 16:40 apiserver-key.pem -rw-r--r-- 1 root root 1692 Sep 25 16:40 apiserver.pem -rw-r--r-- 1 root root 1070 Sep 25 16:39 ca.csr -rw------- 1 root root 1679 Sep 25 16:39 ca-key.pem -rw-r--r-- 1 root root 1363 Sep 25 16:39 ca.pem -rw-r--r-- 1 root root 1082 Sep 25 16:42 controller-manager.csr -rw------- 1 root root 1675 Sep 25 16:42 controller-manager-key.pem -rw-r--r-- 1 root root 1501 Sep 25 16:42 controller-manager.pem drwxr-xr-x 2 root root 84 Sep 25 16:37 etcd -rw-r--r-- 1 root root 891 Sep 25 16:41 front-proxy-ca.csr -rw------- 1 root root 1679 Sep 25 16:41 front-proxy-ca-key.pem -rw-r--r-- 1 root root 1094 Sep 25 16:41 front-proxy-ca.pem -rw-r--r-- 1 root root 903 Sep 25 16:42 front-proxy-client.csr -rw------- 1 root root 1675 Sep 25 16:42 front-proxy-client-key.pem -rw-r--r-- 1 root root 1188 Sep 25 16:42 front-proxy-client.pem -rw------- 1 root root 1679 Sep 25 16:51 sa.key -rw-r--r-- 1 root root 451 Sep 25 16:51 sa.pub -rw-r--r-- 1 root root 1058 Sep 25 16:48 scheduler.csr -rw------- 1 root root 1679 Sep 25 16:48 scheduler-key.pem -rw-r--r-- 1 root root 1476 Sep 25 16:48 scheduler.pem /etc/kubernetes/pki/etcd: total 0 lrwxrwxrwx 1 root root 29 Sep 25 16:37 etcd-ca-key.pem -> /etc/etcd/ssl/etcd-ca-key.pem lrwxrwxrwx 1 root root 25 Sep 25 16:37 etcd-ca.pem -> /etc/etcd/ssl/etcd-ca.pem lrwxrwxrwx 1 root root 26 Sep 25 16:37 etcd-key.pem -> /etc/etcd/ssl/etcd-key.pem lrwxrwxrwx 1 root root 22 Sep 25 16:37 etcd.pem -> /etc/etcd/ssl/etcd.pe [root@k8s-master01 pki]# ls /etc/kubernetes/pki | grep -v etcd |wc -l 23 # 一共23个就对了

[root@k8s-master01 pki]# cd [root@k8s-master01 ~]# mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes [root@k8s-master02 ~]# mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes [root@k8s-master03 ~]# mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes

[root@k8s-master01 ~]# cat > /lib/systemd/system/kube-apiserver.service <<EOF [Unit] Description=Kubernetes API Server Documentation=https://github.com/kubernetes/kubernetes After=network.target [Service] ExecStart=/usr/local/bin/kube-apiserver \\ --v=2 \\ --logtostderr=true \\ --allow-privileged=true \\ --bind-address=0.0.0.0 \\ --secure-port=6443 \\ --advertise-address=172.31.3.101 \\ --service-cluster-ip-range=10.96.0.0/12 \\ --service-node-port-range=30000-32767 \\ --etcd-servers=https://172.31.3.108:2379,https://172.31.3.109:2379,https://172.31.3.110:2379 \\ --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\ --etcd-certfile=/etc/etcd/ssl/etcd.pem \\ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\ --client-ca-file=/etc/kubernetes/pki/ca.pem \\ --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\ --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\ --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\ --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\ --service-account-key-file=/etc/kubernetes/pki/sa.pub \\ --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\ --service-account-issuer=https://kubernetes.default.svc.cluster.local \\ --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \\ --authorization-mode=Node,RBAC \\ --enable-bootstrap-token-auth=true \\ --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\ --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\ --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\ --requestheader-allowed-names=aggregator \\ --requestheader-group-headers=X-Remote-Group \ --requestheader-extra-headers-prefix=X-Remote-Extra- \\ --requestheader-username-headers=X-Remote-User # --token-auth-file=/etc/kubernetes/token.csv Restart=on-failure RestartSec=10s LimitNOFILE=65535 [Install] WantedBy=multi-user.target EOF

[root@k8s-master02 ~]# cat > /lib/systemd/system/kube-apiserver.service <<EOF [Unit] Description=Kubernetes API Server Documentation=https://github.com/kubernetes/kubernetes After=network.target [Service] ExecStart=/usr/local/bin/kube-apiserver \\ --v=2 \\ --logtostderr=true \\ --allow-privileged=true \\ --bind-address=0.0.0.0 \\ --secure-port=6443 \\ --advertise-address=172.31.3.102 \\ --service-cluster-ip-range=10.96.0.0/12 \\ --service-node-port-range=30000-32767 \\ --etcd-servers=https://172.31.3.108:2379,https://172.31.3.109:2379,https://172.31.3.110:2379 \\ --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\ --etcd-certfile=/etc/etcd/ssl/etcd.pem \\ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\ --client-ca-file=/etc/kubernetes/pki/ca.pem \\ --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\ --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\ --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\ --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\ --service-account-key-file=/etc/kubernetes/pki/sa.pub \\ --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\ --service-account-issuer=https://kubernetes.default.svc.cluster.local \\ --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \\ --authorization-mode=Node,RBAC \\ --enable-bootstrap-token-auth=true \\ --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\ --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\ --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\ --requestheader-allowed-names=aggregator \\ --requestheader-group-headers=X-Remote-Group \\ --requestheader-extra-headers-prefix=X-Remote-Extra- \\ --requestheader-username-headers=X-Remote-User # --token-auth-file=/etc/kubernetes/token.csv Restart=on-failure RestartSec=10s LimitNOFILE=65535 [Install] WantedBy=multi-user.target EOF

[root@k8s-master03 ~]# cat > /lib/systemd/system/kube-apiserver.service <<EOF [Unit] Description=Kubernetes API Server Documentation=https://github.com/kubernetes/kubernetes After=network.target [Service] ExecStart=/usr/local/bin/kube-apiserver \\ --v=2 \\ --logtostderr=true \\ --allow-privileged=true \\ --bind-address=0.0.0.0 \\ --secure-port=6443 \\ --advertise-address=172.31.3.103 \\ --service-cluster-ip-range=10.96.0.0/12 \\ --service-node-port-range=30000-32767 \\ --etcd-servers=https://172.31.3.108:2379,https://172.31.3.109:2379,https://172.31.3.110:2379 \\ --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\ --etcd-certfile=/etc/etcd/ssl/etcd.pem \\ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\ --client-ca-file=/etc/kubernetes/pki/ca.pem \\ --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\ --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\ --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\ --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\ --service-account-key-file=/etc/kubernetes/pki/sa.pub \\ --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\ --service-account-issuer=https://kubernetes.default.svc.cluster.local \\ --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \\ --authorization-mode=Node,RBAC \\ --enable-bootstrap-token-auth=true \\ --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\ --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\ --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\ --requestheader-allowed-names=aggregator \\ --requestheader-group-headers=X-Remote-Group \\ --requestheader-extra-headers-prefix=X-Remote-Extra- \\ --requestheader-username-headers=X-Remote-User # --token-auth-file=/etc/kubernetes/token.csv Restart=on-failure RestartSec=10s LimitNOFILE=65535 [Install] WantedBy=multi-user.target EOF

[root@k8s-master01 ~]# systemctl daemon-reload && systemctl enable --now kube-apiserver [root@k8s-master02 ~]# systemctl daemon-reload && systemctl enable --now kube-apiserver [root@k8s-master03 ~]# systemctl daemon-reload && systemctl enable --now kube-apiserver

[root@k8s-master01 ~]# systemctl status kube-apiserver ● kube-apiserver.service - Kubernetes API Server Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled) Active: active (running) since Sun 2022-09-25 16:53:11 CST; 12s ago Docs: https://github.com/kubernetes/kubernetes Main PID: 6154 (kube-apiserver) Tasks: 14 (limit: 23474) Memory: 259.5M CGroup: /system.slice/kube-apiserver.service └─6154 /usr/local/bin/kube-apiserver --v=2 --logtostderr=true --allow-privileged=true --bind-address=0.0.0.0 --secure-port=6443 --a> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: I0925 16:53:19.657786 6154 storage_rbac.go:321] created rolebinding.rbac.a> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: I0925 16:53:19.672982 6154 storage_rbac.go:321] created rolebinding.rbac.a> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: I0925 16:53:19.684131 6154 healthz.go:257] poststarthook/rbac/bootstrap-ro> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: [-]poststarthook/rbac/bootstrap-roles failed: not finished Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: I0925 16:53:19.689793 6154 storage_rbac.go:321] created rolebinding.rbac.a> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: I0925 16:53:19.705317 6154 storage_rbac.go:321] created rolebinding.rbac.a> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: I0925 16:53:19.817048 6154 alloc.go:327] "allocated clusterIPs" service="d> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: W0925 16:53:19.839303 6154 lease.go:250] Resetting endpoints for master se> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: I0925 16:53:19.840909 6154 controller.go:616] quota admission added evalua> Sep 25 16:53:19 k8s-master01.example.local kube-apiserver[6154]: I0925 16:53:19.878881 6154 controller.go:616] quota admission added evalua> [root@k8s-master02 ~]# systemctl status kube-apiserver [root@k8s-master03 ~]# systemctl status kube-apiserver

[root@k8s-master01 ~]# cat > /lib/systemd/system/kube-controller-manager.service <<EOF [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/kubernetes/kubernetes After=network.target [Service] ExecStart=/usr/local/bin/kube-controller-manager \\ --v=2 \\ --logtostderr=true \\ --bind-address=127.0.0.1 \\ --root-ca-file=/etc/kubernetes/pki/ca.pem \\ --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \\ --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\ --service-account-private-key-file=/etc/kubernetes/pki/sa.key \\ --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \\ --leader-elect=true \\ --use-service-account-credentials=true \\ --node-monitor-grace-period=40s \\ --node-monitor-period=5s \\ --pod-eviction-timeout=2m0s \\ --controllers=*,bootstrapsigner,tokencleaner \\ --allocate-node-cidrs=true \\ --cluster-cidr=192.168.0.0/12 \\ --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\ --node-cidr-mask-size=24 Restart=always RestartSec=10s [Install] WantedBy=multi-user.target EOF [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03; do scp /lib/systemd/system/kube-controller-manager.service $NODE:/lib/systemd/system/; done

[root@k8s-master01 ~]# systemctl daemon-reload && systemctl enable --now kube-controller-manager [root@k8s-master02 ~]# systemctl daemon-reload && systemctl enable --now kube-controller-manager [root@k8s-master03 ~]# systemctl daemon-reload && systemctl enable --now kube-controller-manager

[root@k8s-master01 ~]# systemctl status kube-controller-manager ● kube-controller-manager.service - Kubernetes Controller Manager Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled) Active: active (running) since Sun 2022-09-25 16:54:21 CST; 16s ago Docs: https://github.com/kubernetes/kubernetes Main PID: 6212 (kube-controller) Tasks: 11 (limit: 23474) Memory: 35.3M CGroup: /system.slice/kube-controller-manager.service └─6212 /usr/local/bin/kube-controller-manager --v=2 --logtostderr=true --bind-address=127.0.0.1 --root-ca-file=/etc/kubernetes/pki/> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.274767 6212 shared_informer.go:262] Caches are syn> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.274835 6212 taint_manager.go:204] "Starting NoExec> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.274896 6212 taint_manager.go:209] "Sending events > Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.302454 6212 shared_informer.go:262] Caches are syn> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.303795 6212 shared_informer.go:262] Caches are syn> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.303859 6212 resource_quota_controller.go:462] sync> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.643168 6212 shared_informer.go:262] Caches are syn> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.643226 6212 garbagecollector.go:263] synced garbag> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.680675 6212 shared_informer.go:262] Caches are syn> Sep 25 16:54:35 k8s-master01.example.local kube-controller-manager[6212]: I0925 16:54:35.680732 6212 garbagecollector.go:163] Garbage colle> [root@k8s-master02 ~]# systemctl status kube-controller-manager [root@k8s-master03 ~]# systemctl status kube-controller-manager

[root@k8s-master01 ~]# cat > /lib/systemd/system/kube-scheduler.service <<EOF [Unit] Description=Kubernetes Scheduler Documentation=https://github.com/kubernetes/kubernetes After=network.target [Service] ExecStart=/usr/local/bin/kube-scheduler \\ --v=2 \\ --logtostderr=true \\ --bind-address=127.0.0.1 \\ --leader-elect=true \\ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig Restart=always RestartSec=10s [Install] WantedBy=multi-user.target EOF [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03; do scp /lib/systemd/system/kube-scheduler.service $NODE:/lib/systemd/system/; done [root@k8s-master01 ~]# systemctl daemon-reload && systemctl enable --now kube-scheduler [root@k8s-master02 ~]# systemctl daemon-reload && systemctl enable --now kube-scheduler [root@k8s-master03 ~]# systemctl daemon-reload && systemctl enable --now kube-scheduler [root@k8s-master01 ~]# systemctl status kube-scheduler ● kube-scheduler.service - Kubernetes Scheduler Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled) Active: active (running) since Sun 2022-09-25 16:55:26 CST; 14s ago Docs: https://github.com/kubernetes/kubernetes Main PID: 6268 (kube-scheduler) Tasks: 7 (limit: 23474) Memory: 13.9M CGroup: /system.slice/kube-scheduler.service └─6268 /usr/local/bin/kube-scheduler --v=2 --logtostderr=true --bind-address=127.0.0.1 --leader-elect=true --kubeconfig=/etc/kubern> Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: schedulerName: default-scheduler Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: > Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: I0925 16:55:27.648331 6268 server.go:148] "Starting Kubernetes Scheduler" > Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: I0925 16:55:27.648349 6268 server.go:150] "Golang settings" GOGC="" GOMAXP> Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: I0925 16:55:27.650630 6268 tlsconfig.go:200] "Loaded serving cert" certNam> Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: I0925 16:55:27.650785 6268 named_certificates.go:53] "Loaded SNI cert" ind> Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: I0925 16:55:27.650811 6268 secure_serving.go:210] Serving securely on 127.> Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: I0925 16:55:27.651921 6268 tlsconfig.go:240] "Starting DynamicServingCerti> Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: I0925 16:55:27.752899 6268 leaderelection.go:248] attempting to acquire le> Sep 25 16:55:27 k8s-master01.example.local kube-scheduler[6268]: I0925 16:55:27.771438 6268 leaderelection.go:258] successfully acquired le> [root@k8s-master02 ~]# systemctl status kube-scheduler [root@k8s-master03 ~]# systemctl status kube-scheduler

[root@k8s-master01 ~]# cat > bootstrap.secret.yaml <<EOF apiVersion: v1 kind: Secret metadata: name: bootstrap-token-c8ad9c namespace: kube-system type: bootstrap.kubernetes.io/token stringData: description: "The default bootstrap token generated by 'kubelet '." token-id: c8ad9c token-secret: 2e4d610cf3e7426e usage-bootstrap-authentication: "true" usage-bootstrap-signing: "true" auth-extra-groups: system:bootstrappers:default-node-token,system:bootstrappers:worker,system:bootstrappers:ingress --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: kubelet-bootstrap roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:node-bootstrapper subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:default-node-token --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: node-autoapprove-bootstrap roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:certificates.k8s.io:certificatesigningrequests:nodeclient subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:default-node-token --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: node-autoapprove-certificate-rotation roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:nodes --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" labels: kubernetes.io/bootstrapping: rbac-defaults name: system:kube-apiserver-to-kubelet rules: - apiGroups: - "" resources: - nodes/proxy - nodes/stats - nodes/log - nodes/spec - nodes/metrics verbs: - "*" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:kube-apiserver namespace: "" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:kube-apiserver-to-kubelet subjects: - apiGroup: rbac.authorization.k8s.io kind: User name: kube-apiserver EOF # 注意，如果不是高可用集群，172.31.3.188:6443改为master01的地址 [root@k8s-master01 ~]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://172.31.3.188:6443 --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig #执行结果 Cluster "kubernetes" set. [root@k8s-master01 ~]# kubectl config set-credentials tls-bootstrap-token-user --token=c8ad9c.2e4d610cf3e7426e --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig #执行结果 User "tls-bootstrap-token-user" set. [root@k8s-master01 ~]# kubectl config set-context tls-bootstrap-token-user@kubernetes --cluster=kubernetes --user=tls-bootstrap-token-user --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig #执行结果 Context "tls-bootstrap-token-user@kubernetes" modified. [root@k8s-master01 ~]# kubectl config use-context tls-bootstrap-token-user@kubernetes --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig #执行结果 Switched to context "tls-bootstrap-token-user@kubernetes".

[root@k8s-master01 ~]# mkdir -p /root/.kube ; cp /etc/kubernetes/admin.kubeconfig /root/.kube/config #查看集群状态，没问题的话继续后续操作 [root@k8s-master01 ~]# kubectl get cs Warning: v1 ComponentStatus is deprecated in v1.19+ NAME STATUS MESSAGE ERROR scheduler Healthy ok controller-manager Healthy ok etcd-1 Healthy {"health":"true","reason":""} etcd-2 Healthy {"health":"true","reason":""} etcd-0 Healthy {"health":"true","reason":""} [root@k8s-master01 ~]# kubectl create -f bootstrap.secret.yaml [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03; do for FILE in bootstrap-kubelet.kubeconfig; do scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE} done done

[root@k8s-master01 ~]# cat > /lib/systemd/system/kubelet.service <<EOF [Unit] Description=Kubernetes Kubelet Documentation=https://github.com/kubernetes/kubernetes After=containerd.service Requires=containerd.service [Service] ExecStart=/usr/local/bin/kubelet Restart=always StartLimitInterval=0 RestartSec=10 [Install] WantedBy=multi-user.target EOF [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03; do scp /lib/systemd/system/kubelet.service $NODE:/lib/systemd/system/ ;done

[root@k8s-master01 ~]# cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf <<EOF [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig" Environment="KUBELET_SYSTEM_ARGS=--hostname-override=172.31.3.101" Environment="KUBELET_RINTIME=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml" Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node=''" ExecStart= ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS \$KUBELET_RINTIME EOF [root@k8s-master02 ~]# cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf <<EOF [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig" Environment="KUBELET_SYSTEM_ARGS=--hostname-override=172.31.3.102" Environment="KUBELET_RINTIME=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml" Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node=''" ExecStart= ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS \$KUBELET_RINTIME EOF [root@k8s-master03 ~]# cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf <<EOF [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig" Environment="KUBELET_SYSTEM_ARGS=--hostname-override=172.31.3.103" Environment="KUBELET_RINTIME=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml" Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node=''" ExecStart= ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS \$KUBELET_RINTIME EOF

[root@k8s-master01 ~]# nerdctl login harbor.raymonds.cc Enter Username: admin Enter Password: WARN[0003] skipping verifying HTTPS certs for "harbor.raymonds.cc" WARNING: Your password will be stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/engine/reference/commandline/login/#credentials-store Login Succeeded [root@k8s-master01 ~]# cat download_pause_images.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2022-01-11 #FileName: download_pause_images.sh #Description: The test script #Copyright (C): 2022 All rights reserved #********************************************************************************************* COLOR="echo -e \\033[01;31m" END='\033[0m' PAUSE_VERSION=3.8 HARBOR_DOMAIN=harbor.raymonds.cc images_download(){ ${COLOR}"开始下载Pause镜像"${END} nerdctl pull registry.aliyuncs.com/google_containers/pause:${PAUSE_VERSION} nerdctl tag registry.aliyuncs.com/google_containers/pause:${PAUSE_VERSION} ${HARBOR_DOMAIN}/google_containers/pause:${PAUSE_VERSION} nerdctl rmi registry.aliyuncs.com/google_containers/pause:${PAUSE_VERSION} nerdctl push ${HARBOR_DOMAIN}/google_containers/pause:${PAUSE_VERSION} ${COLOR}"Pause镜像下载完成"${END} } images_download [root@k8s-master01 ~]# bash download_pause_images.sh [root@k8s-master01 ~]# nerdctl images REPOSITORY TAG IMAGE ID CREATED PLATFORM SIZE BLOB SIZE harbor.raymonds.cc/google_containers/pause 3.8 900118502363 22 seconds ago linux/amd64 696.0 KiB 304.0 KiB

[root@k8s-master01 ~]# cat > /etc/kubernetes/kubelet-conf.yml <<EOF apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration address: 0.0.0.0 port: 10250 readOnlyPort: 10255 authentication: anonymous: enabled: false webhook: cacheTTL: 2m0s enabled: true x509: clientCAFile: /etc/kubernetes/pki/ca.pem authorization: mode: Webhook webhook: cacheAuthorizedTTL: 5m0s cacheUnauthorizedTTL: 30s cgroupDriver: systemd cgroupsPerQOS: true clusterDNS: - 10.96.0.10 clusterDomain: cluster.local containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true enableDebuggingHandlers: true enforceNodeAllocatable: - pods eventBurst: 10 eventRecordQPS: 5 evictionHard: imagefs.available: 15% memory.available: 100Mi nodefs.available: 10% nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 5m0s failSwapOn: true fileCheckFrequency: 20s hairpinMode: promiscuous-bridge healthzBindAddress: 127.0.0.1 healthzPort: 10248 httpCheckFrequency: 20s imageGCHighThresholdPercent: 85 imageGCLowThresholdPercent: 80 imageMinimumGCAge: 2m0s iptablesDropBit: 15 iptablesMasqueradeBit: 14 kubeAPIBurst: 10 kubeAPIQPS: 5 makeIPTablesUtilChains: true maxOpenFiles: 1000000 maxPods: 110 nodeStatusUpdateFrequency: 10s oomScoreAdj: -999 podPidsLimit: -1 registryBurst: 10 registryPullQPS: 5 resolvConf: /etc/resolv.conf rotateCertificates: true runtimeRequestTimeout: 2m0s serializeImagePulls: true staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 4h0m0s syncFrequency: 1m0s volumeStatsAggPeriod: 1m0s EOF [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03; do scp /etc/kubernetes/kubelet-conf.yml $NODE:/etc/kubernetes/ ;done

[root@k8s-master01 ~]# systemctl daemon-reload && systemctl enable --now kubelet [root@k8s-master02 ~]# systemctl daemon-reload && systemctl enable --now kubelet [root@k8s-master03 ~]# systemctl daemon-reload && systemctl enable --now kubelet [root@k8s-master01 ~]# systemctl status kubelet ● kubelet.service - Kubernetes Kubelet Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled) Drop-In: /etc/systemd/system/kubelet.service.d └─10-kubelet.conf Active: active (running) since Sun 2022-09-25 17:18:36 CST; 30s ago Docs: https://github.com/kubernetes/kubernetes Main PID: 6441 (kubelet) Tasks: 14 (limit: 23474) Memory: 34.6M CGroup: /system.slice/kubelet.service └─6441 /usr/local/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kube> Sep 25 17:18:50 k8s-master01.example.local kubelet[6441]: E0925 17:18:50.215908 6441 kubelet.go:2448] "Error getting node" err="node \"172.> Sep 25 17:18:50 k8s-master01.example.local kubelet[6441]: I0925 17:18:50.317300 6441 kuberuntime_manager.go:1050] "Updating runtime config > Sep 25 17:18:50 k8s-master01.example.local kubelet[6441]: I0925 17:18:50.318585 6441 kubelet_network.go:60] "Updating Pod CIDR" originalPod> Sep 25 17:18:50 k8s-master01.example.local kubelet[6441]: E0925 17:18:50.318962 6441 kubelet.go:2373] "Container runtime network not ready"> Sep 25 17:18:50 k8s-master01.example.local kubelet[6441]: I0925 17:18:50.577724 6441 apiserver.go:52] "Watching apiserver" Sep 25 17:18:50 k8s-master01.example.local kubelet[6441]: I0925 17:18:50.593173 6441 reconciler.go:169] "Reconciler: start to sync state" Sep 25 17:18:51 k8s-master01.example.local kubelet[6441]: E0925 17:18:51.875100 6441 kubelet.go:2373] "Container runtime network not ready"> Sep 25 17:18:56 k8s-master01.example.local kubelet[6441]: E0925 17:18:56.879725 6441 kubelet.go:2373] "Container runtime network not ready"> Sep 25 17:19:01 k8s-master01.example.local kubelet[6441]: E0925 17:19:01.881566 6441 kubelet.go:2373] "Container runtime network not ready"> Sep 25 17:19:06 k8s-master01.example.local kubelet[6441]: E0925 17:19:06.882411 6441 kubelet.go:2373] "Container runtime network not ready"> [root@k8s-master02 ~]# systemctl status kubelet [root@k8s-master03 ~]# systemctl status kubelet

#显示只有如下信息为正常 [root@k8s-master01 ~]# tail -f /var/log/messages #ubuntu命令为"tail -f /var/log/syslog" ... Sep 23 15:11:22 localhost kubelet[16173]: E0923 15:11:22.503340 16173 kubelet.go:2373] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized"

[root@k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 172.31.3.101 NotReady <none> 38s v1.25.2 172.31.3.102 NotReady <none> 35s v1.25.2 172.31.3.103 NotReady <none> 33s v1.25.2

[root@k8s-master01 ~]# kubectl -n kube-system create serviceaccount kube-proxy #执行结果 serviceaccount/kube-proxy created [root@k8s-master01 ~]# kubectl create clusterrolebinding system:kube-proxy --clusterrole system:node-proxier --serviceaccount kube-system:kube-proxy #执行结果 clusterrolebinding.rbac.authorization.k8s.io/system:kube-proxy created [root@k8s-master01 ~]# cat > kube-proxy-scret.yml <<EOF apiVersion: v1 kind: Secret metadata: name: kube-proxy namespace: kube-system annotations: kubernetes.io/service-account.name: "kube-proxy" type: kubernetes.io/service-account-token EOF [root@k8s-master01 ~]# kubectl apply -f kube-proxy-scret.yml #执行结果 secret/kube-proxy created [root@k8s-master01 ~]# JWT_TOKEN=$(kubectl -n kube-system get secret/kube-proxy \ --output=jsonpath='{.data.token}' | base64 -d) [root@k8s-master01 ~]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://172.31.3.188:6443 --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig #执行结果 Cluster "kubernetes" set. [root@k8s-master01 ~]# kubectl config set-credentials kubernetes --token=${JWT_TOKEN} --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig #执行结果 User "kubernetes" set. [root@k8s-master01 ~]# kubectl config set-context kubernetes --cluster=kubernetes --user=kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig #执行结果 Context "kubernetes" created. [root@k8s-master01 ~]# kubectl config use-context kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig #执行结果 Switched to context "kubernetes".

[root@k8s-master01 ~]# cat > /etc/kubernetes/kube-proxy.conf <<EOF apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 clientConnection: acceptContentTypes: "" burst: 10 contentType: application/vnd.kubernetes.protobuf kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig qps: 5 clusterCIDR: 192.168.0.0/12 configSyncPeriod: 15m0s conntrack: max: null maxPerCore: 32768 min: 131072 tcpCloseWaitTimeout: 1h0m0s tcpEstablishedTimeout: 24h0m0s enableProfiling: false healthzBindAddress: 0.0.0.0:10256 hostnameOverride: "" iptables: masqueradeAll: false masqueradeBit: 14 minSyncPeriod: 0s syncPeriod: 30s ipvs: masqueradeAll: true minSyncPeriod: 5s scheduler: "rr" syncPeriod: 30s kind: KubeProxyConfiguration metricsBindAddress: 127.0.0.1:10249 mode: "ipvs" nodePortAddresses: null oomScoreAdj: -999 portRange: "" udpIdleTimeout: 250ms EOF [root@k8s-master01 ~]# cat > /lib/systemd/system/kube-proxy.service <<EOF [Unit] Description=Kubernetes Kube Proxy Documentation=https://github.com/kubernetes/kubernetes After=network.target [Service] ExecStart=/usr/local/bin/kube-proxy \\ --config=/etc/kubernetes/kube-proxy.conf \\ --v=2 Restart=always RestartSec=10s [Install] WantedBy=multi-user.target EOF [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03; do scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig scp /etc/kubernetes/kube-proxy.conf $NODE:/etc/kubernetes/kube-proxy.conf scp /lib/systemd/system/kube-proxy.service $NODE:/lib/systemd/system/kube-proxy.service done

[root@k8s-master01 ~]# systemctl daemon-reload && systemctl enable --now kube-proxy [root@k8s-master02 ~]# systemctl daemon-reload && systemctl enable --now kube-proxy [root@k8s-master03 ~]# systemctl daemon-reload && systemctl enable --now kube-proxy [root@k8s-master01 ~]# systemctl status kube-proxy ● kube-proxy.service - Kubernetes Kube Proxy Loaded: loaded (/usr/lib/systemd/system/kube-proxy.service; enabled; vendor preset: disabled) Active: active (running) since Sun 2022-09-25 17:21:25 CST; 31s ago Docs: https://github.com/kubernetes/kubernetes Main PID: 6633 (kube-proxy) Tasks: 6 (limit: 23474) Memory: 10.5M CGroup: /system.slice/kube-proxy.service └─6633 /usr/local/bin/kube-proxy --config=/etc/kubernetes/kube-proxy.conf --v=2 Sep 25 17:21:25 k8s-master01.example.local kube-proxy[6633]: I0925 17:21:25.255233 6633 flags.go:64] FLAG: --write-config-to="" Sep 25 17:21:25 k8s-master01.example.local kube-proxy[6633]: I0925 17:21:25.256063 6633 server.go:442] "Using lenient decoding as strict de> Sep 25 17:21:25 k8s-master01.example.local kube-proxy[6633]: I0925 17:21:25.256192 6633 feature_gate.go:245] feature gates: &{map[]} Sep 25 17:21:25 k8s-master01.example.local kube-proxy[6633]: I0925 17:21:25.256265 6633 feature_gate.go:245] feature gates: &{map[]} Sep 25 17:21:25 k8s-master01.example.local kube-proxy[6633]: I0925 17:21:25.275422 6633 proxier.go:666] "Failed to load kernel module with > Sep 25 17:21:25 k8s-master01.example.local kube-proxy[6633]: E0925 17:21:25.295761 6633 node.go:152] Failed to retrieve node info: nodes "k> Sep 25 17:21:26 k8s-master01.example.local kube-proxy[6633]: E0925 17:21:26.318886 6633 node.go:152] Failed to retrieve node info: nodes "k> Sep 25 17:21:28 k8s-master01.example.local kube-proxy[6633]: E0925 17:21:28.497970 6633 node.go:152] Failed to retrieve node info: nodes "k> Sep 25 17:21:32 k8s-master01.example.local kube-proxy[6633]: E0925 17:21:32.991308 6633 node.go:152] Failed to retrieve node info: nodes "k> Sep 25 17:21:41 k8s-master01.example.local kube-proxy[6633]: E0925 17:21:41.810292 6633 node.go:152] Failed to retrieve node info: nodes "k> [root@k8s-master02 ~]# systemctl status kube-proxy [root@k8s-master03 ~]# systemctl status kube-proxy [root@k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 172.31.3.101 NotReady <none> 3m29s v1.25.2 172.31.3.102 NotReady <none> 3m26s v1.25.2 172.31.3.103 NotReady <none> 3m24s v1.25.2

[root@k8s-master01 ~]# for NODE in k8s-node01 k8s-node02 k8s-node03; do echo $NODE; scp -o StrictHostKeyChecking=no /usr/local/bin/kube{let,-proxy} $NODE:/usr/local/bin/ ; done

[root@k8s-node01 ~]# mkdir -p /opt/cni/bin [root@k8s-node02 ~]# mkdir -p /opt/cni/bin [root@k8s-node03 ~]# mkdir -p /opt/cni/bin

[root@k8s-node01 ~]# mkdir /etc/etcd/ssl -p [root@k8s-node02 ~]# mkdir /etc/etcd/ssl -p [root@k8s-node03 ~]# mkdir /etc/etcd/ssl -p

[root@k8s-etcd01 pki]# for NODE in k8s-node01 k8s-node02 k8s-node03; do ssh -o StrictHostKeyChecking=no $NODE "mkdir -p /etc/etcd/ssl" for FILE in etcd-ca-key.pem etcd-ca.pem etcd-key.pem etcd.pem; do scp -o StrictHostKeyChecking=no /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE} done done

[root@k8s-node01 ~]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-node02 ~]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-node03 ~]# mkdir /etc/kubernetes/pki/etcd -p [root@k8s-node01 ~]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/ [root@k8s-node02 ~]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/ [root@k8s-node03 ~]# ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/

[root@k8s-node01 ~]# mkdir -p /etc/kubernetes/pki [root@k8s-node02 ~]# mkdir -p /etc/kubernetes/pki [root@k8s-node03 ~]# mkdir -p /etc/kubernetes/pki

[root@k8s-master01 ~]# for NODE in k8s-node01 k8s-node02 k8s-node03; do for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig; do scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE} done done

[root@k8s-node01 ~]# mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes [root@k8s-node02 ~]# mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes [root@k8s-node03 ~]# mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes

[root@k8s-master01 ~]# for NODE in k8s-node01 k8s-node02 k8s-node03; do scp /lib/systemd/system/kubelet.service $NODE:/lib/systemd/system/ ;done

[root@k8s-node01 ~]# cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf <<EOF [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig" Environment="KUBELET_SYSTEM_ARGS=--hostname-override=172.31.3.111" Environment="KUBELET_RINTIME=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml" Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node=''" ExecStart= ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS \$KUBELET_RINTIME EOF [root@k8s-node02 ~]# cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf <<EOF [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig" Environment="KUBELET_SYSTEM_ARGS=--hostname-override=172.31.3.112" Environment="KUBELET_RINTIME=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml" Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node=''" ExecStart= ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS \$KUBELET_RINTIME EOF [root@k8s-node03 ~]# cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf <<EOF [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig" Environment="KUBELET_SYSTEM_ARGS=--hostname-override=172.31.3.113" Environment="KUBELET_RINTIME=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml" Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node=''" ExecStart= ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS \$KUBELET_RINTIME EOF

[root@k8s-master01 ~]# for NODE in k8s-node01 k8s-node02 k8s-node03; do scp /etc/kubernetes/kubelet-conf.yml $NODE:/etc/kubernetes/ ;done

[root@k8s-node01 ~]# systemctl daemon-reload && systemctl enable --now kubelet [root@k8s-node02 ~]# systemctl daemon-reload && systemctl enable --now kubelet [root@k8s-node03 ~]# systemctl daemon-reload && systemctl enable --now kubelet [root@k8s-node01 ~]# systemctl status kubelet ● kubelet.service - Kubernetes Kubelet Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled) Drop-In: /etc/systemd/system/kubelet.service.d └─10-kubelet.conf Active: active (running) since Sun 2022-09-25 17:26:52 CST; 13s ago Docs: https://github.com/kubernetes/kubernetes Main PID: 5957 (kubelet) Tasks: 14 (limit: 23474) Memory: 34.9M CGroup: /system.slice/kubelet.service └─5957 /usr/local/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kube> Sep 25 17:26:53 k8s-node01.example.local kubelet[5957]: I0925 17:26:53.683391 5957 kubelet_network_linux.go:63] "Initialized iptables rules> Sep 25 17:26:53 k8s-node01.example.local kubelet[5957]: I0925 17:26:53.766025 5957 kubelet_network_linux.go:63] "Initialized iptables rules> Sep 25 17:26:53 k8s-node01.example.local kubelet[5957]: I0925 17:26:53.766047 5957 status_manager.go:161] "Starting to sync pod status with> Sep 25 17:26:53 k8s-node01.example.local kubelet[5957]: I0925 17:26:53.766064 5957 kubelet.go:2010] "Starting kubelet main sync loop" Sep 25 17:26:53 k8s-node01.example.local kubelet[5957]: E0925 17:26:53.766104 5957 kubelet.go:2034] "Skipping pod synchronization" err="PLE> Sep 25 17:26:54 k8s-node01.example.local kubelet[5957]: I0925 17:26:54.279327 5957 apiserver.go:52] "Watching apiserver" Sep 25 17:26:54 k8s-node01.example.local kubelet[5957]: I0925 17:26:54.305974 5957 reconciler.go:169] "Reconciler: start to sync state" Sep 25 17:26:58 k8s-node01.example.local kubelet[5957]: E0925 17:26:58.602184 5957 kubelet.go:2373] "Container runtime network not ready" n> Sep 25 17:27:03 k8s-node01.example.local kubelet[5957]: I0925 17:27:03.232179 5957 transport.go:135] "Certificate rotation detected, shutti> Sep 25 17:27:03 k8s-node01.example.local kubelet[5957]: E0925 17:27:03.603901 5957 kubelet.go:2373] "Container runtime network not ready" n> [root@k8s-node02 ~]# systemctl status kubelet [root@k8s-node03 ~]# systemctl status kubelet

[root@k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 172.31.3.101 NotReady <none> 9m8s v1.25.2 172.31.3.102 NotReady <none> 9m5s v1.25.2 172.31.3.103 NotReady <none> 9m3s v1.25.2 172.31.3.111 NotReady <none> 64s v1.25.2 172.31.3.112 NotReady <none> 48s v1.25.2 172.31.3.113 NotReady <none> 46s v1.25.2

[root@k8s-master01 ~]# for NODE in k8s-node01 k8s-node02 k8s-node03; do scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig scp /etc/kubernetes/kube-proxy.conf $NODE:/etc/kubernetes/kube-proxy.conf scp /lib/systemd/system/kube-proxy.service $NODE:/lib/systemd/system/kube-proxy.service done

[root@k8s-node01 ~]# systemctl daemon-reload && systemctl enable --now kube-proxy [root@k8s-node02 ~]# systemctl daemon-reload && systemctl enable --now kube-proxy [root@k8s-node03 ~]# systemctl daemon-reload && systemctl enable --now kube-proxy [root@k8s-node01 ~]# systemctl status kube-proxy ● kube-proxy.service - Kubernetes Kube Proxy Loaded: loaded (/usr/lib/systemd/system/kube-proxy.service; enabled; vendor preset: disabled) Active: active (running) since Sun 2022-09-25 17:28:29 CST; 26s ago Docs: https://github.com/kubernetes/kubernetes Main PID: 6162 (kube-proxy) Tasks: 5 (limit: 23474) Memory: 10.8M CGroup: /system.slice/kube-proxy.service └─6162 /usr/local/bin/kube-proxy --config=/etc/kubernetes/kube-proxy.conf --v=2 Sep 25 17:28:29 k8s-node01.example.local kube-proxy[6162]: I0925 17:28:29.391982 6162 flags.go:64] FLAG: --write-config-to="" Sep 25 17:28:29 k8s-node01.example.local kube-proxy[6162]: I0925 17:28:29.393147 6162 server.go:442] "Using lenient decoding as strict deco> Sep 25 17:28:29 k8s-node01.example.local kube-proxy[6162]: I0925 17:28:29.393298 6162 feature_gate.go:245] feature gates: &{map[]} Sep 25 17:28:29 k8s-node01.example.local kube-proxy[6162]: I0925 17:28:29.393365 6162 feature_gate.go:245] feature gates: &{map[]} Sep 25 17:28:29 k8s-node01.example.local kube-proxy[6162]: I0925 17:28:29.407927 6162 proxier.go:666] "Failed to load kernel module with mo> Sep 25 17:28:29 k8s-node01.example.local kube-proxy[6162]: E0925 17:28:29.427017 6162 node.go:152] Failed to retrieve node info: nodes "k8s> Sep 25 17:28:30 k8s-node01.example.local kube-proxy[6162]: E0925 17:28:30.464338 6162 node.go:152] Failed to retrieve node info: nodes "k8s> Sep 25 17:28:32 k8s-node01.example.local kube-proxy[6162]: E0925 17:28:32.508065 6162 node.go:152] Failed to retrieve node info: nodes "k8s> Sep 25 17:28:36 k8s-node01.example.local kube-proxy[6162]: E0925 17:28:36.760898 6162 node.go:152] Failed to retrieve node info: nodes "k8s> Sep 25 17:28:45 k8s-node01.example.local kube-proxy[6162]: E0925 17:28:45.067601 6162 node.go:152] Failed to retrieve node info: nodes "k8s> [root@k8s-node02 ~]# systemctl status kube-proxy [root@k8s-node03 ~]# systemctl status kube-proxy

[root@k8s-master01 ~]# curl https://docs.projectcalico.org/manifests/calico-etcd.yaml -O

[root@k8s-master01 ~]# grep "etcd_endpoints:.*" calico-etcd.yaml etcd_endpoints: "http://<ETCD_IP>:<ETCD_PORT>" [root@k8s-master01 ~]# sed -i 's#etcd_endpoints: "http://<ETCD_IP>:<ETCD_PORT>"#etcd_endpoints: "https://172.31.3.108:2379,https://172.31.3.109:2379,https://172.31.3.110:2379"#g' calico-etcd.yaml [root@k8s-master01 ~]# grep "etcd_endpoints:.*" calico-etcd.yaml etcd_endpoints: "https://172.31.3.108:2379,https://172.31.3.109:2379,https://172.31.3.110:2379" [root@k8s-master01 ~]# grep -E "(.*etcd-key:.*|.*etcd-cert:.*|.*etcd-ca:.*)" calico-etcd.yaml # etcd-key: null # etcd-cert: null # etcd-ca: null [root@k8s-master01 ~]# ETCD_KEY=`cat /etc/kubernetes/pki/etcd/etcd-key.pem | base64 | tr -d '\n'` [root@k8s-master01 ~]# ETCD_CERT=`cat /etc/kubernetes/pki/etcd/etcd.pem | base64 | tr -d '\n'` [root@k8s-master01 ~]# ETCD_CA=`cat /etc/kubernetes/pki/etcd/etcd-ca.pem | base64 | tr -d '\n'` [root@k8s-master01 ~]# sed -i "s@# etcd-key: null@etcd-key: ${ETCD_KEY}@g; s@# etcd-cert: null@etcd-cert: ${ETCD_CERT}@g; s@# etcd-ca: null@etcd-ca: ${ETCD_CA}@g" calico-etcd.yaml [root@k8s-master01 ~]# grep -E "(.*etcd-key:.*|.*etcd-cert:.*|.*etcd-ca:.*)" calico-etcd.yaml etcd-key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdTBieEdrUE5pM3IyaGdNTlUvU0tjNitjZHVUNWk4N1FIU1Q1Tk9mY0ExTDlPcGxsCm1LTllPTXAzcSt1Wko1bHZ4Ui9QalIxZnV4QUtramQ2THBMQnNBRmxGTm1lNDh4RTJRUVlWQjFDRGFCU0lzcUMKRUxtR3BXdUhkR2RRbzE4YmUxK2hvKzFXMHpzdDV2VGNaTE4vRlhiNURQWFBLZEw4Mm5NZkRmcndGWHEyUCttWQp3M2p0RU1jc3BJcXlTZzhnQ2NpZHhySzNDYkkwYVdkZTE3OVpNU0I2bmtlbXd2Mi9wWDJ6OHZMOUNwZXFuSnR4ClFvcUd6ckpXc0w0K0xyYUVwL0FUQU9pbE42K2VsR2hWL2NHTm45LzJDNStPWTRPTWkxYXl0anVMUVgyWlpMbTIKWGdnWGdGczRHaTF4SWg5aFBGSWxnUW1zRzlxaTZKUW1hNHNuTlFJREFRQUJBb0lCQUErM2VHeDMzRHdrYWFQOQpoWlRTSlB4b2RIMFY0a3QxWThuT1hJdXdHYXE5d0RxMnZPdithVno2d09oUXNWMjlaci9vVjRiRVBGQjZuQ2lCCk4yUEpOVEFNTGV0K0IvT2VKUGtCZXZrMEsyTHhYWE5HQTN4YjFZejBaVDNEbmVUWUNucGtJRENkcm5lenM3cDYKT2pUSlM1VUZrd2tmWis5ZW9aSERyNHVBejcwOWxrSWpidHdxOURtVTFhR1F2QTMxWDFaY2dKUENGWVpOeFRCaQpRbEhGdVJ5d2hCM2g3bzBGUWs0ekEzblBkYlJrdkVXM202ak44NkdKV2oxUHlLYjZiK0c3UXR6cit5MnNQVlN3CnhjaHlSc2VJRkVWQ2lFQzNlMUFYdjRpdXp0Q2o4a3lEbVZmbkZaOGQvNXp1eittUVVWQjB3ZGFleVgyZmE3aFcKM05ZbkZORUNnWUVBeU1TZm9QaVBNVXpkdUFYSkZCM2xUdVlIVnRYYlNkRGZsc3Z2b01HT0NTU1crWUUzS3hQcwpMMERzVENyNVdvVmJqVVppRTRUQ0pHVmdLNVNFemhjOWZrRmFTM29ZR2FJMGdqamk4RStrNWJ3RU8xdkN0eFZOCnpsbjR3RUZMbFFFUFppWjV4SWl0eExuREYrT1VHVEErZVJFMytnL0ZjK3NiNmhMUkU1YjV1UWNDZ1lFQTdzd3oKQldjSS9tUzVSV2VKVU1nZUJ2Vm5OdXJDZzd0ZWRqc0ExU3Y1VXVwajI2d2traE1PbmRLNTZrR1o5Y0FrRU85RgpaMmRSeEM1QmJDWmV4Tnl1Z3lKN2dBWTNmOGNJU0hCMk00cGN2UWZZTTNUbUZHaDVVMWdpOGVEUVZabzZUZ2dyCk1TaU9ZaUJhQUEyMldUc0ppL2pzUXgzVDhscG9SV25YcjF5cnV1TUNnWUJSL3V2ckJGa0hHNHVhTXRLeTRwcmEKcEZ2dS9SeTRneFF1TkZCRDZZa205c2lxVWpuRDREa2YrM1lHamE2Vlo4M0NYektESWo5Z09mOFREVzlIOUhucQo3S29DRlhWdVVxNzdXRnhuSlVBRmk4cDJxNzFVcE9ESUhEcloybEVTSkFLMEI1Ykh5OEtjaS9tLzhmUjBiUjIyCnVHK1NNNHJERXd5dGhzM1pJRm9SVlFLQmdRRGFVQThaRWxDVG1td1Mrb3Y4TVdmYVBzS2szMDBEZCtudE54WVUKelVYOE90TWVRcXVRYkNIQndhUThlTXNUZEJ6RTZxck4xUlJZd1YwSVRhLzRWRFNyS0h3MTEva25OVVBxVGY2UwpNaDJFcDhaTmpNTEh4NWViellqdER2WUlQSjZ2TmlLZXA1QThQNDFvWFNEblJPVCtkWTB4OHZRUXJmeUQ4VGJCCldIeWJ0d0tCZ0MrNlN6YzBpR0ZyM1VoSVc1ZHFOOERJYkl0eHZ5REh5WWtrWlB5UytueVo5OEw4ZlZnNGNYa3UKU0p4WENKNlZwOGZJY09NR0VtakIwRFp5SlRxUTUweFpRKzhkS3F3akV0SjFwNGVNRmIxamxickhHczhGRnJhOAorWkR6dTN1RGZIZXBFZzFub1dBZEo5RWV6VWI1RWdVOWtaeHdoWHNFZzJFMDZGYjhYbm42Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== etcd-cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVCVENDQXUyZ0F3SUJBZ0lVYkw0ckFPcVk0UmQ3WWhrY3h2VGxpNkxZMDBzd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1p6RUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeERUQUxCZ05WQkFvVEJHVjBZMlF4RmpBVUJnTlZCQXNURFVWMFkyUWdVMlZqZFhKcGRIa3hEVEFMCkJnTlZCQU1UQkdWMFkyUXdJQmNOTWpJd09USTFNRGd4T0RBd1doZ1BNakV5TWpBNU1ERXdPREU0TURCYU1HY3gKQ3pBSkJnTlZCQVlUQWtOT01SQXdEZ1lEVlFRSUV3ZENaV2xxYVc1bk1SQXdEZ1lEVlFRSEV3ZENaV2xxYVc1bgpNUTB3Q3dZRFZRUUtFd1JsZEdOa01SWXdGQVlEVlFRTEV3MUZkR05rSUZObFkzVnlhWFI1TVEwd0N3WURWUVFECkV3UmxkR05rTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF1MGJ4R2tQTmkzcjIKaGdNTlUvU0tjNitjZHVUNWk4N1FIU1Q1Tk9mY0ExTDlPcGxsbUtOWU9NcDNxK3VaSjVsdnhSL1BqUjFmdXhBSwpramQ2THBMQnNBRmxGTm1lNDh4RTJRUVlWQjFDRGFCU0lzcUNFTG1HcFd1SGRHZFFvMThiZTEraG8rMVcwenN0CjV2VGNaTE4vRlhiNURQWFBLZEw4Mm5NZkRmcndGWHEyUCttWXczanRFTWNzcElxeVNnOGdDY2lkeHJLM0NiSTAKYVdkZTE3OVpNU0I2bmtlbXd2Mi9wWDJ6OHZMOUNwZXFuSnR4UW9xR3pySldzTDQrTHJhRXAvQVRBT2lsTjYrZQpsR2hWL2NHTm45LzJDNStPWTRPTWkxYXl0anVMUVgyWlpMbTJYZ2dYZ0ZzNEdpMXhJaDloUEZJbGdRbXNHOXFpCjZKUW1hNHNuTlFJREFRQUJvNEdtTUlHak1BNEdBMVVkRHdFQi93UUVBd0lGb0RBZEJnTlZIU1VFRmpBVUJnZ3IKQmdFRkJRY0RBUVlJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvQkFJd0FEQWRCZ05WSFE0RUZnUVU5VmxoYTRpaApjUzVIV3hCUEU2WkhBYldRbExnd1JRWURWUjBSQkQ0d1BJSUthemh6TFdWMFkyUXdNWUlLYXpoekxXVjBZMlF3Ck1vSUthemh6TFdWMFkyUXdNNGNFZndBQUFZY0VyQjhEYkljRXJCOERiWWNFckI4RGJqQU5CZ2txaGtpRzl3MEIKQVFzRkFBT0NBUUVBTXdEbjhldlZEWU9Jc1lyUnJhVGVTdmRPbTZjVGJNd1VaVFVJWDE5L0FPeVRVUzZMUWwzZApGZnpPWGUrV1YwOGhsOUhVd0NueWQ2RW5kVXlWTHVaVTBETGkxazhwcXV2b3pkWDZkajNHRDRGM0lINXU2ZytOClVnTUFuc1lKLzBzNElJcG9ZVWVvNFgrd3drU3RMR2ZuVlZqZnF3OS9VbGE4WEZMWTRGSm9EUGw3VzJuWTFncUUKUlpJbnRFWHFOeVZlSGQzRDNvalVJWTVtTGZiRkdHOGVmSmIwbjIwNEpBTjdBbzZhUkw4cmdiS2tLbTRIWjArYgp3bENBRmc0d2FZdlV3UEp0S3NyN2NFN1l6d0RwbGdBZG1lbzVaZ0RkeVU3WmFPUHJ5SE5YUE5LNEtMUDNFZmVHCis3bGhzc0t4T1Y3WDNxNTJBM1o2eWpab2hEZ1o4Y1RxMWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== etcd-ca: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURvRENDQW9pZ0F3SUJBZ0lVWVQ4Z0ZmSHFHRi9aYlljS2dPbTVUakVtdlowd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1p6RUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeERUQUxCZ05WQkFvVEJHVjBZMlF4RmpBVUJnTlZCQXNURFVWMFkyUWdVMlZqZFhKcGRIa3hEVEFMCkJnTlZCQU1UQkdWMFkyUXdJQmNOTWpJd09USTFNRGd4TnpBd1doZ1BNakV5TWpBNU1ERXdPREUzTURCYU1HY3gKQ3pBSkJnTlZCQVlUQWtOT01SQXdEZ1lEVlFRSUV3ZENaV2xxYVc1bk1SQXdEZ1lEVlFRSEV3ZENaV2xxYVc1bgpNUTB3Q3dZRFZRUUtFd1JsZEdOa01SWXdGQVlEVlFRTEV3MUZkR05rSUZObFkzVnlhWFI1TVEwd0N3WURWUVFECkV3UmxkR05rTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF0VzFEOWZuQ1NYM2cKZitLVlJFb0JJV2RGWitlVkpwZWpWK2Nhckk3WERLWTdiNnpuTjJtMlN1ZDVvck05Q1lDWTBHenlkRk1kMHVabQpIckZ5aE5KKzhuNlRBSW9VUkVxc3d4aXRVanA3SXI2blZQTG52aTNKZ0o5WGpnZmtQbDF4cGNrRnFtbzFwd0FkCk0xQzdORlhldlVkaVF6aHhSYUh4dEo0VDhNQ256dVJ6blNTYUdjbkFWT0xQdjNJdXJkSk9NaDNta00vQXNIN0cKajZKRHNHZEdhSFk2VU1BbEQzOG9kcVFObEpDNHBLelZIbTFWV3BSREx0dFdMcFJHNDdSdjlHUmsrRUdocEVrbwpKNVl4aXpBSngrRDI5ZE5kSm16UkxLMDJrNjhTUTNSRkdaOWVpUTA0amhPWFJMNUVEVVRDQkNON1hvRjFwU0dHClBZcm9WV0czZndJREFRQUJvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBUVl3RHdZRFZSMFRBUUgvQkFVd0F3RUIKL3pBZEJnTlZIUTRFRmdRVWlVRk1Bb3Z5RlBLaWd0bE12YlUyVk56NEF4Z3dEUVlKS29aSWh2Y05BUUVMQlFBRApnZ0VCQUZ4d29tMHgyeDZWNUhVaE9IVkgyVGcyUGdacFFYMTFDS2hGM2JBQUw2M0preCtKK3pUZmJJTGRoM29HCmU3bEJPT2RpVXF5dWI4bmQxY2pIa2VqS3R5RFlOOE5MNnVRSXFOQ2M2SFB1NjZ6TEpVU0JwaFZ4YktVSEFxWGYKTWRuSkQzekoyYkM2Sml5Mi9IRW5heXM5MnllRjA1YUxOYjQ2aXB5Slh5bEk0MmNSbm11cjY0L1ZJdVQ2dkZISgp3bEZDb1FvWmVWT0hYRXJYcVA4V0ozWUpOdGpkWktsMHRQdWVYTG5Ob1VYSmlLYkE1TEFESHJaS1ptUUxWNGpHCkc1T05EV3VCNjIvY1NVMGtRbFFJeU5kMVBxVHFpdlI5Wm9iWnhoZGJIa2g1MGVxU1V3YmN1V2hzQytRelA2ZVgKRFBueUJxaThaVitSV25kR1doMXdQbVdjd2I4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== [root@k8s-master01 ~]# grep -E "(.*etcd_ca:.*|.*etcd_cert:.*|.*etcd_key:.*)" calico-etcd.yaml etcd_ca: "" # "/calico-secrets/etcd-ca" etcd_cert: "" # "/calico-secrets/etcd-cert" etcd_key: "" # "/calico-secrets/etcd-key" [root@k8s-master01 ~]# sed -i 's#etcd_ca: ""#etcd_ca: "/calico-secrets/etcd-ca"#g; s#etcd_cert: ""#etcd_cert: "/calico-secrets/etcd-cert"#g; s#etcd_key: "" #etcd_key: "/calico-secrets/etcd-key" #g' calico-etcd.yaml [root@k8s-master01 ~]# grep -E "(.*etcd_ca:.*|.*etcd_cert:.*|.*etcd_key:.*)" calico-etcd.yaml etcd_ca: "/calico-secrets/etcd-ca" # "/calico-secrets/etcd-ca" etcd_cert: "/calico-secrets/etcd-cert" # "/calico-secrets/etcd-cert" etcd_key: "/calico-secrets/etcd-key" # "/calico-secrets/etcd-key" # 更改此处为自己的pod网段 [root@k8s-master01 ~]# POD_SUBNET="192.168.0.0/12"

[root@k8s-master01 ~]# grep -E "(.*CALICO_IPV4POOL_CIDR.*|.*192.168.0.0.*)" calico-etcd.yaml # - name: CALICO_IPV4POOL_CIDR # value: "192.168.0.0/16" [root@k8s-master01 ~]# sed -i 's@# - name: CALICO_IPV4POOL_CIDR@- name: CALICO_IPV4POOL_CIDR@g; s@# value: "192.168.0.0/16"@ value: '"${POD_SUBNET}"'@g' calico-etcd.yaml [root@k8s-master01 ~]# grep -E "(.*CALICO_IPV4POOL_CIDR.*|.*192.168.0.0.*)" calico-etcd.yaml - name: CALICO_IPV4POOL_CIDR value: 192.168.0.0/12 [root@k8s-master01 ~]# grep "image:" calico-etcd.yaml image: docker.io/calico/cni:v3.24.1 image: docker.io/calico/node:v3.24.1 image: docker.io/calico/node:v3.24.1 image: docker.io/calico/kube-controllers:v3.24.1

[root@k8s-master01 ~]# cat download_calico_images.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2022-01-11 #FileName: download_calico_images.sh #Description: The test script #Copyright (C): 2022 All rights reserved #********************************************************************************************* COLOR="echo -e \\033[01;31m" END='\033[0m' images=$(awk -F "/" '/image:/{print $NF}' calico-etcd.yaml | uniq) HARBOR_DOMAIN=harbor.raymonds.cc images_download(){ ${COLOR}"开始下载Calico镜像"${END} for i in ${images};do nerdctl pull registry.cn-beijing.aliyuncs.com/raymond9/$i nerdctl tag registry.cn-beijing.aliyuncs.com/raymond9/$i ${HARBOR_DOMAIN}/google_containers/$i nerdctl rmi registry.cn-beijing.aliyuncs.com/raymond9/$i nerdctl push ${HARBOR_DOMAIN}/google_containers/$i done ${COLOR}"Calico镜像下载完成"${END} } images_download [root@k8s-master01 ~]# bash download_calico_images.sh [root@k8s-master01 ~]# nerdctl images | grep v3.24.1 harbor.raymonds.cc/google_containers/cni v3.24.1 21df750b80ba About a minute ago linux/amd64 188.3 MiB 83.3 MiB harbor.raymonds.cc/google_containers/kube-controllers v3.24.1 b65317537174 26 seconds ago linux/amd64 68.0 MiB 29.7 MiB harbor.raymonds.cc/google_containers/node v3.24.1 135054e0bc90 37 seconds ago linux/amd64 218.7 MiB 76.5 MiB [root@k8s-master01 ~]# sed -ri 's@(.*image:) docker.io/calico(/.*)@\1 harbor.raymonds.cc/google_containers\2@g' calico-etcd.yaml [root@k8s-master01 ~]# grep "image:" calico-etcd.yaml image: harbor.raymonds.cc/google_containers/cni:v3.24.1 image: harbor.raymonds.cc/google_containers/node:v3.24.1 image: harbor.raymonds.cc/google_containers/node:v3.24.1 image: harbor.raymonds.cc/google_containers/kube-controllers:v3.24.1 [root@k8s-master01 ~]# kubectl apply -f calico-etcd.yaml #查看容器状态 [root@k8s-master01 ~]# kubectl get pod -n kube-system |grep calico calico-kube-controllers-67bd695c56-zt5k8 1/1 Running 0 6m44s calico-node-fm5x2 1/1 Running 0 6m44s calico-node-gbhd2 1/1 Running 0 6m44s calico-node-m7gcc 1/1 Running 0 6m44s calico-node-nnbjq 1/1 Running 0 6m44s calico-node-v9cgw 1/1 Running 0 6m44s calico-node-zlmx6 1/1 Running 0 6m44s #查看集群状态 [root@k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 172.31.3.101 Ready <none> 28m v1.25.2 172.31.3.102 Ready <none> 28m v1.25.2 172.31.3.103 Ready <none> 28m v1.25.2 172.31.3.111 Ready <none> 20m v1.25.2 172.31.3.112 Ready <none> 19m v1.25.2 172.31.3.113 Ready <none> 19m v1.25.2

[root@k8s-master01 ~]# wget https://dl.k8s.io/v1.25.2/kubernetes.tar.gz [root@k8s-master01 ~]# tar xf kubernetes.tar.gz [root@k8s-master01 ~]# cp kubernetes/cluster/addons/dns/coredns/coredns.yaml.base /root/coredns.yaml [root@k8s-master01 ~]# vim coredns.yaml ... data: Corefile: | .:53 { errors health { lameduck 5s } ready kubernetes cluster.local in-addr.arpa ip6.arpa { #修改Cluster DNS Domain pods insecure fallthrough in-addr.arpa ip6.arpa ttl 30 } prometheus :9153 forward . /etc/resolv.conf { max_concurrent 1000 } cache 30 loop reload loadbalance } ... containers: - name: coredns image: registry.k8s.io/coredns/coredns:v1.9.3 imagePullPolicy: IfNotPresent resources: limits: memory: 256Mi #设置内存大小，生产一般内存设置2-3G，CPU会分到2-4核，如果再高就开多副本 requests: cpu: 100m memory: 70Mi ... spec: selector: k8s-app: kube-dns clusterIP: 10.96.0.10 #如果更改了k8s service的网段需要将coredns的serviceIP改成k8s service网段的第十个IP ports: - name: dns port: 53 protocol: UDP - name: dns-tcp port: 53 protocol: TCP - name: metrics port: 9153 protocol: TCP

[root@k8s-master01 ~]# grep "image:" coredns.yaml image: registry.k8s.io/coredns/coredns:v1.9.3 [root@k8s-master01 ~]# cat download_coredns_images.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2022-01-11 #FileName: download_metrics_images. #Description: The test script #Copyright (C): 2022 All rights reserved #********************************************************************************************* COLOR="echo -e \\033[01;31m" END='\033[0m' images=$(awk -F "/" '/image:/{print $NF}' coredns.yaml) HARBOR_DOMAIN=harbor.raymonds.cc images_download(){ ${COLOR}"开始下载Coredns镜像"${END} for i in ${images};do nerdctl pull registry.aliyuncs.com/google_containers/$i nerdctl tag registry.aliyuncs.com/google_containers/$i ${HARBOR_DOMAIN}/google_containers/$i nerdctl rmi registry.aliyuncs.com/google_containers/$i nerdctl push ${HARBOR_DOMAIN}/google_containers/$i done ${COLOR}"Coredns镜像下载完成"${END} } images_download [root@k8s-master01 ~]# bash download_coredns_images.sh [root@k8s-master01 ~]# nerdctl images |grep coredns harbor.raymonds.cc/google_containers/coredns v1.9.3 8e352a029d30 15 seconds ago linux/amd64 46.5 MiB 14.2 MiB [root@k8s-master01 ~]# sed -ri 's@(.*image:) .*coredns(/.*)@\1 harbor.raymonds.cc/google_containers\2@g' coredns.yaml [root@k8s-master01 ~]# grep "image:" coredns.yaml image: harbor.raymonds.cc/google_containers/coredns:v1.9.3 [root@k8s-master01 ~]# kubectl apply -f coredns.yaml #查看状态 [root@k8s-master01 ~]# kubectl get pod -n kube-system |grep coredns coredns-8668f8476d-njcl2 1/1 Running 0 7s

root@k8s-master01:~# kubectl get pod -A -o wide|grep coredns NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system coredns-847c895554-9jqq5 0/1 CrashLoopBackOff 1 8s 192.171.30.65 k8s-master02.example.local <none> <none> #由于ubuntu系统有dns本地缓存,造成coredns不能正常解析 #具体问题请参考官方https://coredns.io/plugins/loop/#troubleshooting root@k8s-master01:~# kubectl edit -n kube-system cm coredns ... # Please edit the object below. Lines beginning with a '#' will be ignored, # and an empty file will abort the edit. If an error occurs while saving this file will be # reopened with the relevant failures. .:53 { errors health { lameduck 5s } ready kubernetes cluster.local in-addr.arpa ip6.arpa { fallthrough in-addr.arpa ip6.arpa } prometheus :9153 forward . /etc/resolv.conf { max_concurrent 1000 } cache 30 loop #将loop插件直接删除，避免内部循环 reload loadbalance } root@k8s-master01:~# kubectl get pod -A -o wide |grep coredns NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system coredns-847c895554-r9tsd 0/1 CrashLoopBackOff 4 3m4s 192.170.21.195 k8s-node03.example.local <none> <none> root@k8s-master01:~# kubectl delete pod coredns-847c895554-r9tsd -n kube-system pod "coredns-847c895554-r9tsd" deleted root@k8s-master01:~# kubectl get pod -A -o wide |greo coredns NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system coredns-847c895554-cqwl5 1/1 Running 0 13s 192.167.195.130 k8s-node02.example.local <none> <none> #现在就正常了

[root@k8s-master01 ~]# wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

[root@k8s-master01 ~]# vim components.yaml ... spec: containers: - args: - --cert-dir=/tmp - --secure-port=4443 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --kubelet-use-node-status-port - --metric-resolution=15s #添加下面内容 - --kubelet-insecure-tls - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem #注意二进制包证书文件是front-proxy-ca.pem - --requestheader-username-headers=X-Remote-User - --requestheader-group-headers=X-Remote-Group - --requestheader-extra-headers-prefix=X-Remote-Extra- ... volumeMounts: - mountPath: /tmp name: tmp-dir #添加下面内容 - name: ca-ssl mountPath: /etc/kubernetes/pki ... volumes: - emptyDir: {} name: tmp-dir #添加下面内容 - name: ca-ssl hostPath: path: /etc/kubernetes/pki

[root@k8s-master01 ~]# grep "image:" components.yaml image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1 [root@k8s-master01 ~]# cat download_metrics_images.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2022-01-11 #FileName: download_metrics_images. #Description: The test script #Copyright (C): 2022 All rights reserved #********************************************************************************************* COLOR="echo -e \\033[01;31m" END='\033[0m' images=$(awk -F "/" '/image:/{print $NF}' components.yaml) HARBOR_DOMAIN=harbor.raymonds.cc images_download(){ ${COLOR}"开始下载Metrics镜像"${END} for i in ${images};do nerdctl pull registry.aliyuncs.com/google_containers/$i nerdctl tag registry.aliyuncs.com/google_containers/$i ${HARBOR_DOMAIN}/google_containers/$i nerdctl rmi registry.aliyuncs.com/google_containers/$i nerdctl push ${HARBOR_DOMAIN}/google_containers/$i done ${COLOR}"Metrics镜像下载完成"${END} } images_download [root@k8s-master01 ~]# bash download_metrics_images.sh [root@k8s-master01 ~]# nerdctl images |grep metric harbor.raymonds.cc/google_containers/metrics-server v0.6.1 5ddc6458eb95 8 seconds ago linux/amd64 68.9 MiB 26.8 MiB [root@k8s-master01 ~]# sed -ri 's@(.*image:) .*metrics-server(/.*)@\1 harbor.raymonds.cc/google_containers\2@g' components.yaml [root@k8s-master01 ~]# grep "image:" components.yaml image: harbor.raymonds.cc/google_containers/metrics-server:v0.6.1

[root@k8s-master01 ~]# kubectl apply -f components.yaml

[root@k8s-master01 ~]# kubectl get pod -n kube-system |grep metrics metrics-server-785dd7cc54-48xhq 1/1 Running 0 31s [root@k8s-master01 ~]# kubectl top node NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% 172.31.3.101 173m 8% 1732Mi 48% 172.31.3.102 111m 5% 964Mi 26% 172.31.3.103 85m 4% 1348Mi 37% 172.31.3.111 49m 2% 588Mi 16% 172.31.3.112 60m 3% 611Mi 16% 172.31.3.113 80m 4% 634Mi 17%

[root@k8s-master01 ~]# wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml [root@k8s-master01 ~]# vim recommended.yaml ... kind: Service apiVersion: v1 metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard namespace: kubernetes-dashboard spec: type: NodePort #添加这行 ports: - port: 443 targetPort: 8443 nodePort: 30005 #添加这行 selector: k8s-app: kubernetes-dashboard ... [root@k8s-master01 ~]# grep "image:" recommended.yaml image: kubernetesui/dashboard:v2.7.0 image: kubernetesui/metrics-scraper:v1.0.8

[root@k8s-master01 ~]# cat download_dashboard_images.sh #!/bin/bash # #********************************************************************************************** #Author: Raymond #Date: 2022-01-11 #FileName: download_dashboard_images.sh #Description: The test script #Copyright (C): 2022 All rights reserved #********************************************************************************************* COLOR="echo -e \\033[01;31m" END='\033[0m' images=$(awk -F "/" '/image:/{print $NF}' recommended.yaml) HARBOR_DOMAIN=harbor.raymonds.cc images_download(){ ${COLOR}"开始下载Dashboard镜像"${END} for i in ${images};do nerdctl pull registry.aliyuncs.com/google_containers/$i nerdctl tag registry.aliyuncs.com/google_containers/$i ${HARBOR_DOMAIN}/google_containers/$i nerdctl rmi registry.aliyuncs.com/google_containers/$i nerdctl push ${HARBOR_DOMAIN}/google_containers/$i done ${COLOR}"Dashboard镜像下载完成"${END} } images_download [root@k8s-master01 ~]# bash download_dashboard_images.sh [root@k8s-master01 ~]# nerdctl images |grep -E "(dashboard|metrics-scraper)" harbor.raymonds.cc/google_containers/dashboard v2.7.0 2e500d29e9d5 11 seconds ago linux/amd64 245.8 MiB 72.3 MiB harbor.raymonds.cc/google_containers/metrics-scraper v1.0.8 76049887f07a 2 seconds ago linux/amd64 41.8 MiB 18.8 MiB [root@k8s-master01 ~]# sed -ri 's@(.*image:) kubernetesui(/.*)@\1 harbor.raymonds.cc/google_containers\2@g' recommended.yaml [root@k8s-master01 ~]# grep "image:" recommended.yaml image: harbor.raymonds.cc/google_containers/dashboard:v2.7.0 image: harbor.raymonds.cc/google_containers/metrics-scraper:v1.0.8 [root@k8s-master01 ~]# kubectl create -f recommended.yaml

[root@k8s-master01 ~]# cat > admin.yaml <<EOF apiVersion: v1 kind: ServiceAccount metadata: name: admin-user namespace: kubernetes-dashboard --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: admin-user roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: admin-user namespace: kubernetes-dashboard EOF [root@k8s-master01 ~]# kubectl apply -f admin.yaml

--test-type --ignore-certificate-errors

[root@k8s-master01 ~]# kubectl get svc kubernetes-dashboard -n kubernetes-dashboard NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes-dashboard NodePort 10.108.134.101 <none> 443:30005/TCP 51s

[root@k8s-master01 ~]# kubectl -n kubernetes-dashboard create token admin-user eyJhbGciOiJSUzI1NiIsImtpZCI6IjdtYlhqaVhELW5yNHBKbmplcHNlQjYzWDZtNU1VWlJSV0NwWDFCQk41MkUifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNjY0MTAzNTc0LCJpYXQiOjE2NjQwOTk5NzQsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJhZG1pbi11c2VyIiwidWlkIjoiN2U3ZWU0YWYtMjhkNS00MzdkLWI0OTgtY2FkZmNkNGE3M2JjIn19LCJuYmYiOjE2NjQwOTk5NzQsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDphZG1pbi11c2VyIn0.VoCfYjT6yWvol_66yiFWkM2ryufknCHuay4t_oKF4G54aI6GdeHLe6OFp5EljvM0-gWcR7SJfZbimrT9sPYdOuyo0QZKqfSD0I73UTKMbWsAfusyf0eeF1PrKDMUb9337J22jhqDIbqYB_e2Yf77Le38mKcS7Cc_Bm1cBKeL15vLLTDwwn_uCsE0YGXWHAsEzrYAbzC7MnPyw2AfuzSvWVuk0DC7j7zlGtjMys07pq84oh2vU4sYV5TUo1lQ5Rug75bkWFjljuJ3wmN1WcNQoN6DGlOSMDGo_LN_wVOHdom7_5RRM7YR0td88FCGTDWsgmkdeNWrhVB7X0-MHLewEA

[root@k8s-master01 ~]# cp /etc/kubernetes/admin.kubeconfig kubeconfig [root@k8s-master01 ~]# vim kubeconfig ... #在最下面添加token token: eyJhbGciOiJSUzI1NiIsImtpZCI6IjdtYlhqaVhELW5yNHBKbmplcHNlQjYzWDZtNU1VWlJSV0NwWDFCQk41MkUifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNjY0MTAzNTc0LCJpYXQiOjE2NjQwOTk5NzQsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJhZG1pbi11c2VyIiwidWlkIjoiN2U3ZWU0YWYtMjhkNS00MzdkLWI0OTgtY2FkZmNkNGE3M2JjIn19LCJuYmYiOjE2NjQwOTk5NzQsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDphZG1pbi11c2VyIn0.VoCfYjT6yWvol_66yiFWkM2ryufknCHuay4t_oKF4G54aI6GdeHLe6OFp5EljvM0-gWcR7SJfZbimrT9sPYdOuyo0QZKqfSD0I73UTKMbWsAfusyf0eeF1PrKDMUb9337J22jhqDIbqYB_e2Yf77Le38mKcS7Cc_Bm1cBKeL15vLLTDwwn_uCsE0YGXWHAsEzrYAbzC7MnPyw2AfuzSvWVuk0DC7j7zlGtjMys07pq84oh2vU4sYV5TUo1lQ5Rug75bkWFjljuJ3wmN1WcNQoN6DGlOSMDGo_LN_wVOHdom7_5RRM7YR0td88FCGTDWsgmkdeNWrhVB7X0-MHLewEA

cat <<EOF | kubectl apply -f - apiVersion: v1 kind: Pod metadata: name: busybox namespace: default spec: containers: - name: busybox image: busybox:1.28 command: - sleep - "3600" imagePullPolicy: IfNotPresent restartPolicy: Always EOF [root@k8s-master01 ~]# kubectl get pod NAME READY STATUS RESTARTS AGE busybox 1/1 Running 0 2m20s

[root@k8s-master01 ~]# kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 3h9m #Pod必须能解析Service [root@k8s-master01 ~]# kubectl exec busybox -n default -- nslookup kubernetes Server: 10.96.0.10 Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local Name: kubernetes Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local #Pod必须能解析跨namespace的Service [root@k8s-master01 ~]# kubectl exec busybox -n default -- nslookup kube-dns.kube-system Server: 10.96.0.10 Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local Name: kube-dns.kube-system Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

[root@k8s-master01 ~]# telnet 10.96.0.1 443 Trying 10.96.0.1... Connected to 10.96.0.1. Escape character is '^]'. ^CConnection closed by foreign host. [root@k8s-master02 ~]# telnet 10.96.0.1 443 [root@k8s-master03 ~]# telnet 10.96.0.1 443 [root@k8s-node01 ~]# telnet 10.96.0.1 443 [root@k8s-node02 ~]# telnet 10.96.0.1 443 [root@k8s-node03 ~]# telnet 10.96.0.1 443 [root@k8s-master01 ~]# kubectl get svc -n kube-system NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 19m metrics-server ClusterIP 10.110.185.15 <none> 443/TCP 16m [root@k8s-master01 ~]# telnet 10.96.0.10 53 Trying 10.96.0.10... Connected to 10.96.0.10. Escape character is '^]'. ^CConnection closed by foreign host. [root@k8s-master02 ~]# telnet 10.96.0.10 53 [root@k8s-master03 ~]# telnet 10.96.0.10 53 [root@k8s-node01 ~]# telnet 10.96.0.10 53 [root@k8s-node02 ~]# telnet 10.96.0.10 53 [root@k8s-node03 ~]# telnet 10.96.0.10 53 [root@k8s-master01 ~]# curl 10.96.0.10:53 curl: (52) Empty reply from server [root@k8s-master02 ~]# curl 10.96.0.10:53 [root@k8s-master03 ~]# curl 10.96.0.10:53 [root@k8s-node01 ~]# curl 10.96.0.10:53 [root@k8s-node02 ~]# curl 10.96.0.10:53 [root@k8s-node03 ~]# curl 10.96.0.10:53

[root@k8s-master01 ~]# kubectl get pod -n kube-system -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES calico-kube-controllers-67bd695c56-zt5k8 1/1 Running 0 32m 172.31.3.101 172.31.3.101 <none> <none> calico-node-fm5x2 1/1 Running 0 32m 172.31.3.111 172.31.3.111 <none> <none> calico-node-gbhd2 1/1 Running 0 32m 172.31.3.103 172.31.3.103 <none> <none> calico-node-m7gcc 1/1 Running 0 32m 172.31.3.101 172.31.3.101 <none> <none> calico-node-nnbjq 1/1 Running 0 32m 172.31.3.112 172.31.3.112 <none> <none> calico-node-v9cgw 1/1 Running 0 32m 172.31.3.102 172.31.3.102 <none> <none> calico-node-zlmx6 1/1 Running 0 32m 172.31.3.113 172.31.3.113 <none> <none> coredns-8668f8476d-njcl2 1/1 Running 0 21m 192.171.30.65 172.31.3.102 <none> <none> metrics-server-785dd7cc54-48xhq 1/1 Running 0 18m 192.170.21.193 172.31.3.113 <none> <none> [root@k8s-master01 ~]# kubectl get pod -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES busybox 1/1 Running 0 5m35s 192.167.195.129 172.31.3.112 <none> <none> [root@k8s-master01 ~]# kubectl exec -it busybox -- sh / # ping 192.170.21.193 PING 192.170.21.193 (192.170.21.193): 56 data bytes 64 bytes from 192.170.21.193: seq=0 ttl=62 time=1.323 ms 64 bytes from 192.170.21.193: seq=1 ttl=62 time=0.643 ms ^C --- 192.170.21.193 ping statistics --- 2 packets transmitted, 2 packets received, 0% packet loss round-trip min/avg/max = 0.643/0.983/1.323 ms / # exit [root@k8s-master01 ~]# kubectl create deploy nginx --image=nginx --replicas=3 deployment.apps/nginx created [root@k8s-master01 ~]# kubectl get deploy NAME READY UP-TO-DATE AVAILABLE AGE nginx 3/3 3 3 93s [root@k8s-master01 ~]# kubectl get pod -o wide |grep nginx nginx-76d6c9b8c-4kcv7 1/1 Running 0 3m29s 192.165.109.66 172.31.3.103 <none> <none> nginx-76d6c9b8c-b87sx 1/1 Running 0 3m29s 192.170.21.194 172.31.3.113 <none> <none> nginx-76d6c9b8c-p5vzc 1/1 Running 0 3m29s 192.169.111.130 172.31.3.111 <none> <none> [root@k8s-master01 ~]# kubectl delete deploy nginx deployment.apps "nginx" deleted [root@k8s-master01 ~]# kubectl delete pod busybox pod "busybox" deleted

vim /etc/docker/daemon.json { "registry-mirrors": [ #docker镜像加速 "https://registry.docker-cn.com", "http://hub-mirror.c.163.com", "https://docker.mirrors.ustc.edu.cn" ], "exec-opts": ["native.cgroupdriver=systemd"], #k8s需要systemd启动docker "max-concurrent-downloads": 10, #并发下载线程数 "max-concurrent-uploads": 5, #并发上传线程数 "log-opts": { "max-size": "300m", #docker日志文件最大300m "max-file": "2" #最大2个文件 }, "live-restore": true #docker服务重启，容器不会重启 }

[root@k8s-master01 ~]# vim /lib/systemd/system/kube-controller-manager.service # --feature-gates=RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true \ #这个是bootstrap自动颁发证书，新版默认就是true，不用设置 --cluster-signing-duration=876000h0m0s \ #用来控制签发证书的有效期限。 [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03; do scp -o StrictHostKeyChecking=no /lib/systemd/system/kube-controller-manager.service $NODE:/lib/systemd/system/; done kube-controller-manager.service 100% 1113 670.4KB/s 00:00 kube-controller-manager.service 100% 1113 1.0MB/s 00:00 [root@k8s-master01 ~]# systemctl daemon-reload && systemctl restart kube-controller-manager [root@k8s-master02 ~]# systemctl daemon-reload && systemctl restart kube-controller-manager [root@k8s-master03 ~]# systemctl daemon-reload && systemctl restart kube-controller-manager

[root@k8s-master01 ~]# vim /etc/systemd/system/kubelet.service.d/10-kubelet.conf Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node='' --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --image-pull-progress-deadline=30m" #--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 #更改k8s的加密算法 #--image-pull-progress-deadline=30m #如果在该参数值所设置的期限之前没有拉取镜像的进展，镜像拉取操作将被取消。仅当容器运行环境设置为 docker 时，此特定于 docker 的参数才有效。 [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02 k8s-node03; do scp -o StrictHostKeyChecking=no /etc/systemd/system/kubelet.service.d/10-kubelet.conf $NODE:/etc/systemd/system/kubelet.service.d/ ;done

[root@k8s-master01 ~]# vim /etc/kubernetes/kubelet-conf.yml #添加如下配置 rotateServerCertificates: true allowedUnsafeSysctls: #允许容器设置内核，有安全风险，根据实际需求设置 - "net.core*" - "net.ipv4.*" kubeReserved: #k8s预留资源 cpu: "1" memory: 1Gi ephemeral-storage: 10Gi systemReserved: #系统预留资源 cpu: "1" memory: 1Gi ephemeral-storage: 10Gi #rotateServerCertificates: true #当证书即将过期时自动从 kube-apiserver 请求新的证书进行轮换。要求启用 RotateKubeletServerCertificate 特性开关，以及对提交的 CertificateSigningRequest 对象进行批复（approve）操作。 [root@k8s-master01 ~]# for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02 k8s-node03; do scp /etc/kubernetes/kubelet-conf.yml $NODE:/etc/kubernetes/ ;done [root@k8s-master01 ~]# systemctl daemon-reload && systemctl restart kubelet [root@k8s-master02 ~]# systemctl daemon-reload && systemctl restart kubelet [root@k8s-master03 ~]# systemctl daemon-reload && systemctl restart kubelet [root@k8s-node01 ~]# systemctl daemon-reload && systemctl restart kubelet [root@k8s-node02 ~]# systemctl daemon-reload && systemctl restart kubelet [root@k8s-node03 ~]# systemctl daemon-reload && systemctl restart kubelet

[root@k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 172.31.3.101 Ready <none> 151m v1.25.2 172.31.3.102 Ready <none> 150m v1.25.2 172.31.3.103 Ready <none> 150m v1.25.2 172.31.3.111 Ready <none> 124m v1.25.2 172.31.3.112 Ready <none> 124m v1.25.2 172.31.3.113 Ready <none> 124m v1.25.2 [root@k8s-master01 ~]# kubectl get node --show-labels NAME STATUS ROLES AGE VERSION LABELS 172.31.3.101 NotReady <none> 63m v1.25.2 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.31.3.101,kubernetes.io/os=linux,node.kubernetes.io/node= 172.31.3.102 Ready <none> 63m v1.25.2 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.31.3.102,kubernetes.io/os=linux,node.kubernetes.io/node= 172.31.3.103 Ready <none> 63m v1.25.2 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.31.3.103,kubernetes.io/os=linux,node.kubernetes.io/node= 172.31.3.111 Ready <none> 55m v1.25.2 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.31.3.111,kubernetes.io/os=linux,node.kubernetes.io/node= 172.31.3.112 Ready <none> 54m v1.25.2 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.31.3.112,kubernetes.io/os=linux,node.kubernetes.io/node= 172.31.3.113 Ready <none> 54m v1.25.2 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.31.3.113,kubernetes.io/os=linux,node.kubernetes.io/node= [root@k8s-master01 ~]# kubectl label node 172.31.3.101 node-role.kubernetes.io/control-plane='' node-role.kubernetes.io/master='' node/172.31.3.101 labeled [root@k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 172.31.3.101 Ready control-plane,master 152m v1.25.2 172.31.3.102 Ready <none> 151m v1.25.2 172.31.3.103 Ready <none> 151m v1.25.2 172.31.3.111 Ready <none> 125m v1.25.2 172.31.3.112 Ready <none> 125m v1.25.2 172.31.3.113 Ready <none> 125m v1.25.2 [root@k8s-master01 ~]# kubectl label node 172.31.3.102 node-role.kubernetes.io/control-plane='' node-role.kubernetes.io/master='' node/172.31.3.102 labeled [root@k8s-master01 ~]# kubectl label node 172.31.3.103 node-role.kubernetes.io/control-plane='' node-role.kubernetes.io/master='' node/172.31.3.103 labeled [root@k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 172.31.3.101 Ready control-plane,master 152m v1.25.2 172.31.3.102 Ready control-plane,master 152m v1.25.2 172.31.3.103 Ready control-plane,master 151m v1.25.2 172.31.3.111 Ready <none> 125m v1.25.2 172.31.3.112 Ready <none> 125m v1.25.2 172.31.3.113 Ready <none> 125m v1.25.2