1 环境准备
1.1 ip划分
#ip地址划分:
easzlab-deploy 10.0.0.120 2c 4G
easzlab-haproxy-keepalive-01 10.0.0.121 2c 4G
easzlab-haproxy-keepalive-02 10.0.0.122 2c 4G
easzlab-haproxy-keepalive-03 10.0.0.123 2c 4G
easzlab-k8s-master-01 10.0.0.114 4c 8G
easzlab-k8s-master-02 10.0.0.115 4c 8G
easzlab-k8s-master-03 10.0.0.116 4c 8G
easzlab-k8s-ceph-01 10.0.0.124 4c 8G
easzlab-k8s-ceph-02 10.0.0.125 4c 8G
easzlab-k8s-ceph-03 10.0.0.126 4c 8G
easzlab-k8s-node-01 10.0.0.117 8c 16G
easzlab-k8s-node-02 10.0.0.118 8c 16G
easzlab-k8s-node-03 10.0.0.119 8c 16G
easzlab-k8s-etcd-01 10.0.0.111 2c 4G
easzlab-k8s-etcd-02 10.0.0.112 2c 4G
easzlab-k8s-etcd-03 10.0.0.113 2c 4G
easzlab-k8s-harbor-01 10.0.0.127 4c 8G
easzlab-k8s-harbor-02 10.0.0.128 4c 8G
easzlab-k8s-harbor-03 10.0.0.129 4c 8G1.2 准备Ubuntu2004克隆模板
#运行set.sh脚本初始化环境
#!/bin/bash
#
#********************************************************************
#Author: shuhong
#QQ: 985347841
#Date: 2022-09-01
#FileName: set.sh
#URL: hhhhh
#Description: The test script
#Copyright (C): 2022 All rights reserved
#********************************************************************
#read -p "请输入地址:" ip
network(){
sed -i -r "s@(^[[:space:]]+- 10.0.0.)[0-9]+@\1$1@" /etc/netplan/00-installer-config.yaml
netplan apply
}
newset(){
#添加apt源
cat > /etc/apt/sources.list <<EOF
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-updates main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-backports main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-security main restricted universe multiverse
# 预发布软件源，不建议启用
# deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse
EOF
#安装所需要的包
apt-get update && apt-get install -y wget curl chrony net-tools python3 tcpdump sshpass
#配置时间同步
cat > /etc/chrony.conf <<EOF
server ntp.aliyun.com iburst
stratumweight 0
driftfile /var/lib/chrony/drift
rtcsync
makestep 10 3
bindcmdaddress 127.0.0.1
bindcmdaddress ::1
keyfile /etc/chrony.keys
commandkey 1
generatecommandkey
logchange 0.5
logdir /var/log/chrony
EOF
#创建python软链接
rm /usr/bin/python -rf
ln -s /usr/bin/python3.8 /usr/bin/python
#启动chronyd服务
systemctl enable chrony && systemctl start chrony
#关闭防火墙
systemctl stop ufw.service && systemctl disable ufw.service
#设置时区
timedatectl set-timezone "Asia/Shanghai"
echo -n "LANG=en_US.UTF-8
LC_TIME=en_DK.UTF-8
" > /etc/default/locale
#清清除iptables防火墙规则
iptables -F && iptables -X && iptables -Z
reboot
}
network $ip
newset1.3 克隆集群
#运行脚本
#!/bin/bash
#
#********************************************************************
#Author: shuhong
#QQ: 985347841
#Date: 2022-09-01
#FileName: clone.sh
#URL: hhhhh
#Description: The test script
#Copyright (C): 2022 All rights reserved
#********************************************************************
#read -p "请输入刻录机器:" base
read -p "请输入集群名称:" name
read -p "请输入节点数:" node
for ((i=0;i<$node;i++)) ;do
let n=$i+1
arg[$i]="$name-0$n"
virt-clone -o ubuntu20.04-template -n ${arg[$i]} -f /var/lib/libvirt/images/${arg[$i]}.qcow2
done
#按IP划分批量修改IP
#!/bin/bash
#
#********************************************************************
#Author: shuhong
#QQ: 985347841
#Date: 2022-09-01
#FileName: network .sh
#URL: hhhhh
#Description: The test script
#Copyright (C): 2022 All rights reserved
#********************************************************************
num=110
for i in virsh list --all|awk '/eas.*/{print $2}';do
echo $i
let num=$num+1
virsh start $i || continue
sleep 60
ssh root@10.0.0.100 "sed -i -r 's@(^[[:space:]]+- 10.0.0.)[0-9]+@\1$num@' /etc/netplan/00-installer-config.yaml;reboot"
done
1.3 修改主机名称
#进入部署机器easzlab-deploy-01,安装ansible
[root@easzlab-deploy-01 ~]#apt-get -y install ansible
#部署机器对所有受控机分发公钥实现免密登录
for i in {111..129};do sshpass -p 'redhat' ssh-copy-id -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa -p 22 root@10.0.0.$i;done
#用ansible 修改主机名
[root@easzlab-deploy-01 ~]#vim /etc/ansible/hosts
[vm1]
10.0.0.120 hostname=easzlab-deploy-01 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.114 hostname=easzlab-k8s-master-01 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.115 hostname=easzlab-k8s-master-02 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.116 hostname=easzlab-k8s-master-03 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.117 hostname=easzlab-k8s-node-01 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.118 hostname=easzlab-k8s-node-03 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.119 hostname=easzlab-k8s-node-03 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.111 hostname=easzlab-k8s-etcd-01 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.112 hostname=easzlab-k8s-etcd-02 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.113 hostname=easzlab-k8s-etcd-03 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.127 hostname=easzlab-k8s-harbor-01 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.128 hostname=easzlab-k8s-harbor-02 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.129 hostname=easzlab-k8s-harbor-03 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.124 hostname=easzlab-k8s-ceph-01 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.125 hostname=easzlab-k8s-ceph-02 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.126 hostname=easzlab-k8s-ceph-03 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.121 hostname=easzlab-haproxy-keepalive-01 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.122 hostname=easzlab-haproxy-keepalive-02 ansible_ssh_port=22 ansible_ssh_pass=redhat
10.0.0.123 hostname=easzlab-haproxy-keepalive-03 ansible_ssh_port=22 ansible_ssh_pass=redhat
[vm]
10.0.0.[111:129]
[root@easzlab-deploy-01 ~]#ansible-playbook name.yaml
#脚本和ansible修改机器的hosts文件
[root@easzlab-deploy-01 ~]#vim add-hosts.sh
cat > /etc/hosts <<EOF
127.0.0.1 localhost
127.0.1.1 magedu
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
10.0.0.120 easzlab-deploy-01
10.0.0.114 easzlab-k8s-master-01
10.0.0.115 easzlab-k8s-master-02
10.0.0.116 easzlab-k8s-master-03
10.0.0.117 easzlab-k8s-node-01
10.0.0.118 easzlab-k8s-node-02
10.0.0.119 easzlab-k8s-node-03
10.0.0.111 easzlab-k8s-etcd-01
10.0.0.112 easzlab-k8s-etcd-02
10.0.0.113 easzlab-k8s-etcd-03
10.0.0.127 easzlab-k8s-harbor-01
10.0.0.128 easzlab-k8s-harbor-02
10.0.0.129 easzlab-k8s-harbor-03
10.0.0.124 easzlab-k8s-ceph-01
10.0.0.125 easzlab-k8s-ceph-02
10.0.0.126 easzlab-k8s-ceph-03
10.0.0.121 easzlab-haproxy-keepalive-01
10.0.0.122 easzlab-haproxy-keepalive-02
10.0.0.123 easzlab-haproxy-keepalive-03
EOF
[root@easzlab-deploy-01 ~]#ansible 'vm' -m script -a "./add-hosts.sh"
#检查ip及主机名
[root@easzlab-deploy-01 yaml-files]#for i in 10.0.0.{111..129};do ssh root@$i "hostname;hostname -I" ;done
easzlab-k8s-etcd-01
10.0.0.111
easzlab-k8s-etcd-02
10.0.0.112
easzlab-k8s-etcd-03
10.0.0.113
easzlab-k8s-master-01
10.0.0.114
easzlab-k8s-master-02
10.0.0.115
easzlab-k8s-master-03
10.0.0.116
easzlab-k8s-node-01
10.0.0.117
easzlab-k8s-node-03
10.0.0.118
easzlab-k8s-node-03
10.0.0.119
easzlab-deploy-01
10.0.0.120
easzlab-haproxy-keepalive-01
10.0.0.121
easzlab-haproxy-keepalive-02
10.0.0.122
easzlab-haproxy-keepalive-03
10.0.0.123
easzlab-k8s-ceph-01
10.0.0.124
easzlab-k8s-ceph-02
10.0.0.125
easzlab-k8s-ceph-03
10.0.0.126
easzlab-k8s-harbor-01
10.0.0.127
easzlab-k8s-harbor-02
10.0.0.128
easzlab-k8s-harbor-03
10.0.0.129
2 配置haproxy+keepalived高可用
2.1 安装配置keepalived
#在ansible的hosts配置文件中添加分组
[root@easzlab-deploy-01 ~]#vim /etc/ansible/hosts
[harbor]
10.0.0.[127:128]
[haproxy-keepalive]
10.0.0.[121:123]
[ceph]
10.0.0.[124:126]
[etcd]
10.0.0.[111:113]
[node]
10.0.0.[117:119]
[master]
10.0.0.[114:116]
#写安装yaml文件
[root@easzlab-deploy-01 yaml-files]#vim install.yaml
---
- name: install haproxy keepalived
hosts: haproxy-keepalive
tasks:
- name: haproxy keepalived
apt:
name:
- haproxy
- keepalived
state: present
[root@easzlab-deploy-01 yaml-files]#ansible-playbook install.yaml
[WARNING]: Invalid characters were found in group names but not replaced, use -vvvv to see details
PLAY [install haproxy keepalived] *****************************************************************************************************
TASK [Gathering Facts] ****************************************************************************************************************
ok: [10.0.0.123]
ok: [10.0.0.121]
ok: [10.0.0.122]
TASK [haproxy keepalived] *************************************************************************************************************
changed: [10.0.0.123]
changed: [10.0.0.122]
changed: [10.0.0.121]
PLAY RECAP ****************************************************************************************************************************
10.0.0.121 : ok=2 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
10.0.0.122 : ok=2 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
10.0.0.123 : ok=2 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
[root@easzlab-deploy-01 yaml-files]#ansible 'haproxy-keepalive' -m shell -a "cp /usived/samples/keepalived.conf.vrrp /etc/keepalived/keepalived.conf"
[WARNING]: Invalid characters were found in group names but not replaced, use -vvvv
10.0.0.121 | CHANGED | rc=0 >>
10.0.0.122 | CHANGED | rc=0 >>
10.0.0.123 | CHANGED | rc=0 >>
[root@easzlab-deploy-01 yaml-files]#for i in 10.0.0.{121..123};do ssh root@$i "hostname;ls /etc/keepalived/keepalived.conf" ;done
easzlab-haproxy-keepalive-01
/etc/keepalived/keepalived.conf
easzlab-haproxy-keepalive-02
/etc/keepalived/keepalived.conf
easzlab-haproxy-keepalive-03
/etc/keepalived/keepalived.conf2.1.1 配置master节点,easzlab-haproxy-keepalive-01
[root@easzlab-haproxy-keepalive-01 haproxy]#vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id easzlab-lvs
}
vrrp_script check_haproxy {
script "/etc/keepalived/check_haproxy.sh"
interval 1
weight -30
fall 3
rise 2
timeout 2
}
vrrp_instance VI_1 {
state MASTER
interface enp1s0
garp_master_delay 10
smtp_alert
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass easzlab111
}
virtual_ipaddress {
10.0.0.200 label enp1s0:1
}
track_script {
check_haproxy
}
}2.1.2 backup-01节点配置,easzlab-haproxy-keepalive-02
[root@easzlab-haproxy-keepalive-02 ~]#vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id easzlab-lvs
}
vrrp_script check_haproxy {
script "/etc/keepalived/check_haproxy.sh"
interval 1
weight -30
fall 3
rise 2
timeout 2
}
vrrp_instance VI_1 {
state BACKUP
interface enp1s0
garp_master_delay 10
smtp_alert
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass easzlab111
}
virtual_ipaddress {
10.0.0.200 label enp1s0:1
}
track_script {
check_haproxy
}
}2.1.3 backup-02节点配置,easzlab-haproxy-keepalive-03
[root@easzlab-haproxy-keepalive-03 keepalived]#vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id easzlab-lvs
}
vrrp_script check_haproxy {
script "/etc/keepalived/check_haproxy.sh"
interval 1
weight -30
fall 3
rise 2
timeout 2
}
vrrp_instance VI_1 {
state BACKUP
interface enp1s0
garp_master_delay 10
smtp_alert
virtual_router_id 51
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass easzlab111
}
virtual_ipaddress {
10.0.0.200 label enp1s0:1
}
track_script {
check_haproxy
}
}2.1.5 服务检查脚本
#三台机器上创建脚本
[root@easzlab-haproxy-keepalive-03 keepalived]#vim check_haproxy.sh
#!/bin/bash
#
#********************************************************************
#Author: shuhong
#QQ: 985347841
#Date: 2022-09-03
#FileName: check_haproxy.sh
#URL: hhhhh
#Description: The test script
#Copyright (C): 2022 All rights reserved
#********************************************************************
/usr/bin/killall -0 haproxy || systemctl restart haproxy
#三塔机器启动keepalived服务
systemclt restart keepalived && systemctl enable keepalived 2.2 配置haproxy服务
#三台机器编写文件
[root@easzlab-haproxy-keepalive-03 ~]#vim /etc/haproxy/haproxy.cfg
###########全局配置#########
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
daemon
nbproc 1 #进程数量
maxconn 4096 #最大连接数
user haproxy #运行用户
group haproxy #运行组
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
# Default SSL material locations
ca-base /etc/ssl/certs
crt-base /etc/ssl/private
# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
########默认配置############
defaults
log global
mode http #默认模式{ tcp|http|health }
option httplog #日志类别,采用httplog
option dontlognull #不记录健康检查日志信息
retries 2 #2次连接失败不可用
# option forwardfor #后端服务获得真实ip
option httpclose #请求完毕后主动关闭http通道
option abortonclose #服务器负载很高，自动结束比较久的链接
maxconn 4096 #最大连接数
timeout connect 5m #连接超时
timeout client 1m #客户端超时
timeout server 31m #服务器超时
timeout check 10s #心跳检测超时
balance roundrobin #负载均衡方式，轮询
########统计页面配置########
listen stats
bind 10.0.0.200:1080
mode http
option httplog
log 127.0.0.1 local0 err
maxconn 10 #最大连接数
stats refresh 30s
stats uri /admin #状态页面 http//ip:1080/admin访问
stats realm Haproxy\ Statistics
stats auth admin:admin #用户和密码:admin
stats hide-version #隐藏版本信息
stats admin if TRUE #设置手工启动/禁用
#############K8S###############
listen k8s_api_nodes_6443
bind 10.0.0.200:6443
mode tcp
server easzlab-k8s-master-01 10.0.0.114:6443 check inter 2000 fall 3 rise 5
server easzlab-k8s-master-02 10.0.0.115:6443 check inter 2000 fall 3 rise 5
server easzlab-k8s-master-03 10.0.0.116:6443 check inter 2000 fall 3 rise 注意:等待后面k8s master节点部署成功后在启动haproxy服务
3 配置harbor高可用
3.1 下载harbor离线安装包
[root@easzlab-deploy-01 ~]#wget https://github.com/goharbor/harbor/releases/download/v2.5.3/harbor-offline-installer-v2.5.3.tgz
[root@easzlab-deploy-01 ~]#scp harbor-offline-installer-v2.5.3.tgz root@10.0.0.127:
[root@easzlab-deploy-01 ~]#scp harbor-offline-installer-v2.5.3.tgz root@10.0.0.128:
[root@easzlab-deploy-01 ~]#scp harbor-offline-installer-v2.5.3.tgz root@10.0.0.129:
[root@easzlab-k8s-harbor-01 ~]#mkdir /apps
[root@easzlab-k8s-harbor-01 ~]#tar xf harbor-offline-installer-v2.5.3.tgz -C /apps
[root@easzlab-k8s-harbor-01 ~]#cd /apps/harbor/
[root@easzlab-k8s-harbor-01 harbor]#cp harbor.yml.tmpl harbor.yml
[root@easzlab-k8s-harbor-01 harbor]#mkdir /apps/harbor/certs
[root@easzlab-k8s-harbor-01 harbor]#cd /apps/harbor/certs
[root@easzlab-k8s-harbor-01 harbor]#vim harbor.yml
root@easzlab-k8s-harbor-01:/apps/harbor# egrep "^$|^#|^[[:space:]]+#" -v harbor.yml
hostname: harbor.magedu.net
http:
port: 80
https:
port: 443
certificate: /apps/harbor/certs/magedu.net.crt
private_key: /apps/harbor/certs/magedu.net.key
harbor_admin_password: Harbor12345
database:
password: root123
max_idle_conns: 100
max_open_conns: 900
data_volume: /data
trivy:
ignore_unfixed: false
skip_update: false
offline_scan: false
insecure: false
jobservice:
max_job_workers: 10
notification:
webhook_job_max_retry: 10
chart:
absolute_url: disabled
log:
level: info
local:
rotate_count: 50
rotate_size: 200M
location: /var/log/harbor
_version: 2.5.0
proxy:
http_proxy:
https_proxy:
no_proxy:
components:
- core
- jobservice
- trivy
upload_purging:
enabled: true
age: 168h
interval: 24h
dryrun: false
root@easzlab-k8s-harbor-01:/apps/harbor#
3.2 创建CA证书
[root@easzlab-k8s-harbor-01 certs]#openssl genrsa -out ca.key 4096
Generating RSA private key, 4096 bit long modulus (2 primes)
..................................................................................................................................................................++++
......................................................................................................................................................................................++++
e is 65537 (0x010001)
[root@easzlab-k8s-harbor-01 certs]#openssl req -x509 -new -nodes -sha512 -days 3650 \
> -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=magedu.com" \
> -key ca.key \
> -out ca.crt
[root@easzlab-k8s-harbor-01 certs]#openssl genrsa -out magedu.net.key 4096
Generating RSA private key, 4096 bit long modulus (2 primes)
.............++++
..........................................................................................................................................................++++
e is 65537 (0x010001)
[root@easzlab-k8s-harbor-01 certs]#openssl req -sha512 -new \
> -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=magedu.net" \
> -key magedu.net.key \
> -out magedu.net.csr
[root@easzlab-k8s-harbor-01 certs]#cat > v3.ext <<-EOF
> authorityKeyIdentifier=keyid,issuer
> basicConstraints=CA:FALSE
> keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
> extendedKeyUsage = serverAuth
> subjectAltName = @alt_names
>
> [alt_names]
> DNS.1=magedu.com
> DNS.2=harbor.magedu.net
> DNS.3=harbor.magedu.local
> EOF
[root@easzlab-k8s-harbor-01 certs]#openssl x509 -req -sha512 -days 3650 \
> -extfile v3.ext \
> -CA ca.crt -CAkey ca.key -CAcreateserial \
> -in magedu.net.csr \
> -out magedu.net.crt
Signature ok
subject=C = CN, ST = Beijing, L = Beijing, O = example, OU = Personal, CN = magedu.net
Getting CA Private Key
[root@easzlab-k8s-harbor-01 certs]#ll -h
total 28K
drwxr-xr-x 2 root root 128 Sep 3 16:30 ./
drwxr-xr-x 3 root root 153 Sep 3 16:21 ../
-rw-r--r-- 1 root root 2.0K Sep 3 16:27 ca.crt
-rw------- 1 root root 3.2K Sep 3 16:26 ca.key
-rw-r--r-- 1 root root 41 Sep 3 16:30 ca.srl
-rw-r--r-- 1 root root 2.1K Sep 3 16:30 magedu.net.crt
-rw-r--r-- 1 root root 1.7K Sep 3 16:28 magedu.net.csr
-rw------- 1 root root 3.2K Sep 3 16:27 magedu.net.key
-rw-r--r-- 1 root root 279 Sep 3 16:29 v3.ext3.3 安装docker
#本地准备好的docker安装文件传给harbor节点
[root@easzlab-deploy-01 yaml-files]##scp docker-20.10.17-binary-install.tar.gz root@10.0.0.127:
docker-20.10.17-binary-install.tar.gz 100% 73MB 110.6MB/s 00:00
[root@easzlab-k8s-harbor-01 ~]#tar xf docker-20.10.17-binary-install.tar.gz -C docker/
[root@easzlab-k8s-harbor-01 docker]#bash ./docker-install.sh 3.4 安装habor服务
[root@easzlab-k8s-harbor-01 harbor]#cd /apps/harbor/
[root@easzlab-k8s-harbor-01 harbor]#vim harbor.yml
hostname: easzlab-k8s-harbor-01
# https related config
#https:
# https port for harbor, default is 443
# port: 443
# The path of cert and key files for nginx
# certificate: /your/certificate/path
# private_key: /your/private/key/path
[root@easzlab-k8s-harbor-01 harbor]#./prepare
prepare base dir is set to /apps/harbor
WARNING:root:WARNING: HTTP protocol is insecure. Harbor will deprecate http protocol in the future. Please make sure to upgrade to https
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
Generated and saved secret to file: /data/secret/keys/secretkey
Successfully called func: create_root_cert
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
[root@easzlab-k8s-harbor-01 harbor]#./install.sh --with-trivy --with-chartmuseum
[Step 0]: checking if docker is installed ...
Note: docker version: 20.10.17
[Step 1]: checking docker-compose is installed ...
Note: docker-compose version: 1.28.6
[Step 2]: loading Harbor images ...
eb50d8bbd990: Loading layer [==================================================>] 7.668MB/7.668MB
04e75300c772: Loading layer [==================================================>] 7.362MB/7.362MB
e6830bb442bf: Loading layer [==================================================>] 1MB/1MB
Loaded image: goharbor/harbor-portal:v2.5.3
7e761f0c6325: Loading layer [==================================================>] 8.898MB/8.898MB
bea2d99bdd9a: Loading layer [==================================================>] 3.584kB/3.584kB
7635b8507a3f: Loading layer [==================================================>] 2.56kB/2.56kB
5374b1e2b14a: Loading layer [==================================================>] 78.75MB/78.75MB
3c111582434e: Loading layer [==================================================>] 5.632kB/5.632kB
c634a4d49b0c: Loading layer [==================================================>] 102.9kB/102.9kB
4edf106f0e4f: Loading layer [==================================================>] 15.87kB/15.87kB
732b0f7f2241: Loading layer [==================================================>] 79.66MB/79.66MB
8191a56b80ca: Loading layer [==================================================>] 2.56kB/2.56kB
Loaded image: goharbor/harbor-core:v2.5.3
005d5db57e06: Loading layer [==================================================>] 119.7MB/119.7MB
0e3d87aacbc9: Loading layer [==================================================>] 3.072kB/3.072kB
b4e26556ed44: Loading layer [==================================================>] 59.9kB/59.9kB
55f587609a73: Loading layer [==================================================>] 61.95kB/61.95kB
Loaded image: goharbor/redis-photon:v2.5.3
Loaded image: goharbor/prepare:v2.5.3
a86a26c0452a: Loading layer [==================================================>] 1.096MB/1.096MB
1025dfd257d2: Loading layer [==================================================>] 5.889MB/5.889MB
cd51e6d945dd: Loading layer [==================================================>] 168.8MB/168.8MB
c68c45fe177d: Loading layer [==================================================>] 16.58MB/16.58MB
fa18680022f9: Loading layer [==================================================>] 4.096kB/4.096kB
9f470cfcecff: Loading layer [==================================================>] 6.144kB/6.144kB
d9d256f40e6f: Loading layer [==================================================>] 3.072kB/3.072kB
f02862555d46: Loading layer [==================================================>] 2.048kB/2.048kB
8cc2449c3a33: Loading layer [==================================================>] 2.56kB/2.56kB
53e7545b8c1b: Loading layer [==================================================>] 2.56kB/2.56kB
62fbef76d294: Loading layer [==================================================>] 2.56kB/2.56kB
7e2d721c6c91: Loading layer [==================================================>] 8.704kB/8.704kB
Loaded image: goharbor/harbor-db:v2.5.3
7b5e699985f2: Loading layer [==================================================>] 5.755MB/5.755MB
17bb7303d841: Loading layer [==================================================>] 90.86MB/90.86MB
146be4872a18: Loading layer [==================================================>] 3.072kB/3.072kB
7f44df31c7df: Loading layer [==================================================>] 4.096kB/4.096kB
ec5f15201a56: Loading layer [==================================================>] 91.65MB/91.65MB
Loaded image: goharbor/chartmuseum-photon:v2.5.3
25ed0962037c: Loading layer [==================================================>] 8.898MB/8.898MB
96bf61ca4a6d: Loading layer [==================================================>] 3.584kB/3.584kB
faed05a35aaa: Loading layer [==================================================>] 2.56kB/2.56kB
6b2cce967e64: Loading layer [==================================================>] 90.8MB/90.8MB
47d73d2ec8c4: Loading layer [==================================================>] 91.59MB/91.59MB
Loaded image: goharbor/harbor-jobservice:v2.5.3
1b8a5b56dd8f: Loading layer [==================================================>] 5.755MB/5.755MB
ef6a1d16e324: Loading layer [==================================================>] 4.096kB/4.096kB
60cf083bf2b3: Loading layer [==================================================>] 17.34MB/17.34MB
54308a335bf1: Loading layer [==================================================>] 3.072kB/3.072kB
b507f0c5f1e1: Loading layer [==================================================>] 29.17MB/29.17MB
79b24972e581: Loading layer [==================================================>] 47.31MB/47.31MB
Loaded image: goharbor/harbor-registryctl:v2.5.3
35239a1e0d7a: Loading layer [==================================================>] 7.668MB/7.668MB
Loaded image: goharbor/nginx-photon:v2.5.3
e0776ca3d7c2: Loading layer [==================================================>] 5.75MB/5.75MB
c90a80564f89: Loading layer [==================================================>] 8.543MB/8.543MB
86c0504b8fcb: Loading layer [==================================================>] 14.47MB/14.47MB
abde74115d1a: Loading layer [==================================================>] 29.29MB/29.29MB
3ad37faaa958: Loading layer [==================================================>] 22.02kB/22.02kB
1d3c37158629: Loading layer [==================================================>] 14.47MB/14.47MB
Loaded image: goharbor/notary-signer-photon:v2.5.3
fa27c9d81dc3: Loading layer [==================================================>] 127MB/127MB
9ca66cb9252f: Loading layer [==================================================>] 3.584kB/3.584kB
09ce0e15f5ba: Loading layer [==================================================>] 3.072kB/3.072kB
d0ba49c5841f: Loading layer [==================================================>] 2.56kB/2.56kB
04623512f2e5: Loading layer [==================================================>] 3.072kB/3.072kB
083acf89058c: Loading layer [==================================================>] 3.584kB/3.584kB
5f2000f524c8: Loading layer [==================================================>] 20.99kB/20.99kB
Loaded image: goharbor/harbor-log:v2.5.3
425045210126: Loading layer [==================================================>] 8.898MB/8.898MB
a0ef3ff89e82: Loading layer [==================================================>] 21.05MB/21.05MB
7facb153a2bf: Loading layer [==================================================>] 4.608kB/4.608kB
ca36c2356dc0: Loading layer [==================================================>] 21.84MB/21.84MB
Loaded image: goharbor/harbor-exporter:v2.5.3
abd4886cf446: Loading layer [==================================================>] 5.755MB/5.755MB
a662294ced4c: Loading layer [==================================================>] 4.096kB/4.096kB
e1e02d95f798: Loading layer [==================================================>] 3.072kB/3.072kB
54535cb3135b: Loading layer [==================================================>] 17.34MB/17.34MB
a8556cd12eb5: Loading layer [==================================================>] 18.13MB/18.13MB
Loaded image: goharbor/registry-photon:v2.5.3
01427a3d3d67: Loading layer [==================================================>] 5.75MB/5.75MB
5cd7cb12cabb: Loading layer [==================================================>] 8.543MB/8.543MB
564dcad1be91: Loading layer [==================================================>] 15.88MB/15.88MB
b3020f432a85: Loading layer [==================================================>] 29.29MB/29.29MB
05bbf70fd214: Loading layer [==================================================>] 22.02kB/22.02kB
7cb2819f6977: Loading layer [==================================================>] 15.88MB/15.88MB
Loaded image: goharbor/notary-server-photon:v2.5.3
8cc02d219629: Loading layer [==================================================>] 6.283MB/6.283MB
09856854b73c: Loading layer [==================================================>] 4.096kB/4.096kB
c53bbce8e1c4: Loading layer [==================================================>] 3.072kB/3.072kB
ca0011850458: Loading layer [==================================================>] 91.21MB/91.21MB
0e7337dca995: Loading layer [==================================================>] 12.65MB/12.65MB
c1e6b3a22dfd: Loading layer [==================================================>] 104.6MB/104.6MB
Loaded image: goharbor/trivy-adapter-photon:v2.5.3
[Step 3]: preparing environment ...
[Step 4]: preparing harbor configs ...
prepare base dir is set to /apps/harbor
WARNING:root:WARNING: HTTP protocol is insecure. Harbor will deprecate http protocol in the future. Please make sure to upgrade to https
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/log/rsyslog_docker.conf
Clearing the configuration file: /config/nginx/nginx.conf
Clearing the configuration file: /config/core/env
Clearing the configuration file: /config/core/app.conf
Clearing the configuration file: /config/registry/passwd
Clearing the configuration file: /config/registry/config.yml
Clearing the configuration file: /config/registryctl/env
Clearing the configuration file: /config/registryctl/config.yml
Clearing the configuration file: /config/db/env
Clearing the configuration file: /config/jobservice/env
Clearing the configuration file: /config/jobservice/config.yml
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
loaded secret from file: /data/secret/keys/secretkey
Generated configuration file: /config/trivy-adapter/env
Generated configuration file: /config/chartserver/env
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
[Step 5]: starting Harbor ...
Creating network "harbor_harbor" with the default driver
Creating network "harbor_harbor-chartmuseum" with the default driver
Creating harbor-log ... done
Creating registry ... done
Creating registryctl ... done
Creating harbor-portal ... done
Creating redis ... done
Creating chartmuseum ... done
Creating harbor-db ... done
Creating harbor-core ... done
Creating trivy-adapter ... done
Creating nginx ... done
Creating harbor-jobservice ... done
✔ ----Harbor has been installed and started successfully.----
3.5 执行docker-compose命令检查harbor服务是否启动成功
[root@easzlab-k8s-harbor-01 harbor]#docker-compose ps
Name Command State Ports
--------------------------------------------------------------------------------------------------------
chartmuseum ./docker-entrypoint.sh Up (healthy)
harbor-core /harbor/entrypoint.sh Up (healthy)
harbor-db /docker-entrypoint.sh 96 13 Up (healthy)
harbor-jobservice /harbor/entrypoint.sh Up (healthy)
harbor-log /bin/sh -c /usr/local/bin/ ... Up (healthy) 127.0.0.1:1514->10514/tcp
harbor-portal nginx -g daemon off; Up (healthy)
nginx nginx -g daemon off; Up (healthy) 0.0.0.0:80->8080/tcp,:::80->8080/tcp
redis redis-server /etc/redis.conf Up (healthy)
registry /home/harbor/entrypoint.sh Up (healthy)
registryctl /home/harbor/start.sh Up (healthy)
trivy-adapter /home/scanner/entrypoint.sh Up (healthy)
[root@easzlab-k8s-harbor-01 harbor]#ss -tnlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 4096 127.0.0.1:43023 0.0.0.0:* users:(("containerd",pid=4783,fd=15))
LISTEN 0 4096 0.0.0.0:80 0.0.0.0:* users:(("docker-proxy",pid=7609,fd=4))
LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=639,fd=13))
LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=713,fd=3))
LISTEN 0 4096 127.0.0.1:1514 0.0.0.0:* users:(("docker-proxy",pid=6560,fd=4))
LISTEN 0 4096 [::]:80 [::]:* users:(("docker-proxy",pid=7617,fd=4))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=713,fd=4)) 3.6 测试验证
3.7 虚机登录测试
[root@easzlab-deploy-01 harbor.magedu.net]#mkdir /etc/docker/certs.d/harbor.magedu.net -p
[root@easzlab-deploy-01 ~]#vim /etc/hosts
10.0.0.127 easzlab-k8s-harbor-01 harbor.magedu.net
[root@easzlab-k8s-harbor-01 certs]#cd /apps/harbor/certs/
[root@easzlab-k8s-harbor-01 certs]#scp magedu.net.crt root@10.0.0.120:/etc/docker/certs.d/harbor.magedu.net
[root@easzlab-deploy-01 harbor.magedu.net]#docker login https://harbor.magedu.net
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded4 安装easzlab
4.1 下载安装easzlab安装包
#推送公钥到他其他节点
#生成公钥：ssh-keygen
[root@easzlab-deploy-01 ~]#vim ssh-key.sh
#!/bin/bash
#
#********************************************************************
#Author: shuhong
#QQ: 985347841
#Date: 2022-09-03
#FileName: ssh-key.sh
#URL: hhhhh
#Description: The test script
#Copyright (C): 2022 All rights reserved
#********************************************************************
for i in {111..129};
do
sshpass -p 'redhat' ssh-copy-id -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa -p 22 root@10.0.0.$i;
done
[root@easzlab-deploy-01 ~]#./ssh-key.sh
#下载工具脚本ezdown，举例使用kubeasz版本3.3.1
[root@easzlab-deploy-01 yaml-files]#export release=3.3.1
[root@easzlab-deploy-01 yaml-files]#wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
[root@easzlab-deploy-01 yaml-files]#chmod +x ./ezdown
#下载kubeasz代码、二进制、默认容器镜像（更多关于ezdown的参数，运行./ezdown 查看）
# 海外环境
/ezdown -D -m standard
# 国内环境
[root@easzlab-deploy-01 yaml-files]#./ezdown -D
2022-09-03 19:21:50 INFO Action begin: download_all
2022-09-03 19:21:50 INFO downloading docker binaries, version 20.10.16
--2022-09-03 19:21:50-- https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/static/stable/x86_64/docker-20.10.16.tgz
Resolving mirrors.tuna.tsinghua.edu.cn (mirrors.tuna.tsinghua.edu.cn)... 101.6.15.130, 2402:f000:1:400::2
Connecting to mirrors.tuna.tsinghua.edu.cn (mirrors.tuna.tsinghua.edu.cn)|101.6.15.130|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 64969189 (62M) [application/octet-stream]
Saving to: ‘docker-20.10.16.tgz’
docker-20.10.16.tgz 100%[=======================================================>] 61.96M 9.15MB/s in 6.4s
2022-09-03 19:22:03 (9.63 MB/s) - ‘docker-20.10.16.tgz’ saved [64969189/64969189]
2022-09-03 19:22:06 DEBUG generate docker service file
2022-09-03 19:22:06 DEBUG generate docker config: /etc/docker/daemon.json
2022-09-03 19:22:06 DEBUG prepare register mirror for CN
2022-09-03 19:22:06 DEBUG enable and start docker
Removed /etc/systemd/system/multi-user.target.wants/docker.service.
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /etc/systemd/system/docker.service.
2022-09-03 19:22:11 INFO downloading kubeasz: 3.3.1
2022-09-03 19:22:12 DEBUG run a temporary container
Unable to find image 'easzlab/kubeasz:3.3.1' locally
3.3.1: Pulling from easzlab/kubeasz
540db60ca938: Pull complete
d037ddac5dde: Pull complete
05d0edf52df4: Pull complete
54d94e388fb8: Pull complete
b25964b87dc1: Pull complete
aedfadb13329: Pull complete
8f6f8140f32b: Pull complete
Digest: sha256:c0cfc314c4caea45a7582a5e03b090901177c4c48210c3df8b209f5b03045f70
Status: Downloaded newer image for easzlab/kubeasz:3.3.1
564317e0ca49c5cd7efd6cb2a4b49343898a611d34da40249a4449b427e64771
2022-09-03 19:22:31 DEBUG cp kubeasz code from the temporary container
2022-09-03 19:22:32 DEBUG stop&remove temporary container
temp_easz
2022-09-03 19:22:32 INFO downloading kubernetes: v1.24.2 binaries
v1.24.2: Pulling from easzlab/kubeasz-k8s-bin
1b7ca6aea1dd: Pull complete
d2339c028cfd: Pull complete
Digest: sha256:1a41943faa18d7a69e243f4cd9b7b6f1cd7268be7c6358587170c3d3e9e1a34c
Status: Downloaded newer image for easzlab/kubeasz-k8s-bin:v1.24.2
docker.io/easzlab/kubeasz-k8s-bin:v1.24.2
2022-09-03 19:23:03 DEBUG run a temporary container
708c9b16e92aec069aabc9285e83f060c2bff39e907cf141adf57ffc1c4a51f2
2022-09-03 19:23:05 DEBUG cp k8s binaries
2022-09-03 19:23:10 DEBUG stop&remove temporary container
temp_k8s_bin
2022-09-03 19:23:12 INFO downloading extral binaries kubeasz-ext-bin:1.2.0
1.2.0: Pulling from easzlab/kubeasz-ext-bin
1b7ca6aea1dd: Already exists
4a494a9b7425: Pull complete
b11479c0b3c6: Pull complete
0351e344774e: Pull complete
1c1e5d29db2d: Pull complete
Digest: sha256:a40f30978cca518503811db70ec7734b98ab4378a5c06546bf22de37900f252d
Status: Downloaded newer image for easzlab/kubeasz-ext-bin:1.2.0
docker.io/easzlab/kubeasz-ext-bin:1.2.0
2022-09-03 19:23:58 DEBUG run a temporary container
513f6273d9b5dd0e49c148e04662483a0b46eaf1731eea79e713a6219193971f
2022-09-03 19:24:01 DEBUG cp extral binaries
2022-09-03 19:24:06 DEBUG stop&remove temporary container
temp_ext_bin
2: Pulling from library/registry
213ec9aee27d: Pull complete
5299e6f78605: Pull complete
4c2fb79b7ce6: Pull complete
74a97d2d84d9: Pull complete
44c4c74a95e4: Pull complete
Digest: sha256:83bb78d7b28f1ac99c68133af32c93e9a1c149bcd3cb6e683a3ee56e312f1c96
Status: Downloaded newer image for registry:2
docker.io/library/registry:2
2022-09-03 19:24:20 INFO start local registry ...
cb35d6a7e965349338b83b362bd20872af44b28be197e0fbcc5774704e27bafe
2022-09-03 19:24:22 INFO download default images, then upload to the local registry
v3.19.4: Pulling from calico/cni
f3894d312a4e: Pull complete
8244094b678e: Pull complete
45b915a54b66: Pull complete
Digest: sha256:a866562105d3c18486879d313830d8b4918e8ba25ccd23b7dd84d65093d03c62
Status: Downloaded newer image for calico/cni:v3.19.4
docker.io/calico/cni:v3.19.4
v3.19.4: Pulling from calico/pod2daemon-flexvol
99aa522a8a66: Pull complete
beb35b03ed9b: Pull complete
8c61f8de6c67: Pull complete
622403455de3: Pull complete
a26eec45c530: Pull complete
b02e2914a61e: Pull complete
91f16e6ede78: Pull complete
Digest: sha256:d698fbda7a2e895ad45b478ab0b5fdd572cd80629e558dbfcf6e401c6ee6275e
Status: Downloaded newer image for calico/pod2daemon-flexvol:v3.19.4
docker.io/calico/pod2daemon-flexvol:v3.19.4
v3.19.4: Pulling from calico/kube-controllers
0a1506fb14ea: Pull complete
6abc1e849f8f: Pull complete
0cfea6002588: Pull complete
91d785239eb0: Pull complete
Digest: sha256:b15521e60d8bb04a501fe0ef4bf791fc8c164a175dd49a2328fb3f2b89838a68
Status: Downloaded newer image for calico/kube-controllers:v3.19.4
docker.io/calico/kube-controllers:v3.19.4
v3.19.4: Pulling from calico/node
7563b432e373: Pull complete
f1ad2d4094a4: Pull complete
Digest: sha256:df027832d91944516046f6baf3f6e74c5130046d2c56f88dc96296681771bc6a
Status: Downloaded newer image for calico/node:v3.19.4
docker.io/calico/node:v3.19.4
The push refers to repository [easzlab.io.local:5000/calico/cni]
e190560973d0: Pushed
237eb7dff52b: Pushed
7bdb7ca6a5a4: Pushed
v3.19.4: digest: sha256:9e1da653e987232cf18df3eb6967c9555a1235d212189b3e4c26f6f9d1601297 size: 946
The push refers to repository [easzlab.io.local:5000/calico/pod2daemon-flexvol]
0312eef4fc3a: Pushed
aeeffe0f6b8b: Pushed
672e236e33e9: Pushed
e5816bd252f3: Pushed
e29ee4bf6f3f: Pushed
9dd9977906c2: Pushed
cdc78476cc38: Pushed
v3.19.4: digest: sha256:152415638f6cc10fcbc2095069c5286df262c591422fb2608a14c7eee554c259 size: 1788
The push refers to repository [easzlab.io.local:5000/calico/kube-controllers]
568d0e1941e4: Pushed
7094539af214: Pushed
44bbcee30afb: Pushed
e47767779496: Pushed
v3.19.4: digest: sha256:214b5384028bac797ff16531d71d28f7d658ef3a26837db6bf5466bc5f113bfd size: 1155
The push refers to repository [easzlab.io.local:5000/calico/node]
f03078b73155: Pushed
14ec913b26f5: Pushed
v3.19.4: digest: sha256:393ff601623e04e685add605920e6c984a1ac74e23cc4232cec7f5013ba8caad size: 737
1.9.3: Pulling from coredns/coredns
d92bdee79785: Downloading
f2401d57212f: Downloading
1.9.3: Pulling from coredns/coredns
d92bdee79785: Pull complete
f2401d57212f: Pull complete
Digest: sha256:8e352a029d304ca7431c6507b56800636c321cb52289686a581ab70aaa8a2e2a
Status: Downloaded newer image for coredns/coredns:1.9.3
docker.io/coredns/coredns:1.9.3
The push refers to repository [easzlab.io.local:5000/coredns/coredns]
df1818f16337: Pushed
256bc5c338a6: Pushed
1.9.3: digest: sha256:bdb36ee882c13135669cfc2bb91c808a33926ad1a411fee07bd2dc344bb8f782 size: 739
1.21.1: Pulling from easzlab/k8s-dns-node-cache
20b09fbd3037: Pull complete
af833073aa95: Pull complete
Digest: sha256:04c4f6b1f2f2f72441dadcea1c8eec611af4d963315187ceb04b939d1956782f
Status: Downloaded newer image for easzlab/k8s-dns-node-cache:1.21.1
docker.io/easzlab/k8s-dns-node-cache:1.21.1
The push refers to repository [easzlab.io.local:5000/easzlab/k8s-dns-node-cache]
8391095a8344: Pushed
87b6a930c8d0: Pushed
1.21.1: digest: sha256:04c4f6b1f2f2f72441dadcea1c8eec611af4d963315187ceb04b939d1956782f size: 741
v2.5.1: Pulling from kubernetesui/dashboard
d1d01ae59b08: Pull complete
a25bff2a339f: Pull complete
Digest: sha256:cc746e7a0b1eec0db01cbabbb6386b23d7af97e79fa9e36bb883a95b7eb96fe2
Status: Downloaded newer image for kubernetesui/dashboard:v2.5.1
docker.io/kubernetesui/dashboard:v2.5.1
The push refers to repository [easzlab.io.local:5000/kubernetesui/dashboard]
e98b3744f758: Pushed
dab46c9f5775: Pushed
v2.5.1: digest: sha256:0c82e96241aa683fe2f8fbdf43530e22863ac8bfaddb0d7d30b4e3a639d4e8c5 size: 736
v1.0.8: Pulling from kubernetesui/metrics-scraper
978be80e3ee3: Pull complete
5866d2c04d96: Pull complete
Digest: sha256:76049887f07a0476dc93efc2d3569b9529bf982b22d29f356092ce206e98765c
Status: Downloaded newer image for kubernetesui/metrics-scraper:v1.0.8
docker.io/kubernetesui/metrics-scraper:v1.0.8
The push refers to repository [easzlab.io.local:5000/kubernetesui/metrics-scraper]
bcec7eb9e567: Pushed
d01384fea991: Pushed
v1.0.8: digest: sha256:43227e8286fd379ee0415a5e2156a9439c4056807e3caa38e1dd413b0644807a size: 736
v0.5.2: Pulling from easzlab/metrics-server
e8614d09b7be: Downloading
334ef31a5c43: Download complete
v0.5.2: Pulling from easzlab/metrics-server
e8614d09b7be: Pull complete
334ef31a5c43: Pull complete
Digest: sha256:6879d1d3e42c06fa383aed4988fc8f39901d46fb29d25a5b41c88f9d4b6854b1
Status: Downloaded newer image for easzlab/metrics-server:v0.5.2
docker.io/easzlab/metrics-server:v0.5.2
The push refers to repository [easzlab.io.local:5000/easzlab/metrics-server]
b2839a50be1a: Pushed
6d75f23be3dd: Pushed
v0.5.2: digest: sha256:6879d1d3e42c06fa383aed4988fc8f39901d46fb29d25a5b41c88f9d4b6854b1 size: 739
3.7: Pulling from easzlab/pause
7582c2cc65ef: Pull complete
Digest: sha256:445a99db22e9add9bfb15ddb1980861a329e5dff5c88d7eec9cbf08b6b2f4eb1
Status: Downloaded newer image for easzlab/pause:3.7
docker.io/easzlab/pause:3.7
The push refers to repository [easzlab.io.local:5000/easzlab/pause]
1cb555415fd3: Pushed
3.7: digest: sha256:445a99db22e9add9bfb15ddb1980861a329e5dff5c88d7eec9cbf08b6b2f4eb1 size: 526
3.3.1: Pulling from easzlab/kubeasz
Digest: sha256:c0cfc314c4caea45a7582a5e03b090901177c4c48210c3df8b209f5b03045f70
Status: Image is up to date for easzlab/kubeasz:3.3.1
docker.io/easzlab/kubeasz:3.3.1
2022-09-03 19:32:55 INFO Action successed: download_all
4.2 创建集群配置实例
#容器化运行
[root@easzlab-deploy-01 yaml-files]#./ezdown -S
2022-09-03 19:34:05 INFO Action begin: start_kubeasz_docker
2022-09-03 19:34:05 INFO try to run kubeasz in a container
2022-09-03 19:34:05 DEBUG get host IP: 10.0.0.120
7ca8d9db78193144af113c8e896df3d564f3208b5ccc276c8201bfe8dab21183
2022-09-03 19:34:05 INFO Action successed: start_kubeasz_docker
#创建新集群k8s-01
[root@easzlab-deploy-01 ~]#docker exec -it kubeasz ezctl new k8s-01
2022-09-03 11:34:57 DEBUG generate custom cluster files in /etc/kubeasz/clusters/k8s-01
2022-09-03 11:34:57 DEBUG set versions
2022-09-03 11:34:57 DEBUG cluster k8s-01: files successfully created.
2022-09-03 11:34:57 INFO next steps 1: to config '/etc/kubeasz/clusters/k8s-01/hosts'
2022-09-03 11:34:57 INFO next steps 2: to config '/etc/kubeasz/clusters/k8s-01/config.yml'
[root@easzlab-deploy-01 ~]#ls /etc/kubeasz/
README.md ansible.cfg bin clusters docs down example ezctl ezdown manifests pics playbooks roles tools
[root@easzlab-deploy-01 ~]#ls /etc/kubeasz/manifests/
efk es-cluster ingress jenkins mariadb-cluster mysql-cluster redis-cluster storage
[root@easzlab-deploy-01 ~]#ls /etc/kubeasz/clusters/
k8s-01
[root@easzlab-deploy-01 ~]#ls /etc/kubeasz/clusters/k8s-01/
config.yml hosts
4.3 修改config
4.4 修改hosts
4.5 安装
[root@easzlab-deploy-01 kubeasz]#ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml playbooks/01.prepare.yml
PLAY [kube_master,kube_node,etcd,ex_lb,chrony] ****************************************************************************************
TASK [Gathering Facts] ****************************************************************************************************************
ok: [10.0.0.117]
ok: [10.0.0.118]
ok: [10.0.0.111]
ok: [10.0.0.115]
ok: [10.0.0.114]
ok: [10.0.0.113]
ok: [10.0.0.112]
PLAY [localhost] **********************************************************************************************************************
TASK [Gathering Facts] ****************************************************************************************************************
ok: [localhost]
TASK [deploy : prepare some dirs] *****************************************************************************************************
changed: [localhost] => (item=/etc/kubeasz/clusters/k8s-01/ssl)
changed: [localhost] => (item=/etc/kubeasz/clusters/k8s-01/backup)
changed: [localhost] => (item=/etc/kubeasz/clusters/k8s-01/yml)
ok: [localhost] => (item=~/.kube)
TASK [deploy : 本地设置 bin 目录权限] *********************************************************************************************************
changed: [localhost]
TASK [deploy : 读取ca证书stat信息] **********************************************************************************************************
ok: [localhost]
TASK [deploy : 准备CA配置文件和签名请求] *********************************************************************************************************
changed: [localhost] => (item=ca-config.json)
changed: [localhost] => (item=ca-csr.json)
TASK [deploy : 生成 CA 证书和私钥] ***********************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kubectl使用的admin证书签名请求] ***********************************************************************************************
changed: [localhost]
TASK [deploy : 创建admin证书与私钥] **********************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] ****************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置客户端认证参数] *************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] ***************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] ***************************************************************************************************************
changed: [localhost]
TASK [deploy : 安装kubeconfig] **********************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-proxy 证书签名请求] ***************************************************************************************************
changed: [localhost]
TASK [deploy : 创建 kube-proxy证书与私钥] ****************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] ****************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置客户端认证参数] *************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] ***************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] ***************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-controller-manager 证书签名请求] **************************************************************************************
changed: [localhost]
TASK [deploy : 创建 kube-controller-manager证书与私钥] ***************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] ****************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置认证参数] ****************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] ***************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] ***************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-scheduler 证书签名请求] ***********************************************************************************************
changed: [localhost]
TASK [deploy : 创建 kube-scheduler证书与私钥] ************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] ****************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置认证参数] ****************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] ***************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] ***************************************************************************************************************
changed: [localhost]
TASK [deploy : 本地创建 ezdown/ezctl 工具的软连接] **********************************************************************************************
changed: [localhost] => (item=ezdown)
changed: [localhost] => (item=ezctl)
TASK [deploy : ansible 控制端创建 kubectl 软链接] *********************************************************************************************
changed: [localhost]
PLAY [kube_master,kube_node,etcd] *****************************************************************************************************
TASK [prepare : apt更新缓存刷新] ************************************************************************************************************
ok: [10.0.0.117]
ok: [10.0.0.115]
ok: [10.0.0.114]
ok: [10.0.0.118]
ok: [10.0.0.111]
ok: [10.0.0.112]
ok: [10.0.0.113]
TASK [prepare : 删除ubuntu默认安装] *********************************************************************************************************
changed: [10.0.0.117] => (item=ufw)
changed: [10.0.0.114] => (item=ufw)
changed: [10.0.0.117] => (item=lxd)
changed: [10.0.0.114] => (item=lxd)
changed: [10.0.0.117] => (item=lxd-client)
changed: [10.0.0.118] => (item=ufw)
changed: [10.0.0.111] => (item=ufw)
changed: [10.0.0.115] => (item=ufw)
changed: [10.0.0.114] => (item=lxd-client)
changed: [10.0.0.117] => (item=lxcfs)
changed: [10.0.0.118] => (item=lxd)
changed: [10.0.0.114] => (item=lxcfs)
changed: [10.0.0.111] => (item=lxd)
changed: [10.0.0.115] => (item=lxd)
changed: [10.0.0.117] => (item=lxc-common)
changed: [10.0.0.114] => (item=lxc-common)
changed: [10.0.0.111] => (item=lxd-client)
changed: [10.0.0.118] => (item=lxd-client)
changed: [10.0.0.115] => (item=lxd-client)
changed: [10.0.0.111] => (item=lxcfs)
changed: [10.0.0.115] => (item=lxcfs)
changed: [10.0.0.118] => (item=lxcfs)
changed: [10.0.0.111] => (item=lxc-common)
changed: [10.0.0.115] => (item=lxc-common)
changed: [10.0.0.118] => (item=lxc-common)
changed: [10.0.0.112] => (item=ufw)
changed: [10.0.0.113] => (item=ufw)
changed: [10.0.0.112] => (item=lxd)
changed: [10.0.0.113] => (item=lxd)
changed: [10.0.0.112] => (item=lxd-client)
changed: [10.0.0.113] => (item=lxd-client)
changed: [10.0.0.112] => (item=lxcfs)
changed: [10.0.0.113] => (item=lxcfs)
changed: [10.0.0.112] => (item=lxc-common)
changed: [10.0.0.113] => (item=lxc-common)
TASK [prepare : 安装 ubuntu/debian基础软件] *************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 准备 journal 日志相关目录] ****************************************************************************************************
changed: [10.0.0.114] => (item=/etc/systemd/journald.conf.d)
changed: [10.0.0.115] => (item=/etc/systemd/journald.conf.d)
changed: [10.0.0.117] => (item=/etc/systemd/journald.conf.d)
changed: [10.0.0.118] => (item=/etc/systemd/journald.conf.d)
changed: [10.0.0.111] => (item=/etc/systemd/journald.conf.d)
ok: [10.0.0.115] => (item=/var/log/journal)
ok: [10.0.0.114] => (item=/var/log/journal)
ok: [10.0.0.111] => (item=/var/log/journal)
ok: [10.0.0.117] => (item=/var/log/journal)
changed: [10.0.0.112] => (item=/etc/systemd/journald.conf.d)
changed: [10.0.0.113] => (item=/etc/systemd/journald.conf.d)
ok: [10.0.0.112] => (item=/var/log/journal)
ok: [10.0.0.113] => (item=/var/log/journal)
ok: [10.0.0.118] => (item=/var/log/journal)
TASK [prepare : 优化设置 journal 日志] ******************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.114]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.115]
changed: [10.0.0.113]
changed: [10.0.0.112]
TASK [prepare : 重启 journald 服务] *******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.111]
changed: [10.0.0.118]
changed: [10.0.0.113]
changed: [10.0.0.112]
changed: [10.0.0.117]
TASK [prepare : 禁用系统 swap] ************************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.117]
changed: [10.0.0.115]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 删除fstab swap 相关配置] ****************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.114]
changed: [10.0.0.113]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.115]
changed: [10.0.0.112]
TASK [prepare : 转换内核版本为浮点数] ***********************************************************************************************************
ok: [10.0.0.114]
ok: [10.0.0.115]
ok: [10.0.0.117]
ok: [10.0.0.118]
ok: [10.0.0.111]
ok: [10.0.0.112]
ok: [10.0.0.113]
TASK [prepare : 加载内核模块] ***************************************************************************************************************
changed: [10.0.0.118] => (item=br_netfilter)
changed: [10.0.0.115] => (item=br_netfilter)
changed: [10.0.0.111] => (item=br_netfilter)
changed: [10.0.0.117] => (item=br_netfilter)
changed: [10.0.0.114] => (item=br_netfilter)
changed: [10.0.0.118] => (item=ip_vs)
changed: [10.0.0.115] => (item=ip_vs)
changed: [10.0.0.111] => (item=ip_vs)
changed: [10.0.0.117] => (item=ip_vs)
changed: [10.0.0.114] => (item=ip_vs)
changed: [10.0.0.118] => (item=ip_vs_rr)
changed: [10.0.0.117] => (item=ip_vs_rr)
changed: [10.0.0.114] => (item=ip_vs_rr)
changed: [10.0.0.115] => (item=ip_vs_rr)
changed: [10.0.0.111] => (item=ip_vs_rr)
changed: [10.0.0.117] => (item=ip_vs_wrr)
changed: [10.0.0.114] => (item=ip_vs_wrr)
changed: [10.0.0.118] => (item=ip_vs_wrr)
changed: [10.0.0.117] => (item=ip_vs_sh)
changed: [10.0.0.114] => (item=ip_vs_sh)
ok: [10.0.0.117] => (item=nf_conntrack)
changed: [10.0.0.115] => (item=ip_vs_wrr)
changed: [10.0.0.111] => (item=ip_vs_wrr)
ok: [10.0.0.114] => (item=nf_conntrack)
changed: [10.0.0.118] => (item=ip_vs_sh)
changed: [10.0.0.115] => (item=ip_vs_sh)
changed: [10.0.0.111] => (item=ip_vs_sh)
changed: [10.0.0.113] => (item=br_netfilter)
ok: [10.0.0.118] => (item=nf_conntrack)
ok: [10.0.0.115] => (item=nf_conntrack)
changed: [10.0.0.112] => (item=br_netfilter)
changed: [10.0.0.113] => (item=ip_vs)
ok: [10.0.0.111] => (item=nf_conntrack)
changed: [10.0.0.113] => (item=ip_vs_rr)
changed: [10.0.0.112] => (item=ip_vs)
changed: [10.0.0.113] => (item=ip_vs_wrr)
changed: [10.0.0.112] => (item=ip_vs_rr)
changed: [10.0.0.113] => (item=ip_vs_sh)
changed: [10.0.0.112] => (item=ip_vs_wrr)
ok: [10.0.0.113] => (item=nf_conntrack)
changed: [10.0.0.112] => (item=ip_vs_sh)
ok: [10.0.0.112] => (item=nf_conntrack)
TASK [prepare : 尝试加载nf_conntrack_ipv4] ************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.111]
changed: [10.0.0.117]
changed: [10.0.0.118]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 启用systemd自动加载模块服务] ****************************************************************************************************
ok: [10.0.0.114]
ok: [10.0.0.115]
ok: [10.0.0.117]
ok: [10.0.0.118]
ok: [10.0.0.111]
ok: [10.0.0.113]
ok: [10.0.0.112]
TASK [prepare : 增加内核模块开机加载配置] *********************************************************************************************************
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.114]
changed: [10.0.0.117]
changed: [10.0.0.115]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 设置系统参数] ***************************************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.111]
changed: [10.0.0.114]
changed: [10.0.0.118]
changed: [10.0.0.117]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 生效系统参数] ***************************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 创建 systemd 配置目录] ******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 设置系统 ulimits] *********************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.117]
changed: [10.0.0.115]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 把SCTP列入内核模块黑名单] *******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.117]
changed: [10.0.0.115]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : prepare some dirs] ****************************************************************************************************
changed: [10.0.0.114] => (item=/opt/kube/bin)
changed: [10.0.0.117] => (item=/opt/kube/bin)
changed: [10.0.0.115] => (item=/opt/kube/bin)
changed: [10.0.0.111] => (item=/opt/kube/bin)
changed: [10.0.0.118] => (item=/opt/kube/bin)
changed: [10.0.0.114] => (item=/etc/kubernetes/ssl)
changed: [10.0.0.117] => (item=/etc/kubernetes/ssl)
changed: [10.0.0.115] => (item=/etc/kubernetes/ssl)
changed: [10.0.0.118] => (item=/etc/kubernetes/ssl)
changed: [10.0.0.111] => (item=/etc/kubernetes/ssl)
changed: [10.0.0.114] => (item=/root/.kube)
changed: [10.0.0.117] => (item=/root/.kube)
changed: [10.0.0.115] => (item=/root/.kube)
changed: [10.0.0.111] => (item=/root/.kube)
changed: [10.0.0.118] => (item=/root/.kube)
changed: [10.0.0.114] => (item=/etc/cni/net.d)
changed: [10.0.0.117] => (item=/etc/cni/net.d)
changed: [10.0.0.115] => (item=/etc/cni/net.d)
changed: [10.0.0.111] => (item=/etc/cni/net.d)
changed: [10.0.0.118] => (item=/etc/cni/net.d)
changed: [10.0.0.112] => (item=/opt/kube/bin)
changed: [10.0.0.113] => (item=/opt/kube/bin)
changed: [10.0.0.112] => (item=/etc/kubernetes/ssl)
changed: [10.0.0.113] => (item=/etc/kubernetes/ssl)
changed: [10.0.0.112] => (item=/root/.kube)
changed: [10.0.0.113] => (item=/root/.kube)
changed: [10.0.0.112] => (item=/etc/cni/net.d)
changed: [10.0.0.113] => (item=/etc/cni/net.d)
TASK [prepare : symlink /usr/bin/python -> /usr/bin/python3] **************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 写入环境变量$PATH] **********************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [prepare : 添加 kubectl 自动补全] ******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.113]
changed: [10.0.0.112]
TASK [prepare : 添加 local registry hosts 解析] *******************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
changed: [10.0.0.111]
changed: [10.0.0.113]
changed: [10.0.0.112]
TASK [prepare : 分发 kubeconfig配置文件] ****************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [prepare : 分发 kube-proxy.kubeconfig配置文件] *****************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [prepare : 分发controller/scheduler kubeconfig配置文件] ********************************************************************************
changed: [10.0.0.114] => (item=kube-controller-manager.kubeconfig)
changed: [10.0.0.115] => (item=kube-controller-manager.kubeconfig)
changed: [10.0.0.114] => (item=kube-scheduler.kubeconfig)
changed: [10.0.0.115] => (item=kube-scheduler.kubeconfig)
PLAY RECAP ****************************************************************************************************************************
10.0.0.111 : ok=24 changed=20 unreachable=0 failed=0 skipped=116 rescued=0 ignored=0
10.0.0.112 : ok=24 changed=20 unreachable=0 failed=0 skipped=116 rescued=0 ignored=0
10.0.0.113 : ok=24 changed=20 unreachable=0 failed=0 skipped=116 rescued=0 ignored=0
10.0.0.114 : ok=27 changed=23 unreachable=0 failed=0 skipped=113 rescued=0 ignored=0
10.0.0.115 : ok=27 changed=23 unreachable=0 failed=0 skipped=113 rescued=0 ignored=0
10.0.0.117 : ok=26 changed=22 unreachable=0 failed=0 skipped=114 rescued=0 ignored=0
10.0.0.118 : ok=26 changed=22 unreachable=0 failed=0 skipped=114 rescued=0 ignored=0
localhost : ok=33 changed=31 unreachable=0 failed=0 skipped=11 rescued=0 ignored=0
[root@easzlab-deploy-01 kubeasz]#./ezctl setup k8s-01 02
ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml playbooks/02.etcd.yml
2022-09-05 21:56:26 INFO cluster:k8s-01 setup step:02 begins in 5s, press any key to abort:
PLAY [etcd] ***************************************************************************************************************************
TASK [Gathering Facts] ****************************************************************************************************************
ok: [10.0.0.113]
ok: [10.0.0.112]
ok: [10.0.0.111]
TASK [etcd : prepare some dirs] *******************************************************************************************************
changed: [10.0.0.111]
changed: [10.0.0.113]
changed: [10.0.0.112]
TASK [etcd : 下载etcd二进制文件] *************************************************************************************************************
changed: [10.0.0.111] => (item=etcd)
changed: [10.0.0.113] => (item=etcd)
changed: [10.0.0.112] => (item=etcd)
changed: [10.0.0.113] => (item=etcdctl)
changed: [10.0.0.111] => (item=etcdctl)
changed: [10.0.0.112] => (item=etcdctl)
TASK [etcd : 创建etcd证书请求] **************************************************************************************************************
changed: [10.0.0.113]
ok: [10.0.0.112]
ok: [10.0.0.111]
TASK [etcd : 创建 etcd证书和私钥] ************************************************************************************************************
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [etcd : 分发etcd证书相关] **************************************************************************************************************
changed: [10.0.0.111] => (item=ca.pem)
changed: [10.0.0.113] => (item=ca.pem)
changed: [10.0.0.112] => (item=ca.pem)
changed: [10.0.0.111] => (item=etcd.pem)
changed: [10.0.0.113] => (item=etcd.pem)
changed: [10.0.0.112] => (item=etcd.pem)
changed: [10.0.0.111] => (item=etcd-key.pem)
changed: [10.0.0.113] => (item=etcd-key.pem)
changed: [10.0.0.112] => (item=etcd-key.pem)
TASK [etcd : 创建etcd的systemd unit文件] ***************************************************************************************************
changed: [10.0.0.111]
changed: [10.0.0.112]
changed: [10.0.0.113]
TASK [etcd : 开机启用etcd服务] **************************************************************************************************************
changed: [10.0.0.112]
changed: [10.0.0.111]
changed: [10.0.0.113]
TASK [etcd : 开启etcd服务] ****************************************************************************************************************
changed: [10.0.0.113]
changed: [10.0.0.111]
changed: [10.0.0.112]
TASK [etcd : 以轮询的方式等待服务同步完成] **********************************************************************************************************
changed: [10.0.0.111]
changed: [10.0.0.113]
changed: [10.0.0.112]
PLAY RECAP ****************************************************************************************************************************
10.0.0.111 : ok=10 changed=8 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
10.0.0.112 : ok=10 changed=8 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
10.0.0.113 : ok=10 changed=9 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
[root@easzlab-deploy-01 kubeasz]#./ezctl setup k8s-01 03
ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml playbooks/03.runtime.yml
2022-09-05 21:57:33 INFO cluster:k8s-01 setup step:03 begins in 5s, press any key to abort:
PLAY [kube_master,kube_node] **********************************************************************************************************
TASK [Gathering Facts] ****************************************************************************************************************
ok: [10.0.0.118]
ok: [10.0.0.114]
ok: [10.0.0.117]
ok: [10.0.0.115]
TASK [containerd : 获取是否已经安装containerd] ************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.115]
changed: [10.0.0.118]
changed: [10.0.0.114]
TASK [containerd : 准备containerd相关目录] **************************************************************************************************
ok: [10.0.0.117] => (item=/opt/kube/bin)
ok: [10.0.0.115] => (item=/opt/kube/bin)
ok: [10.0.0.118] => (item=/opt/kube/bin)
ok: [10.0.0.114] => (item=/opt/kube/bin)
changed: [10.0.0.115] => (item=/etc/containerd)
changed: [10.0.0.117] => (item=/etc/containerd)
changed: [10.0.0.118] => (item=/etc/containerd)
changed: [10.0.0.114] => (item=/etc/containerd)
TASK [containerd : 加载内核模块 overlay] ****************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.114]
changed: [10.0.0.118]
changed: [10.0.0.115]
TASK [containerd : 下载 containerd 二进制文件] ***********************************************************************************************
changed: [10.0.0.114] => (item=containerd)
changed: [10.0.0.115] => (item=containerd)
changed: [10.0.0.117] => (item=containerd)
changed: [10.0.0.118] => (item=containerd)
changed: [10.0.0.118] => (item=containerd-shim)
changed: [10.0.0.115] => (item=containerd-shim)
changed: [10.0.0.114] => (item=containerd-shim)
changed: [10.0.0.117] => (item=containerd-shim)
changed: [10.0.0.115] => (item=containerd-shim-runc-v1)
changed: [10.0.0.117] => (item=containerd-shim-runc-v1)
changed: [10.0.0.114] => (item=containerd-shim-runc-v1)
changed: [10.0.0.118] => (item=containerd-shim-runc-v1)
changed: [10.0.0.117] => (item=containerd-shim-runc-v2)
changed: [10.0.0.114] => (item=containerd-shim-runc-v2)
changed: [10.0.0.115] => (item=containerd-shim-runc-v2)
changed: [10.0.0.118] => (item=containerd-shim-runc-v2)
changed: [10.0.0.114] => (item=crictl)
changed: [10.0.0.117] => (item=crictl)
changed: [10.0.0.115] => (item=crictl)
changed: [10.0.0.118] => (item=crictl)
changed: [10.0.0.115] => (item=ctr)
changed: [10.0.0.117] => (item=ctr)
changed: [10.0.0.114] => (item=ctr)
changed: [10.0.0.118] => (item=ctr)
changed: [10.0.0.118] => (item=runc)
changed: [10.0.0.114] => (item=runc)
changed: [10.0.0.117] => (item=runc)
changed: [10.0.0.115] => (item=runc)
TASK [containerd : 创建 containerd 配置文件] ************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [containerd : 创建systemd unit文件] **************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.118]
changed: [10.0.0.117]
TASK [containerd : 创建 crictl 配置] ******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [containerd : 开机启用 containerd 服务] ************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.117]
changed: [10.0.0.118]
changed: [10.0.0.115]
TASK [containerd : 开启 containerd 服务] **************************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [containerd : 轮询等待containerd服务运行] ************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
changed: [10.0.0.117]
changed: [10.0.0.118]
PLAY RECAP ****************************************************************************************************************************
10.0.0.114 : ok=11 changed=10 unreachable=0 failed=0 skipped=18 rescued=0 ignored=0
10.0.0.115 : ok=11 changed=10 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0
10.0.0.117 : ok=11 changed=10 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0
10.0.0.118 : ok=11 changed=10 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0
[root@easzlab-deploy-01 kubeasz]#./ezctl setup k8s-01 04
ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml playbooks/04.kube-master.yml
2022-09-05 21:59:22 INFO cluster:k8s-01 setup step:04 begins in 5s, press any key to abort:
PLAY [kube_master] ********************************************************************************************************************
TASK [Gathering Facts] ****************************************************************************************************************
ok: [10.0.0.114]
ok: [10.0.0.115]
TASK [kube-lb : prepare some dirs] ****************************************************************************************************
changed: [10.0.0.114] => (item=/etc/kube-lb/sbin)
changed: [10.0.0.115] => (item=/etc/kube-lb/sbin)
changed: [10.0.0.114] => (item=/etc/kube-lb/logs)
changed: [10.0.0.115] => (item=/etc/kube-lb/logs)
changed: [10.0.0.115] => (item=/etc/kube-lb/conf)
changed: [10.0.0.114] => (item=/etc/kube-lb/conf)
TASK [kube-lb : 下载二进制文件kube-lb(nginx)] ************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-lb : 创建kube-lb的配置文件] *******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-lb : 创建kube-lb的systemd unit文件] *********************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-lb : 开机启用kube-lb服务] ********************************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
TASK [kube-lb : 开启kube-lb服务] **********************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] **************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-master : 下载 kube_master 二进制] ***********************************************************************************************
changed: [10.0.0.114] => (item=kube-apiserver)
changed: [10.0.0.115] => (item=kube-apiserver)
changed: [10.0.0.114] => (item=kube-controller-manager)
changed: [10.0.0.115] => (item=kube-controller-manager)
changed: [10.0.0.114] => (item=kube-scheduler)
changed: [10.0.0.115] => (item=kube-scheduler)
changed: [10.0.0.114] => (item=kubectl)
changed: [10.0.0.115] => (item=kubectl)
TASK [kube-master : 注册变量 KUBERNETES_SVC_IP] *******************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-master : 设置变量 CLUSTER_KUBERNETES_SVC_IP] ***********************************************************************************
ok: [10.0.0.114]
ok: [10.0.0.115]
TASK [kube-master : 创建 kubernetes 证书签名请求] *********************************************************************************************
changed: [10.0.0.115]
ok: [10.0.0.114]
TASK [kube-master : 创建 kubernetes 证书和私钥] **********************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-master : 创建 aggregator proxy证书签名请求] ****************************************************************************************
changed: [10.0.0.114]
ok: [10.0.0.115]
TASK [kube-master : 创建 aggregator-proxy证书和私钥] *****************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
TASK [kube-master : 分发 kubernetes证书] **************************************************************************************************
changed: [10.0.0.114] => (item=ca.pem)
changed: [10.0.0.115] => (item=ca.pem)
changed: [10.0.0.114] => (item=ca-key.pem)
changed: [10.0.0.115] => (item=ca-key.pem)
changed: [10.0.0.114] => (item=kubernetes.pem)
changed: [10.0.0.115] => (item=kubernetes.pem)
changed: [10.0.0.114] => (item=kubernetes-key.pem)
changed: [10.0.0.115] => (item=kubernetes-key.pem)
changed: [10.0.0.114] => (item=aggregator-proxy.pem)
changed: [10.0.0.115] => (item=aggregator-proxy.pem)
changed: [10.0.0.115] => (item=aggregator-proxy-key.pem)
changed: [10.0.0.114] => (item=aggregator-proxy-key.pem)
TASK [kube-master : 替换 kubeconfig 的 apiserver 地址] *************************************************************************************
changed: [10.0.0.114] => (item=/root/.kube/config)
changed: [10.0.0.115] => (item=/root/.kube/config)
changed: [10.0.0.114] => (item=/etc/kubernetes/kube-controller-manager.kubeconfig)
changed: [10.0.0.115] => (item=/etc/kubernetes/kube-controller-manager.kubeconfig)
changed: [10.0.0.114] => (item=/etc/kubernetes/kube-scheduler.kubeconfig)
changed: [10.0.0.115] => (item=/etc/kubernetes/kube-scheduler.kubeconfig)
TASK [kube-master : 创建 master 服务的 systemd unit 文件] ************************************************************************************
changed: [10.0.0.114] => (item=kube-apiserver.service)
changed: [10.0.0.115] => (item=kube-apiserver.service)
changed: [10.0.0.114] => (item=kube-controller-manager.service)
changed: [10.0.0.115] => (item=kube-controller-manager.service)
changed: [10.0.0.114] => (item=kube-scheduler.service)
changed: [10.0.0.115] => (item=kube-scheduler.service)
TASK [kube-master : enable master 服务] *************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-master : 启动 master 服务] *****************************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
TASK [kube-master : 轮询等待kube-apiserver启动] *********************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-master : 轮询等待kube-controller-manager启动] ************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
TASK [kube-master : 轮询等待kube-scheduler启动] *********************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-master : 以轮询的方式等待master服务启动完成] *********************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
TASK [kube-master : 获取user:kubernetes是否已经绑定对应角色] **************************************************************************************
changed: [10.0.0.114]
TASK [kube-master : 创建user:kubernetes角色绑定] ********************************************************************************************
changed: [10.0.0.114]
TASK [kube-node : 创建kube_node 相关目录] ***************************************************************************************************
changed: [10.0.0.114] => (item=/var/lib/kubelet)
changed: [10.0.0.115] => (item=/var/lib/kubelet)
changed: [10.0.0.114] => (item=/var/lib/kube-proxy)
changed: [10.0.0.115] => (item=/var/lib/kube-proxy)
ok: [10.0.0.114] => (item=/etc/cni/net.d)
ok: [10.0.0.115] => (item=/etc/cni/net.d)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] ***************************************************************************
ok: [10.0.0.115] => (item=kubectl)
ok: [10.0.0.114] => (item=kubectl)
changed: [10.0.0.115] => (item=kubelet)
changed: [10.0.0.114] => (item=kubelet)
changed: [10.0.0.115] => (item=kube-proxy)
changed: [10.0.0.115] => (item=bridge)
changed: [10.0.0.115] => (item=host-local)
changed: [10.0.0.114] => (item=kube-proxy)
changed: [10.0.0.115] => (item=loopback)
changed: [10.0.0.114] => (item=bridge)
changed: [10.0.0.114] => (item=host-local)
changed: [10.0.0.114] => (item=loopback)
TASK [kube-node : 替换 kubeconfig 的 apiserver 地址] ***************************************************************************************
ok: [10.0.0.114]
ok: [10.0.0.115]
TASK [kube-node : 准备kubelet 证书签名请求] ***************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 创建 kubelet 证书与私钥] ***************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 分发ca 证书] ************************************************************************************************************
ok: [10.0.0.115]
ok: [10.0.0.114]
TASK [kube-node : 分发kubelet 证书] *******************************************************************************************************
changed: [10.0.0.114] => (item=kubelet.pem)
changed: [10.0.0.115] => (item=kubelet.pem)
changed: [10.0.0.114] => (item=kubelet-key.pem)
changed: [10.0.0.115] => (item=kubelet-key.pem)
TASK [kube-node : 设置集群参数] *************************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 设置客户端认证参数] **********************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 设置上下文参数] ************************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 选择默认上下文] ************************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 准备 cni配置文件] *********************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 注册变量 DNS_SVC_IP] ****************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 设置变量 CLUSTER_DNS_SVC_IP] ********************************************************************************************
ok: [10.0.0.114]
ok: [10.0.0.115]
TASK [kube-node : 创建kubelet的配置文件] *****************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 创建kubelet的systemd unit文件] *******************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 开机启用kubelet 服务] *****************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 开启kubelet 服务] *******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] ****************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 创建kube-proxy 配置] ****************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 创建kube-proxy 服务文件] **************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 开机启用kube-proxy 服务] **************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 开启kube-proxy 服务] ****************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 轮询等待kube-proxy启动] ***************************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
TASK [kube-node : 轮询等待kubelet启动] ******************************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
TASK [kube-node : 轮询等待node达到Ready状态] **************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-node : 设置node节点role] *******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [Making master nodes SchedulingDisabled] *****************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [Setting master role name] *******************************************************************************************************
changed: [10.0.0.114]
changed: [10.0.0.115]
PLAY RECAP ****************************************************************************************************************************
10.0.0.114 : ok=55 changed=49 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
10.0.0.115 : ok=53 changed=47 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
[root@easzlab-deploy-01 kubeasz]#./ezctl setup k8s-01 05
ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml playbooks/05.kube-node.yml
2022-09-05 22:02:55 INFO cluster:k8s-01 setup step:05 begins in 5s, press any key to abort:
PLAY [kube_node] **********************************************************************************************************************
TASK [Gathering Facts] ****************************************************************************************************************
ok: [10.0.0.118]
ok: [10.0.0.117]
TASK [kube-lb : prepare some dirs] ****************************************************************************************************
changed: [10.0.0.117] => (item=/etc/kube-lb/sbin)
changed: [10.0.0.118] => (item=/etc/kube-lb/sbin)
changed: [10.0.0.118] => (item=/etc/kube-lb/logs)
changed: [10.0.0.117] => (item=/etc/kube-lb/logs)
changed: [10.0.0.118] => (item=/etc/kube-lb/conf)
changed: [10.0.0.117] => (item=/etc/kube-lb/conf)
TASK [kube-lb : 下载二进制文件kube-lb(nginx)] ************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-lb : 创建kube-lb的配置文件] *******************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-lb : 创建kube-lb的systemd unit文件] *********************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-lb : 开机启用kube-lb服务] ********************************************************************************************************
changed: [10.0.0.118]
changed: [10.0.0.117]
TASK [kube-lb : 开启kube-lb服务] **********************************************************************************************************
changed: [10.0.0.118]
changed: [10.0.0.117]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] **************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 创建kube_node 相关目录] ***************************************************************************************************
changed: [10.0.0.117] => (item=/var/lib/kubelet)
changed: [10.0.0.118] => (item=/var/lib/kubelet)
changed: [10.0.0.117] => (item=/var/lib/kube-proxy)
changed: [10.0.0.118] => (item=/var/lib/kube-proxy)
ok: [10.0.0.117] => (item=/etc/cni/net.d)
ok: [10.0.0.118] => (item=/etc/cni/net.d)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] ***************************************************************************
changed: [10.0.0.118] => (item=kubectl)
changed: [10.0.0.117] => (item=kubectl)
changed: [10.0.0.117] => (item=kubelet)
changed: [10.0.0.118] => (item=kubelet)
changed: [10.0.0.117] => (item=kube-proxy)
changed: [10.0.0.118] => (item=kube-proxy)
changed: [10.0.0.118] => (item=bridge)
changed: [10.0.0.117] => (item=bridge)
changed: [10.0.0.118] => (item=host-local)
changed: [10.0.0.117] => (item=host-local)
changed: [10.0.0.118] => (item=loopback)
changed: [10.0.0.117] => (item=loopback)
TASK [kube-node : 替换 kubeconfig 的 apiserver 地址] ***************************************************************************************
changed: [10.0.0.118]
changed: [10.0.0.117]
TASK [kube-node : 准备kubelet 证书签名请求] ***************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 创建 kubelet 证书与私钥] ***************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 分发ca 证书] ************************************************************************************************************
changed: [10.0.0.118]
changed: [10.0.0.117]
TASK [kube-node : 分发kubelet 证书] *******************************************************************************************************
changed: [10.0.0.117] => (item=kubelet.pem)
changed: [10.0.0.118] => (item=kubelet.pem)
changed: [10.0.0.117] => (item=kubelet-key.pem)
changed: [10.0.0.118] => (item=kubelet-key.pem)
TASK [kube-node : 设置集群参数] *************************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 设置客户端认证参数] **********************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 设置上下文参数] ************************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 选择默认上下文] ************************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 准备 cni配置文件] *********************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 注册变量 DNS_SVC_IP] ****************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 设置变量 CLUSTER_DNS_SVC_IP] ********************************************************************************************
ok: [10.0.0.117]
ok: [10.0.0.118]
TASK [kube-node : 创建kubelet的配置文件] *****************************************************************************************************
changed: [10.0.0.118]
changed: [10.0.0.117]
TASK [kube-node : 创建kubelet的systemd unit文件] *******************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 开机启用kubelet 服务] *****************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 开启kubelet 服务] *******************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] ****************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 创建kube-proxy 配置] ****************************************************************************************************
changed: [10.0.0.118]
changed: [10.0.0.117]
TASK [kube-node : 创建kube-proxy 服务文件] **************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 开机启用kube-proxy 服务] **************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 开启kube-proxy 服务] ****************************************************************************************************
changed: [10.0.0.118]
changed: [10.0.0.117]
TASK [kube-node : 轮询等待kube-proxy启动] ***************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 轮询等待kubelet启动] ******************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 轮询等待node达到Ready状态] **************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-node : 设置node节点role] *******************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
PLAY RECAP ****************************************************************************************************************************
10.0.0.117 : ok=35 changed=33 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
10.0.0.118 : ok=35 changed=33 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
[root@easzlab-deploy-01 kubeasz]#kubectl get node -owide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
10.0.0.114 Ready,SchedulingDisabled master 3m23s v1.24.2 10.0.0.114 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
10.0.0.115 Ready,SchedulingDisabled master 3m23s v1.24.2 10.0.0.115 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
10.0.0.117 Ready node 49s v1.24.2 10.0.0.117 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
10.0.0.118 Ready node 49s v1.24.2 10.0.0.118 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
5 扩展集群
[root@easzlab-deploy-01 kubeasz]#./ezctl add-master k8s-01 10.0.0.116
ln: failed to create symbolic link '/usr/bin/python': File exists
2022-09-06 08:55:39 INFO add 10.0.0.116 into 'kube_master' group
2022-09-06 08:55:39 INFO start to add a master node:10.0.0.116 into cluster:k8s-01
PLAY [10.0.0.116] *************************************************************************************************************************************************
TASK [Gathering Facts] ********************************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : apt更新缓存刷新] ****************************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 删除ubuntu默认安装] *************************************************************************************************************************************
changed: [10.0.0.116] => (item=ufw)
changed: [10.0.0.116] => (item=lxd)
changed: [10.0.0.116] => (item=lxd-client)
changed: [10.0.0.116] => (item=lxcfs)
changed: [10.0.0.116] => (item=lxc-common)
TASK [prepare : 安装 ubuntu/debian基础软件] *****************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 准备 journal 日志相关目录] ********************************************************************************************************************************
ok: [10.0.0.116] => (item=/etc/systemd/journald.conf.d)
ok: [10.0.0.116] => (item=/var/log/journal)
TASK [prepare : 优化设置 journal 日志] **********************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 重启 journald 服务] ***********************************************************************************************************************************
changed: [10.0.0.116]
TASK [prepare : 禁用系统 swap] ****************************************************************************************************************************************
changed: [10.0.0.116]
TASK [prepare : 删除fstab swap 相关配置] ********************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 转换内核版本为浮点数] ***************************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 加载内核模块] *******************************************************************************************************************************************
ok: [10.0.0.116] => (item=br_netfilter)
ok: [10.0.0.116] => (item=ip_vs)
ok: [10.0.0.116] => (item=ip_vs_rr)
ok: [10.0.0.116] => (item=ip_vs_wrr)
ok: [10.0.0.116] => (item=ip_vs_sh)
ok: [10.0.0.116] => (item=nf_conntrack)
TASK [prepare : 尝试加载nf_conntrack_ipv4] ****************************************************************************************************************************
changed: [10.0.0.116]
TASK [prepare : 启用systemd自动加载模块服务] ********************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 增加内核模块开机加载配置] *************************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 设置系统参数] *******************************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 生效系统参数] *******************************************************************************************************************************************
changed: [10.0.0.116]
TASK [prepare : 创建 systemd 配置目录] **********************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 设置系统 ulimits] *************************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 把SCTP列入内核模块黑名单] ***********************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : prepare some dirs] ********************************************************************************************************************************
ok: [10.0.0.116] => (item=/opt/kube/bin)
changed: [10.0.0.116] => (item=/etc/kubernetes/ssl)
ok: [10.0.0.116] => (item=/root/.kube)
changed: [10.0.0.116] => (item=/etc/cni/net.d)
TASK [prepare : symlink /usr/bin/python -> /usr/bin/python3] ******************************************************************************************************
changed: [10.0.0.116]
TASK [prepare : 写入环境变量$PATH] **************************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 添加 kubectl 自动补全] **********************************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 添加 local registry hosts 解析] ***********************************************************************************************************************
ok: [10.0.0.116]
TASK [prepare : 分发 kubeconfig配置文件] ********************************************************************************************************************************
changed: [10.0.0.116]
TASK [prepare : 分发 kube-proxy.kubeconfig配置文件] *********************************************************************************************************************
changed: [10.0.0.116]
TASK [prepare : 分发controller/scheduler kubeconfig配置文件] ************************************************************************************************************
changed: [10.0.0.116] => (item=kube-controller-manager.kubeconfig)
changed: [10.0.0.116] => (item=kube-scheduler.kubeconfig)
TASK [containerd : 获取是否已经安装containerd] ****************************************************************************************************************************
changed: [10.0.0.116]
TASK [containerd : 准备containerd相关目录] ******************************************************************************************************************************
ok: [10.0.0.116] => (item=/opt/kube/bin)
changed: [10.0.0.116] => (item=/etc/containerd)
TASK [containerd : 加载内核模块 overlay] ********************************************************************************************************************************
ok: [10.0.0.116]
TASK [containerd : 下载 containerd 二进制文件] ***************************************************************************************************************************
ok: [10.0.0.116] => (item=containerd)
ok: [10.0.0.116] => (item=containerd-shim)
ok: [10.0.0.116] => (item=containerd-shim-runc-v1)
ok: [10.0.0.116] => (item=containerd-shim-runc-v2)
ok: [10.0.0.116] => (item=crictl)
ok: [10.0.0.116] => (item=ctr)
ok: [10.0.0.116] => (item=runc)
TASK [containerd : 创建 containerd 配置文件] ****************************************************************************************************************************
changed: [10.0.0.116]
TASK [containerd : 创建systemd unit文件] ******************************************************************************************************************************
changed: [10.0.0.116]
TASK [containerd : 创建 crictl 配置] **********************************************************************************************************************************
changed: [10.0.0.116]
TASK [containerd : 开机启用 containerd 服务] ****************************************************************************************************************************
changed: [10.0.0.116]
TASK [containerd : 开启 containerd 服务] ******************************************************************************************************************************
changed: [10.0.0.116]
TASK [containerd : 轮询等待containerd服务运行] ****************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-lb : prepare some dirs] ********************************************************************************************************************************
changed: [10.0.0.116] => (item=/etc/kube-lb/sbin)
changed: [10.0.0.116] => (item=/etc/kube-lb/logs)
changed: [10.0.0.116] => (item=/etc/kube-lb/conf)
TASK [kube-lb : 下载二进制文件kube-lb(nginx)] ****************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-lb : 创建kube-lb的配置文件] ***********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-lb : 创建kube-lb的systemd unit文件] *************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-lb : 开机启用kube-lb服务] ************************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-lb : 开启kube-lb服务] **************************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] ******************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 下载 kube_master 二进制] ***************************************************************************************************************************
ok: [10.0.0.116] => (item=kube-apiserver)
ok: [10.0.0.116] => (item=kube-controller-manager)
ok: [10.0.0.116] => (item=kube-scheduler)
ok: [10.0.0.116] => (item=kubectl)
TASK [kube-master : 注册变量 KUBERNETES_SVC_IP] ***********************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 设置变量 CLUSTER_KUBERNETES_SVC_IP] ***************************************************************************************************************
ok: [10.0.0.116]
TASK [kube-master : 创建 kubernetes 证书签名请求] *************************************************************************************************************************
ok: [10.0.0.116]
TASK [kube-master : 创建 kubernetes 证书和私钥] **************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 创建 aggregator proxy证书签名请求] ********************************************************************************************************************
ok: [10.0.0.116]
TASK [kube-master : 创建 aggregator-proxy证书和私钥] *********************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 分发 kubernetes证书] ******************************************************************************************************************************
changed: [10.0.0.116] => (item=ca.pem)
changed: [10.0.0.116] => (item=ca-key.pem)
changed: [10.0.0.116] => (item=kubernetes.pem)
changed: [10.0.0.116] => (item=kubernetes-key.pem)
changed: [10.0.0.116] => (item=aggregator-proxy.pem)
changed: [10.0.0.116] => (item=aggregator-proxy-key.pem)
TASK [kube-master : 替换 kubeconfig 的 apiserver 地址] *****************************************************************************************************************
changed: [10.0.0.116] => (item=/root/.kube/config)
changed: [10.0.0.116] => (item=/etc/kubernetes/kube-controller-manager.kubeconfig)
changed: [10.0.0.116] => (item=/etc/kubernetes/kube-scheduler.kubeconfig)
TASK [kube-master : 创建 master 服务的 systemd unit 文件] ****************************************************************************************************************
changed: [10.0.0.116] => (item=kube-apiserver.service)
changed: [10.0.0.116] => (item=kube-controller-manager.service)
changed: [10.0.0.116] => (item=kube-scheduler.service)
TASK [kube-master : enable master 服务] *****************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 启动 master 服务] *********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 轮询等待kube-apiserver启动] *************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 轮询等待kube-controller-manager启动] ****************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 轮询等待kube-scheduler启动] *************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 以轮询的方式等待master服务启动完成] *************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-master : 获取user:kubernetes是否已经绑定对应角色] ******************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 创建kube_node 相关目录] *******************************************************************************************************************************
changed: [10.0.0.116] => (item=/var/lib/kubelet)
changed: [10.0.0.116] => (item=/var/lib/kube-proxy)
ok: [10.0.0.116] => (item=/etc/cni/net.d)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] *******************************************************************************************************
ok: [10.0.0.116] => (item=kubectl)
ok: [10.0.0.116] => (item=kubelet)
ok: [10.0.0.116] => (item=kube-proxy)
ok: [10.0.0.116] => (item=bridge)
ok: [10.0.0.116] => (item=host-local)
ok: [10.0.0.116] => (item=loopback)
TASK [kube-node : 替换 kubeconfig 的 apiserver 地址] *******************************************************************************************************************
ok: [10.0.0.116]
TASK [kube-node : 准备kubelet 证书签名请求] *******************************************************************************************************************************
ok: [10.0.0.116]
TASK [kube-node : 创建 kubelet 证书与私钥] *******************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 分发ca 证书] ****************************************************************************************************************************************
ok: [10.0.0.116]
TASK [kube-node : 分发kubelet 证书] ***********************************************************************************************************************************
changed: [10.0.0.116] => (item=kubelet.pem)
changed: [10.0.0.116] => (item=kubelet-key.pem)
TASK [kube-node : 设置集群参数] *****************************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 设置客户端认证参数] **************************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 设置上下文参数] ****************************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 选择默认上下文] ****************************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 准备 cni配置文件] *************************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 注册变量 DNS_SVC_IP] ********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 设置变量 CLUSTER_DNS_SVC_IP] ************************************************************************************************************************
ok: [10.0.0.116]
TASK [kube-node : 创建kubelet的配置文件] *********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 创建kubelet的systemd unit文件] ***********************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 开机启用kubelet 服务] *********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 开启kubelet 服务] ***********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] ********************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 创建kube-proxy 配置] ********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 创建kube-proxy 服务文件] ******************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 开机启用kube-proxy 服务] ******************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 开启kube-proxy 服务] ********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 轮询等待kube-proxy启动] *******************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 轮询等待kubelet启动] **********************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 轮询等待node达到Ready状态] ******************************************************************************************************************************
changed: [10.0.0.116]
TASK [kube-node : 设置node节点role] ***********************************************************************************************************************************
changed: [10.0.0.116]
TASK [calico : 在节点创建相关目录] *****************************************************************************************************************************************
changed: [10.0.0.116] => (item=/etc/calico/ssl)
TASK [calico : 创建calico 证书请求] *************************************************************************************************************************************
ok: [10.0.0.116]
TASK [calico : 创建 calico证书和私钥] ************************************************************************************************************************************
changed: [10.0.0.116]
TASK [calico : 分发calico证书相关] **************************************************************************************************************************************
changed: [10.0.0.116] => (item=ca.pem)
changed: [10.0.0.116] => (item=calico.pem)
changed: [10.0.0.116] => (item=calico-key.pem)
TASK [calico : get calico-etcd-secrets info] **********************************************************************************************************************
changed: [10.0.0.116]
TASK [calico : 配置 calico DaemonSet yaml文件] ************************************************************************************************************************
ok: [10.0.0.116]
TASK [calico : 运行 calico网络] ***************************************************************************************************************************************
changed: [10.0.0.116]
TASK [calico : 删除默认cni配置] *****************************************************************************************************************************************
changed: [10.0.0.116]
TASK [calico : 下载calicoctl 客户端] ***********************************************************************************************************************************
ok: [10.0.0.116] => (item=calicoctl)
TASK [calico : 准备 calicoctl配置文件] **********************************************************************************************************************************
changed: [10.0.0.116]
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (14 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (13 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (12 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (11 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (10 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (9 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (8 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (7 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (6 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (5 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (4 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (3 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (2 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (1 retries left).
TASK [calico : 轮询等待calico-node 运行，视下载镜像速度而定] **********************************************************************************************************************
fatal: [10.0.0.116]: FAILED! => {"attempts": 15, "changed": true, "cmd": "/opt/kube/bin/kubectl get pod -n kube-system -o wide|grep 'calico-node'|grep ' 10.0.0.116ta": "0:00:00.267966", "end": "2022-09-06 09:02:20.735467", "rc": 0, "start": "2022-09-06 09:02:20.467501", "stderr": "", "stderr_lines": [], "stdout": "Init:0/2",2"]}
...ignoring
TASK [Making master nodes SchedulingDisabled] *********************************************************************************************************************
changed: [10.0.0.116]
TASK [Setting master role name] ***********************************************************************************************************************************
changed: [10.0.0.116]
PLAY RECAP ********************************************************************************************************************************************************
10.0.0.116 : ok=101 changed=70 unreachable=0 failed=0 skipped=169 rescued=0 ignored=1
2022-09-06 09:02:25 INFO reconfigure and restart 'kube-lb' service
PLAY [kube_master,kube_node,etcd,ex_lb,chrony] ********************************************************************************************************************
TASK [Gathering Facts] ********************************************************************************************************************************************
ok: [10.0.0.116]
ok: [10.0.0.117]
ok: [10.0.0.114]
ok: [10.0.0.115]
ok: [10.0.0.118]
ok: [10.0.0.111]
ok: [10.0.0.113]
ok: [10.0.0.112]
PLAY [localhost] **************************************************************************************************************************************************
TASK [Gathering Facts] ********************************************************************************************************************************************
ok: [localhost]
PLAY [kube_master,kube_node,etcd] *********************************************************************************************************************************
PLAY [etcd] *******************************************************************************************************************************************************
PLAY [kube_master,kube_node] **************************************************************************************************************************************
PLAY [kube_master] ************************************************************************************************************************************************
TASK [kube-lb : 创建kube-lb的配置文件] ***********************************************************************************************************************************
ok: [10.0.0.116]
changed: [10.0.0.114]
changed: [10.0.0.115]
TASK [kube-lb : 创建kube-lb的systemd unit文件] *************************************************************************************************************************
ok: [10.0.0.116]
ok: [10.0.0.114]
ok: [10.0.0.115]
TASK [kube-lb : 开启kube-lb服务] **************************************************************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
changed: [10.0.0.116]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] ******************************************************************************************************************************
changed: [10.0.0.115]
changed: [10.0.0.114]
changed: [10.0.0.116]
PLAY [kube_node] **************************************************************************************************************************************************
TASK [kube-lb : 创建kube-lb的配置文件] ***********************************************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-lb : 创建kube-lb的systemd unit文件] *************************************************************************************************************************
ok: [10.0.0.117]
ok: [10.0.0.118]
TASK [kube-lb : 开启kube-lb服务] **************************************************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] ******************************************************************************************************************************
changed: [10.0.0.117]
changed: [10.0.0.118]
PLAY [kube_master,kube_node] **************************************************************************************************************************************
PLAY [kube_node] **************************************************************************************************************************************************
PLAY RECAP ********************************************************************************************************************************************************
10.0.0.111 : ok=1 changed=0 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
10.0.0.112 : ok=1 changed=0 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
10.0.0.113 : ok=1 changed=0 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
10.0.0.114 : ok=5 changed=3 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
10.0.0.115 : ok=5 changed=3 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
10.0.0.116 : ok=5 changed=2 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
10.0.0.117 : ok=5 changed=3 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
10.0.0.118 : ok=5 changed=3 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
localhost : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
2022-09-06 09:02:48 INFO reconfigure and restart 'ex-lb' service
PLAY [ex_lb] ******************************************************************************************************************************************************
skipping: no hosts matched
PLAY RECAP ********************************************************************************************************************************************************
[root@easzlab-k8s-node-01 harbor.magedu.net]#for i in 10.0.0.{111..116} 10.0.0.{121..129};do ssh root@$i "hostname;mkdir -p /etc/docker/certs.d/harbor.magedu.net";done
[root@easzlab-k8s-node-01 harbor.magedu.net]#for i in 10.0.0.{111..116} 10.0.0.{121..129};do scp /etc/docker/certs.d/harbor.magedu.net/magedu.net.crt root@$i:/etc/docker/certs.d/harbor.magedu.net/ ;done
100% 2126 1.0MB/s 00:00
[root@easzlab-k8s-node-01 harbor.magedu.net]#for i in 10.0.0.{111..116} 10.0.0.{121..129};do scp /lib/systemd/system/buildkit.service root@$i:/lib/systemd/system/ ;done
100% 231 142.2KB/s 00:00
[root@easzlab-k8s-node-01 harbor.magedu.net]#for i in 10.0.0.{111..116} 10.0.0.{121..129};do scp /lib/systemd/system/buildkit.socket root@$i:/lib/systemd/system/ ;done
[root@easzlab-k8s-node-01 ~]#for i in 10.0.0.{111..116} 10.0.0.{121..129};do scp buildkit-v0.10.3.linux-amd64.tar.gz nerdctl-0.22.2-linux-amd64.tar.gz root@$i: ;done
[root@easzlab-k8s-node-01 ~]#for i in 10.0.0.{111..116} 10.0.0.{121..129};do ssh root@$i "hostname;tar -xvzf nerdctl-0.22.2-linux-amd64.tar.gz -C /usr/local/bin/;tar -xvzf buildkit-v0.10.3.linux-amd64.tar.gz -C /usr/local/bin/";done
[root@easzlab-k8s-node-01 ~]#for i in 10.0.0.{111..116} 10.0.0.{121..129};do ssh root@$i "hostname;mv /usr/local/bin/bin/buildctl /usr/local/bin/bin/buildkitd /usr/local/bin/";done
[root@easzlab-k8s-node-01 ~]#for i in 10.0.0.{111..116} 10.0.0.{121..129};do ssh root@$i "hostname;mv /usr/local/bin/bin/buildctl /usr/local/bin/bin/buildkitd /usr/local/bin/";done
6 升级集群
#升级前状态
[root@easzlab-deploy-01 docker]#kubectl get node -owide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
10.0.0.114 Ready,SchedulingDisabled master 2d21h v1.24.2 10.0.0.114 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
10.0.0.115 Ready,SchedulingDisabled master 2d21h v1.24.2 10.0.0.115 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
10.0.0.116 Ready,SchedulingDisabled master 2d10h v1.24.2 10.0.0.116 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
10.0.0.117 Ready node 2d21h v1.24.2 10.0.0.117 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
10.0.0.118 Ready node 34h v1.24.2 10.0.0.118 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
10.0.0.119 Ready node 34h v1.24.2 10.0.0.119 <none> Ubuntu 20.04.4 LTS 5.4.0-125-generic containerd://1.6.4
#升级前准备
#下载kubernetes组件
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md#downloads-for-v1243
wget https://dl.k8s.io/v1.24.3/bin/linux/amd64/kubectl
wget https://dl.k8s.io/v1.24.3/bin/linux/amd64/kubelet
wget https://dl.k8s.io/v1.24.3/bin/linux/amd64/kubeadm
wget https://dl.k8s.io/v1.24.3/bin/linux/amd64/kube-scheduler
wget https://dl.k8s.io/v1.24.3/bin/linux/amd64/kube-proxy
wget https://dl.k8s.io/v1.24.3/bin/linux/amd64/kube-controller-manager
wget https://dl.k8s.io/v1.24.3/bin/linux/amd64/kube-apiserver
[root@easzlab-deploy-01 docker]#chmod +x ./*
#上传docker-20.10.17-binary-install.tar包并解压
[root@easzlab-deploy-01 docker]#ll
total 598756
drwxr-xr-x 2 root root 189 Sep 8 19:32 ./
drwx------ 10 root root 4096 Sep 8 19:14 ../
-rw-r--r-- 1 root root 76724391 Sep 8 19:32 docker-20.10.17-binary-install.tar.gz
-rwxr-xr-x 1 root root 125865984 Jul 14 03:55 kube-apiserver*
-rwxr-xr-x 1 root root 115515392 Jul 14 03:55 kube-controller-manager*
-rwxr-xr-x 1 root root 41762816 Jul 14 03:55 kube-proxy*
-rwxr-xr-x 1 root root 47144960 Jul 14 03:55 kube-scheduler*
-rwxr-xr-x 1 root root 44376064 Jul 14 03:55 kubeadm*
-rwxr-xr-x 1 root root 45711360 Jul 14 03:55 kubectl*
-rwxr-xr-x 1 root root 116013432 Jul 14 03:55 kubelet*
升级master containerd,注意,如果此时集群节点有pod需要提前关闭pod节点调度，在驱逐上面pod，相应命令如下：
kubectl cordon <node name> #设置节点不可调度
kubectl drain <node name> #驱逐节点上pod
kubectl uncordon <node name> #恢复节点pod调度
[root@easzlab-deploy-01 docker]#rsync ./* 10.0.0.114:~/update
[root@easzlab-k8s-master-01 update]#ll
total 598756
drwxr-xr-x 2 root root 189 Sep 8 19:39 ./
drwx------ 11 root root 4096 Sep 8 19:38 ../
-rw-r--r-- 1 root root 76724391 Sep 8 19:38 docker-20.10.17-binary-install.tar.gz
-rwxr-xr-x 1 root root 125865984 Sep 8 19:38 kube-apiserver*
-rwxr-xr-x 1 root root 115515392 Sep 8 19:38 kube-controller-manager*
-rwxr-xr-x 1 root root 41762816 Sep 8 19:38 kube-proxy*
-rwxr-xr-x 1 root root 47144960 Sep 8 19:39 kube-scheduler*
-rwxr-xr-x 1 root root 44376064 Sep 8 19:39 kubeadm*
-rwxr-xr-x 1 root root 45711360 Sep 8 19:39 kubectl*
-rwxr-xr-x 1 root root 116013432 Sep 8 19:39 kubelet*
7 启动haproxy
[root@easzlab-haproxy-keepalive-01 ~]#vim /etc/sysctl.conf
[root@easzlab-haproxy-keepalive-01 ~]#sysctl -p
net.ipv4.ip_nonlocal_bind = 1
[root@easzlab-haproxy-keepalive-01 ~]#systemctl enable --now haproxy keepalived
