Haporxy+Keepalived

#此处脚本在脚本章节(注意如果网卡名不是eth那么会提示服务安装失败,是因为启动keepalived失败,后期修改配置正常启动就可以)
[root@easzlab-haproxy-keepalive-01 ~]#ls *keep*
install_keepalived.sh
[root@easzlab-haproxy-keepalive-01 ~]#bash install_keepalived.sh
....
KEEPALIVED编译安装成功                                     [  OK  ]
Keepalived 服务安装成功!                                   [  OK  ]

#重复以上步骤安装easzlab-haproxy-keepalive-02和easzlab-haproxy-keepalive-03

#此处脚本在脚本章节有
[root@easzlab-haproxy-keepalive-01 ~]#bash install_haproxy.sh 
请输入haproxy版本(2.6.6):2.6.6
....
HAPROXY编译安装成功                                        [  OK  ]
Created symlink /etc/systemd/system/multi-user.target.wants/haproxy.service → /lib/systemd/system/haproxy.service.
HAPROXY安装完成!                                           [  OK  ]
-------------------------------------------------------------------
请访问链接: http://10.0.0.110:9999/haproxy-status
用户和密码: admin/123456

#重复以上步骤安装easzlab-haproxy-keepalive-02和easzlab-haproxy-keepalive-03

[root@easzlab-haproxy-keepalive-01 ~]#vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
  
global_defs {
   router_id keepalived01
   vrrp_skip_check_adv_addr
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
include /etc/keepalived/conf.d/*.conf

[root@easzlab-haproxy-keepalive-01 ~]#mkdir /etc/keepalived/conf.d
[root@easzlab-haproxy-keepalive-01 keepalived]#cd /etc/keepalived/conf.d/

[root@easzlab-haproxy-keepalive-01 keepalived]#vim /etc/keepalived/conf.d/sh.harbor.com.conf 
vrrp_script check_haproxy {
    script "/etc/keepalived/check_harproxy.sh"
    interval 1
    weight -30
    fall 3
    rise 5
    timeout 2
}
vrrp_instance VI_1 {
    state MASTER
    interface enp1s0
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.20.0.201 dev enp6s0 label enp6s0:1
    }
    unicast_src_ip 10.0.0.110
    unicast_peer{
        10.0.0.111
        10.0.0.112
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    track_script {
        check_haproxy
    }
}

[root@easzlab-haproxy-keepalive-01 keepalived]#vim notify.sh 
#!/bin/bash
#
#********************************************************************
#Author:            shuhong
#QQ:                985347841
#Date:              2022-10-26
#FileName:          notify.sh
#URL:               hhhhh
#Description:       The test script
#Copyright (C):     2022 All rights reserved
#********************************************************************
contact='985347841@qq.com'
email_send='985347841@qq.com'
email_passwd='hgxbxniljwlfbdii'
email_smtp_server='smtp.qq.com'

. /etc/os-release

msg_error() {
  echo -e "\033[1;31m$1\033[0m"
}

msg_info() {
  echo -e "\033[1;32m$1\033[0m"
}

msg_warn() {
  echo -e "\033[1;33m$1\033[0m"
}

color () {
    RES_COL=60
    MOVE_TO_COL="echo -en \\033[${RES_COL}G"
    SETCOLOR_SUCCESS="echo -en \\033[1;32m"
    SETCOLOR_FAILURE="echo -en \\033[1;31m"
    SETCOLOR_WARNING="echo -en \\033[1;33m"
    SETCOLOR_NORMAL="echo -en \E[0m"
    echo -n "$1" && $MOVE_TO_COL
    echo -n "["
    if [ $2 = "success" -o $2 = "0" ] ;then
        ${SETCOLOR_SUCCESS}
        echo -n $"  OK  "    
    elif [ $2 = "failure" -o $2 = "1"  ] ;then 
        ${SETCOLOR_FAILURE}
        echo -n $"FAILED"
    else
        ${SETCOLOR_WARNING}
        echo -n $"WARNING"
    fi
    ${SETCOLOR_NORMAL}
    echo -n "]"
    echo 
}


install_sendemail () {
    if [[ $ID =~ rhel|centos|rocky ]];then
        rpm -q sendemail &> /dev/null ||  yum install -y sendemail
    elif [ $ID = 'ubuntu' ];then
        dpkg -l |grep -q sendemail  || { apt update; apt install -y libio-socket-ssl-perl libnet-ssleay-perl sendemail ; } 
    else
        color "不支持此操作系统，退出!" 1
        exit
    fi
}

send_email () {
    local email_receive="$1"
    local email_subject="$2"
    local email_message="$3"
    sendemail -f $email_send -t $email_receive -u $email_subject -m $email_message -s $email_smtp_server -o message-charset=utf-8 -o tls=yes -xu $email_send -xp $email_passwd
    [ $? -eq 0 ] && color "邮件发送成功!" 0 || color "邮件发送失败!" 1 
}

notify() {
    if [[ $1 =~ ^(master|backup|fault)$ ]];then
        mailsubject="$(hostname) to be $1, vip floating"
        mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
        send_email "$contact" "$mailsubject" "$mailbody"
   else
        echo "Usage: $(basename $0) {master|backup|fault}"
        exit 1
   fi
}

install_sendemail 
notify $1

[root@easzlab-haproxy-keepalive-01 keepalived]#vim check_haproxy.sh 
#!/bin/bash
# 
#********************************************************************
#Author:            shuhong
#QQ:                985347841
#Date:              2022-10-27
#FileName:          check_haproxy.sh
#URL:               hhhhh
#Description:       The test script
#Copyright (C):     2022 All rights reserved
#********************************************************************
curl -s -u admin:123456 http://localhost:1080/haproxy-status  |egrep -o "Statistics Report for pid" &> /dev/null

[root@easzlab-haproxy-keepalive-01 keepalived]#chmod +x check_haproxy.sh notify.sh 
[root@easzlab-haproxy-keepalive-01 keepalived]#systemctl restart keepalived.service 
[root@easzlab-haproxy-keepalive-01 keepalived]#hostname -I
10.0.0.110 172.20.0.110 172.20.0.201 

[root@easzlab-haproxy-keepalive-01 ~]#cd /etc/haproxy/
[root@easzlab-haproxy-keepalive-01 haproxy]#mkdir conf.d


[root@easzlab-haproxy-keepalive-01 haproxy]#vim /lib/systemd/system/haproxy.service 
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg  -f /etc/haproxy/conf.d/ -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg  -f /etc/haproxy/conf.d/ -p /var/lib/haproxy/haproxy.pid
ExecReload=/bin/kill -USR2 

[Install]
WantedBy=multi-user.target
[root@easzlab-haproxy-keepalive-01 haproxy]#systemctl daemon-reload 
[root@easzlab-haproxy-keepalive-01 haproxy]#systemctl restart haproxy.service 

[root@easzlab-haproxy-keepalive-01 haproxy]#cat haproxy.cfg 
global
maxconn 100000
stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
uid 99
gid 99
daemon

pidfile /var/lib/haproxy/haproxy.pid
log 127.0.0.1 local3 info

defaults
option http-keep-alive
option forwardfor

option httplog       #日志类别,采用httplog
option dontlognull   #不记录健康检查日志信息  
retries 2            #2次连接失败不可用
option httpclose     #请求完毕后主动关闭http通道
option abortonclose  #服务器负载很高，自动结束比较久的链接  
timeout client 1m    #客户端超时  
timeout server 31m   #服务器超时  
timeout check 10s    #心跳检测超时  
maxconn 100000
mode http


listen stats
    mode http
    bind 0.0.0.0:9999
    stats enable
    log global
    stats uri /haproxy-status
    stats auth admin:123456
    stats refresh 30s
    stats hide-version       #隐藏版本信息  

[root@easzlab-haproxy-keepalive-01 haproxy]#vim conf.d/sh.harbor.com.cfg 
listen harbor
  bind 172.20.0.201:80
  mode http
  log  global
  option httpchk HEAD / HTTP/1.1\r\nHost:\ 172.20.0.201
  cookie harborcookie insert nocache indirect
  server harbor01 10.0.0.150:80 weight 1  check inter 3000 fall 2 rise 5 cookie harbor01
  server harbor02 10.0.0.151:80 weight 1  check inter 3000 fall 2 rise 5 cookie harbor02

[root@easzlab-haproxy-keepalive-01 haproxy]#vim /etc/sysctl.conf 
[root@easzlab-haproxy-keepalive-01 haproxy]#sysctl -p
net.ipv4.ip_nonlocal_bind = 1

[root@easzlab-haproxy-keepalive-01 haproxy]#systemctl restart haproxy.service 

[root@easzlab-haproxy-keepalive-01 keepalived]#scp check_haproxy.sh notify.sh keepalived.conf 10.0.0.111:/etc/keepalived/
[root@easzlab-haproxy-keepalive-02 keepalived]#mkdir conf.d
[root@easzlab-haproxy-keepalive-01 keepalived]#scp conf.d/sh.harbor.com.conf 10.0.0.111:/etc/keepalived/conf.d/

[root@easzlab-haproxy-keepalive-02 conf.d]#vim sh.harbor.com.conf 
vrrp_script check_haproxy {
    script "/etc/keepalived/check_harproxy.sh"
    interval 1
    weight -30
    fall 3
    rise 5
    timeout 2
}
vrrp_instance VI_1 {
    state BACKUP
    interface enp1s0
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.20.0.201 dev enp6s0 label enp6s0:1
    }
    unicast_src_ip 10.0.0.111
    unicast_peer{
        10.0.0.110
        10.0.0.112
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    track_script {
        check_haproxy
    }
}
[root@easzlab-haproxy-keepalive-02 conf.d]#systemctl restart keepalived.service 

[root@easzlab-haproxy-keepalive-01 haproxy]#scp haproxy.cfg 10.0.0.111:/etc/haproxy/
[root@easzlab-haproxy-keepalive-01 haproxy]#scp /lib/systemd/system/haproxy.service 10.0.0.111:/lib/systemd/system/haproxy.service
[root@easzlab-haproxy-keepalive-02 conf.d]#mkdir /etc/haproxy/conf.d
[root@easzlab-haproxy-keepalive-01 haproxy]#scp conf.d/sh.harbor.com.cfg 10.0.0.111:/etc/haproxy/conf.d/
[root@easzlab-haproxy-keepalive-02 conf.d]#systemctl daemon-reload 
[root@easzlab-haproxy-keepalive-02 conf.d]#systemctl restart haproxy.service 
[root@easzlab-haproxy-keepalive-02 conf.d]#vim /etc/sysctl.conf 
[root@easzlab-haproxy-keepalive-02 conf.d]#sysctl -p
net.ipv4.ip_nonlocal_bind = 1

[root@easzlab-haproxy-keepalive-01 keepalived]#scp check_haproxy.sh notify.sh keepalived.conf 10.0.0.112:/etc/keepalived/
[root@easzlab-haproxy-keepalive-03 keepalived]#mkdir conf.d
[root@easzlab-haproxy-keepalive-01 keepalived]#scp conf.d/sh.harbor.com.conf 10.0.0.112:/etc/keepalived/conf.d/
[root@easzlab-haproxy-keepalive-03 keepalived]#vim conf.d/sh.harbor.com.conf 

vrrp_script check_haproxy {
    script "/etc/keepalived/check_harproxy.sh"
    interval 1
    weight -30
    fall 3
    rise 5
    timeout 2
}
vrrp_instance VI_1 {
    state MASTER
    interface enp1s0
    virtual_router_id 66
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.20.0.201 dev enp6s0 label enp6s0:1
    }
    unicast_src_ip 10.0.0.112
    unicast_peer{
        10.0.0.110
        10.0.0.111
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    track_script {
        check_haproxy
    }
}
[root@easzlab-haproxy-keepalive-03 keepalived]#systemctl restart keepalived.service 

[root@easzlab-haproxy-keepalive-01 haproxy]#scp haproxy.cfg 10.0.0.112:/etc/haproxy/
[root@easzlab-haproxy-keepalive-01 haproxy]#scp /lib/systemd/system/haproxy.service 10.0.0.112:/lib/systemd/system/haproxy.service
[root@easzlab-haproxy-keepalive-03 keepalived]#mkdir /etc/haproxy/conf.d
[root@easzlab-haproxy-keepalive-01 haproxy]#scp conf.d/sh.harbor.com.cfg 10.0.0.112:/etc/haproxy/conf.d/
[root@easzlab-haproxy-keepalive-02 conf.d]#systemctl daemon-reload 
[root@easzlab-haproxy-keepalive-03 keepalived]#vim /etc/sysctl.conf
[root@easzlab-haproxy-keepalive-03 keepalived]#sysctl -p
net.ipv4.ip_nonlocal_bind = 1
[root@easzlab-haproxy-keepalive-03 keepalived]#systemctl restart haproxy.service 

[root@easzlab-haproxy-keepalive-01 ~]#hostname -I
10.0.0.110 172.20.0.110 172.20.0.201 

[root@easzlab-haproxy-keepalive-01 ~]#hostname -I
10.0.0.110 172.20.0.110 172.20.0.201 

[root@easzlab-haproxy-keepalive-01 ~]#systemctl stop haproxy.service
[root@easzlab-haproxy-keepalive-01 ~]#tcpdump -i enp1s0 -nn host 10.0.0.111
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on enp1s0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:12:11.389918 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
17:12:12.390993 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
17:12:13.391768 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
17:12:14.392375 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
17:12:15.393026 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
17:12:16.393625 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
17:12:17.394266 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
17:12:18.395144 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
17:12:19.395978 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 70, authtype simple, intvl 1s, length 20
17:12:20.396551 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 70, authtype simple, intvl 1s, length 20
17:12:21.397116 IP 10.0.0.110 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 70, authtype simple, intvl 1s, length 20
17:12:22.045186 IP 10.0.0.111 > 10.0.0.110: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:12:23.048449 IP 10.0.0.111 > 10.0.0.110: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:12:24.048904 IP 10.0.0.111 > 10.0.0.110: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:12:25.049335 IP 10.0.0.111 > 10.0.0.110: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20

[root@easzlab-haproxy-keepalive-02 ~]#hostname -I
10.0.0.111 172.20.0.111 172.20.0.201 

#easzlab-haproxy-keepalive-01的haproxy服务故障依旧没有恢复的情况下,easzlab-haproxy-keepalive-02的haproxy服务故障
[root@easzlab-haproxy-keepalive-02 ~]#hostname -I
10.0.0.111 172.20.0.111 172.20.0.201
[root@easzlab-haproxy-keepalive-02 ~]#systemctl stop haproxy.service 

[root@easzlab-haproxy-keepalive-02 ~]#tcpdump -i enp1s0 -nn host 10.0.0.112
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on enp1s0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:17:25.847145 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:17:26.847543 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:17:27.847937 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:17:28.848605 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:17:29.849669 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:17:30.850432 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 90, authtype simple, intvl 1s, length 20
17:17:31.851188 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 60, authtype simple, intvl 1s, length 20
17:17:32.851990 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 60, authtype simple, intvl 1s, length 20
17:17:33.852952 IP 10.0.0.111 > 10.0.0.112: VRRPv2, Advertisement, vrid 66, prio 60, authtype simple, intvl 1s, length 20
17:17:34.539181 IP 10.0.0.112 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 80, authtype simple, intvl 1s, length 20
17:17:35.542839 IP 10.0.0.112 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 80, authtype simple, intvl 1s, length 20
17:17:36.543040 IP 10.0.0.112 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 80, authtype simple, intvl 1s, length 20
17:17:37.543417 IP 10.0.0.112 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 80, authtype simple, intvl 1s, length 20
17:17:38.543676 IP 10.0.0.112 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 80, authtype simple, intvl 1s, length 20
17:17:39.543813 IP 10.0.0.112 > 10.0.0.111: VRRPv2, Advertisement, vrid 66, prio 80, authtype simple, intvl 1s, length 20
[root@easzlab-haproxy-keepalive-03 ~]#hostname -I
10.0.0.112 172.20.0.112 172.20.0.201

[root@easzlab-haproxy-keepalive-01 haproxy]#cd /etc/haproxy/
[root@easzlab-haproxy-keepalive-01 haproxy]#mkdir certs
[root@easzlab-haproxy-keepalive-01 haproxy]#cd certs/

[root@easzlab-haproxy-keepalive-01 certs]#bash crts.sh 
请输入(/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=shuhong.com):/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=sh.harbor.com
请输入申请的域名(shuhong.com):sh.harbor.com
Generating RSA private key, 4096 bit long modulus (2 primes)
..................................++++
.................................++++
e is 65537 (0x010001)
Generating RSA private key, 4096 bit long modulus (2 primes)
......++++
....++++
e is 65537 (0x010001)
Signature ok
subject=C = CN, ST = Beijing, L = Beijing, O = example, OU = Personal, CN = sh.harbor.com
Getting CA Private Key
[root@easzlab-haproxy-keepalive-01 certs]#cat sh.harbor.com.key sh.harbor.com.crt > sh.harbor.com.pem
[root@easzlab-haproxy-keepalive-01 certs]#vim ../conf.d/sh.harbor.com.cfg 
listen harbor
  bind 172.20.0.201:80
  bind 172.20.0.201:443 ssl crt /etc/haproxy/certs/sh.harbor.com.pem
  redirect scheme https if !{ ssl_fc }
  mode http
  log  global
  option httpchk HEAD / HTTP/1.1\r\nHost:\ 172.20.0.201
  cookie harborcookie insert nocache indirect                                               #此处不能使用分发cookie的方式来保持会话不丢失,因为使用docker命令登陆时,不会保存cooki信息会导致在两台harbor服务器间互跳导致登陆失败
  server harbor01 10.0.0.150:80 weight 1  check inter 3000 fall 2 rise 5 cookie harbor01
  server harbor02 10.0.0.151:80 weight 1  check inter 3000 fall 2 rise 5 cookie harbor02
[root@easzlab-haproxy-keepalive-01 certs]#systemctl restart haproxy.service 

#将文件拷贝到其他机器上重启haproxy即可
[root@easzlab-haproxy-keepalive-02 ~]#mkdir /etc/haproxy/certs
[root@easzlab-haproxy-keepalive-03 ~]#mkdir /etc/haproxy/certs

[root@easzlab-haproxy-keepalive-01 certs]#scp sh.harbor.com.pem 10.0.0.111:/etc/haproxy/certs
[root@easzlab-haproxy-keepalive-01 certs]#scp sh.harbor.com.pem 10.0.0.112:/etc/haproxy/certs

[root@easzlab-haproxy-keepalive-01 conf.d]#scp sh.harbor.com.cfg 10.0.0.111:/etc/haproxy/conf.d
[root@easzlab-haproxy-keepalive-01 conf.d]#scp sh.harbor.com.cfg 10.0.0.112:/etc/haproxy/conf.d

[root@easzlab-haproxy-keepalive-02 ~]#systemctl restart haproxy.service
[root@easzlab-haproxy-keepalive-03 ~]#systemctl restart haproxy.service