环境架构与实现

[toc]

实验环境介绍

实现逻辑架构图

注: 上图node{1..7}是指nginx upstream 的各节点

ansible 环境准备

node1 ansible 环境的准备

# vim /etc/hosts
10.180.66.11 node1 node1.ssjinyao.com
10.180.66.12 node2 node2.ssjinyao.com
10.180.66.13 node3 node3.ssjinyao.com
# for i in {1..3}; do ssh-copy-id -i ~/.ssh/id_rsa.pub root@node$i ; done # 双机互信
# yum -y install epel-release
# yum -y install ansible
# vim /etc/ansible/hosts
[keepalived]
10.180.66.12
10.180.66.13

keepalived 手动安装

node2 与 node3 keepalived手动安装

# yum -y install  gcc \
            openssl-devel \
            libnl3-devel \
            ipset-devel \
            iptables-devel \
            libnfnetlink-devel \
            net-snmp-devel  \
            procps-ng  \
            psmisc  \
            lsof \
            grep
# ntpdate ntp.aliyun.com
# timedatectl set-timezone Asia/Shanghai
# cd /usr/local/src
# 上传keepalived-2.0.6.tar.gz

没有keepavlied包的当然也可以在keepalived 官网下载

# cd /usr/local/src/keepalived-2.0.6
# ./configure --prefix=/usr/local/keepalived 
# make && make install
# mkdir /var/log/keepalived/
# vim /etc/rsyslog.conf  # 最后加入
local0.*                                                /var/log/keepalived/keepalived.log
# 
# systemctl restart rsyslog
# vim /usr/local/keepalived/etc/sysconfig/keepalived # 将最后一行改为
KEEPALIVED_OPTIONS="-D -S 0 -f /usr/local/keepalived/etc/keepalived/keepalived.conf"

node2 keepalived 的配置

# vim /usr/local/keepalived/etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   }

vrrp_script chk_nginx {
    script "/bin/bash /usr/local/keepalived/etc/keepalived/nginx_check.sh"
    interval 5
    weight -20
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 55
    priority 80
    nopreempt
    advert_int 1
    notify_master "/usr/local/keepalived/etc/keepalived/message.sh  master"
    notify_backup "/usr/local/keepalived/etc/keepalived/message.sh  backup"
    notify_fault  "/usr/local/keepalived/etc/keepalived/message.sh  fault"
    unicast_src_ip 10.180.66.12
    unicast_peer {
     10.180.66.13
    }
    track_script {
       chk_nginx
    }
    authentication {
        auth_type PASS
        auth_pass testLLb
    }

    virtual_ipaddress {
      10.180.66.100
    }
}

node3 keepalived 的配置

# vim /usr/local/keepalived/etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   }

vrrp_script chk_nginx {
    script "/bin/bash /usr/local/keepalived/etc/keepalived/nginx_check.sh"
    interval 5
    weight -20
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 55
    priority 80
    nopreempt
    advert_int 1
    notify_master "/usr/local/keepalived/etc/keepalived/message.sh  master"
    notify_backup "/usr/local/keepalived/etc/keepalived/message.sh  backup"
    notify_fault  "/usr/local/keepalived/etc/keepalived/message.sh  fault"
    unicast_src_ip 10.180.66.13
    unicast_peer {
     10.180.66.12
    }
    track_script {
       chk_nginx
    }
    authentication {
        auth_type PASS
        auth_pass testLLb
    }

    virtual_ipaddress {
      10.180.66.100
    }
}

安装启用nginx

# yum -y install nginx 
# systemctl start nginx 
# systemctl enable nginx 
node2 # echo "node2.ssjinyao.com" > /usr/share/nginx/html/index.html
node3 # echo "node3.ssjinyao.com" > /usr/share/nginx/html/index.html
# 至于配置nginx的负载均衡之前写的文章有提到过

node2,node3 缩写keepalived nginx服务监测脚本

# vim /usr/local/keepalived/etc/keepalived/nginx_check.sh
#!/bin/bash

package() {
    rpm -qf /bin/ps &> /dev/null || yum -y install procps-ng &> /dev/null
    rpm -qf /bin/grep &> /dev/null || yum -y install grep &> /dev/null
    rpm -qf /usr/bin/killall & > /dev/null || yum -y install psmisc &> /dev/null
}
package

nginx_state () {
    ps aux | grep nginx &> /dev/null && ps -C nginx --no-header &> /dev/null
}
nginx_state ; nginx_state_num=$? ; echo $nginx_state_num

nginx_port () {
       netstat -tnlup | egrep ":80\>" &> /dev/null
}

kill_keepalived () {
      kill -9 `cat /var/run/keepalived.pid`
}

if [ $nginx_state_num -ne 0 ] ; then
   echo "`date +"%Y-%m-%d-%H-%M-%S"` nginx_status_one_check problem " >> /var/log/keepalived/check_nginx.log
   nginx_port ; nginx_port_num=$? ; echo $nginx_port_num
   if [ $nginx_port_num -ne 0 ] ; then
       echo "`date +"%Y-%m-%d-%H-%M-%S"` nginx_port_one_check problem" >> /var/log/keepalived/check_nginx.log
    systemctl restart nginx
        sleep 2
   fi
fi
nginx_state ; nginx_state_num=$? ; echo $nginx_state_num
if [ $nginx_state_num -ne 0 ] ; then
    echo "`date +"%Y-%m-%d-%H-%M-%S"` restart nginx and nginx_status_two_check problem " >> /var/log/keepalived/check_nginx.log
    nginx_port ; nginx_port_num=$? ; echo $nginx_port_num
    if [ $nginx_port_num -ne 0 ] ; then
        echo "`date +"%Y-%m-%d-%H-%M-%S"` restart nginx and nginx_port_two_check problem " >> /var/log/keepalived/check_nginx.log
        kill_keepalived
   fi
fi

注: keepalived 中调用命令时不要用lsof;
当访问量较高时，lsof 查询进程很慢，会造成keepalived频繁切换;

node2,node3 上的keepalived 现在可以启动了

# systemctl start keepalived
# systemctl stop  keepalived
注:需要关闭selinux 和iptables 
# iptables -F
# setenforce 0 

测试效果
目前ip 在node2上

tcmpdump 监测单播心跳信息

当把node2上的nginx停止时


当node2上的nginx用监测脚本无法自动恢复时


当node2 nginx恢复时


可以多次服务宕了时，规范划的检查nginx的启动日志

ansible 编写keepalived 自动化roles

在 node1 上编写 keepalived 自动化脚本

# tree
.
├── keepalived.yml
└── roles
    └── keepalived
        ├── files
        │   ├── keepalived
        │   ├── keepalived-2.0.6.tar.gz
        │   ├── keepalived.conf.bak
        │   ├── message.sh
        │   ├── nginx_check.sh
        │   └── rsyslog.conf
        ├── tasks
        │   └── main.yml
        ├── templates
        │   └── keepalived.conf
        └── vars
            └── main.yml

6 directories, 10 files

# vim keepalived.yml
---
- hosts: "{{ keepalivedhosts }}"
  remote_user: root
  roles:
  - keepalived
# vim roles/keepalived/tasks/main.yml
---
  - name: set off selinux
    shell: setenforce 0
  - name: set of iptables
    shell: iptables -F
  - name: install required packges
    yum: name={{ item }} state=present
    with_items:
    - gcc
    - openssl-devel
    - libnl3-devel
    - ipset-devel
    - iptables-devel
    - libnfnetlink-devel
    - net-snmp-devel
    - procps-ng
    - psmisc
    - lsof
    - grep

  - name: ntpdate time to keepavlied hosts
    shell: ntpdate ntp.aliyun.com
  - name: site time zone to Asia/Shanghai
    shell: timedatectl set-timezone Asia/Shanghai
  - name: copy keepalived_file to remote servers
    copy: src=keepalived-{{ keepalivedversion }}.tar.gz  dest={{ keepalivedsrc }}
  - name: decompression keepalived Source code package
    shell: cd /usr/local/src/ && tar -xvf keepalived-2.0.6.tar.gz
  - name: make install keepalived
    shell: cd /usr/local/src/keepalived-{{ keepalivedversion }} && ./configure --prefix=/usr/local/keepalived && make && make install
  - name: mkdir keepalived log floder
    shell: mkdir {{ keepalivedlogpath }} -pv
  - name: copy rsyslog config to keepalived
    copy: src=rsyslog.conf dest=/etc
  - name: restart rsyslog
    service: name=rsyslog state=restarted
  - name: copy keepalived  sysconfig file to remote server
    copy: src=keepalived  dest=/usr/local/keepalived/etc/sysconfig
  - name: copy keepalived nginx monitor bash script to remote server
    copy: src=nginx_check.sh  dest={{ keepalivedpath }}etc/keepalived/
  - name: copy keepalived nginx messge bash script to remote server
    copy: src=message.sh  dest={{ keepalivedpath }}etc/keepalived/ mode=755
  - name: copy keepalived config file to remote server
    template: src=keepalived.conf dest={{ keepalivedpath }}etc/keepalived/
# vim roles/keepalived/vars/main.yml
---
keepalivedhosts: keepalived
keepalivedversion: 2.0.6
keepalivedpath: /usr/local/keepalived/
keepalivedsrc: /usr/local/src/
keepalivedlogpath: /var/log/keepalived/
priority: 100
unicast_src_ip: keepalived
unicast_peer1: 10.180.66.13
unicast_peer2: 10.180.66.12
virtual_ipaddress: 10.180.66.100

# vim roles/keepalived/templates/keepalived.conf
 ! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   }

vrrp_script chk_nginx {
    script "/bin/bash /usr/local/keepalived/etc/keepalived/nginx_check.sh"
    interval 5
    weight -20
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 55
    priority {{ priority }}
    nopreempt
    advert_int 1
    notify_master "/usr/local/keepalived/etc/keepalived/message.sh  master"
    notify_backup "/usr/local/keepalived/etc/keepalived/message.sh  backup"
    notify_fault  "/usr/local/keepalived/etc/keepalived/message.sh  fault"
    unicast_src_ip xxx.xx.xxx.xx
    unicast_peer {
     xxx.xx.xxx.xx
    }
    track_script {
       chk_nginx
    }
    authentication {
        auth_type PASS
        auth_pass zE2kNsRQ
    }

    virtual_ipaddress {
    xxx.xx.xxx.xxx
    }
}
# 注：为避免意外停止生产中的nginx服务，在生产中，最好先把有关nginx监测配置的注释了
# vim roles/keepalived/files/message.sh
#!/bin/bash
/usr/bin/echo "`date +"%Y-%m-%d-%H-%M-%S"` node2 notify $1 " >> /var/log/keepalived/shell-message.log
# 其余文keepalived、keepalived-2.0.6.tar.gz、
keepalived.conf.bak、message.sh、nginx_check.sh、rsyslog.conf件与以上的使用的一样

接下来把node2,node3恢复快照到初始状态跑一次试试

为避免生产中出现突发情况，检查变更完keepalived 主配置文件后手动启动服务

rsync+inotify nginx配置文件实时同步

主服务器配置10.180.66.13

1、 安装rsync

# cd /usr/local/src
# 上传rsync源码包
# tar -xvf rsync-3.0.9.tar.gz
# cd /usr/local/src/rsync-3.0.9
# ./configure --prefix=/usr/local/rsync
# make 
# make install

2、 建立密码认证文件

# echo 'Tpz99YJV1p' >> /usr/local/rsync/rsync.passwd
# cd /usr/local/rsync/
# chown root.root rsync.passwd
# chmod 600 rsync.passwd
# ll rsync.passwd # 查看文件权限是否正确

3、 安装 inotify

# cd /usr/local/src/
# 上传inotify 源码包
# tar -xvf inotify-tools-3.14.tar.gz
# cd /usr/local/src/inotify-tools-3.14
# ./configure --prefix=/usr/local/inotify
# make
# make install

4、创建 rsync复制脚本(要注意同步的方向与操作)

#!/bin/bash
host=10.180.66.13
src=/etc/nginx/
des=web
user=usernode
src_ssl=/tmp/
des_ssl=ssl
/usr/local/inotify/bin/inotifywait -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f%e' -e modify,delete,create,attrib $src $src_ssl | while read files
do
/usr/local/rsync/bin/rsync -vzrtopg --delete --progress --password-file=/usr/local/rsync/rsync.passwd $src $user@$host::$des
/usr/local/rsync/bin/rsync -vzrtopg --delete --progress --password-file=/usr/local/rsync/rsync.passwd $src_ssl $user@$host::$des_ssl
echo "${files} was rsynced" &>> /var/log/rsync.log
done

# echo 'bash /root/rsync.sh &> /dev/null &' >> /etc/rc.local
# chmod +x /etc/rc.d/rc.local  

备服务器同步配置10.180.66.13

1、 安装rsync(备服务器只安装rsync，也就是说同步文件的目标服务)

# cd /usr/local/src
# 上传rsync源码包
# tar -xvf rsync-3.0.9.tar.gz
# cd /usr/local/src/rsync-3.0.9
# ./configure --prefix=/usr/local/rsync
# make 
# make install

2、 建立用户与密码认证文件

# cd /usr/local/rsync/
# echo 'usernode:Tpz99YJV1p' >> rsync.passwd
# chown root.root rsync.passwd
# chmod 600 rsync.passwd
# ll # 查看文件权限是否正确

3、 建立rsync的启动配置文件

# cd /usr/local/rsync
# vim rsync.conf

uid = root
gid = root
use chroot = no
max connections = 10
strict modes = yes
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /var/log/rsyncd.log
[web]
path = /etc/nginx/
comment = web file
ignore errors
read only = no
write only = no
hosts allow = 10.180.66.12
hosts deny = *
list = false
uid = root
gid = root
auth users = usernode
secrets file = /usr/local/rsync/rsync.passwd

[ssl]
path = /tmp/
comment = ssl file
ignore errors
read only = no
write only = no
hosts allow = 10.180.66.12
hosts deny = *
list = false
uid = root
gid = root
auth users = usernode
secrets file = /usr/local/rsync/rsync.passwd

#  /usr/local/rsync/bin/rsync --daemon --config=/usr/local/rsync/rsync.conf

# echo '/usr/local/rsync/bin/rsync --daemon --config=/usr/local/rsync/rsync.conf' >> /etc/rc.d/rc.local
# chmod +x /etc/rc.d/rc.local

在10.180.66.12 开始往 10.180.66.13上面同步

# bash /root/rsync.sh &> /dev/null &  注: shell

配置同步结果如下

注意:
nginx 同步配置的源地址与目标地址一定要注意，如果对以上配置不是很清楚，一定要测试无误后
再上生产环境， 不要把生产中的配置替换了
keepalived 在自动化初始化时，不要启用nginx监测，另外注意在生产中的虚拟ip冲突;