# 在所有lb节点，安装pacemaker组件，启动pcsd服务，并配置hacluster账户
$ yum install -y pacemaker pcs psmisc policycoreutils-python
$ systemctl start pcsd.service
$ systemctl enable pcsd.service
$ echo <some_password> | passwd hacluster --stdin

# 在任意一个节点初始化lb集群
$ pcs cluster auth ip1 ip2 ip3
Username: hacluster
Password:
server-33: Authorized
server-34: Authorized
server-35: Authorized
$ pcs cluster setup --name lb_cluster ip1 ip2 ip3

# 在所有lb节点，启动corosync & pacemaker服务
$ systemctl start corosync.service
$ systemctl enable corosync.service
$ systemctl start pacemaker.service
$ systemctl enable pacemaker.service


# 配置lb集群的特性
$ pcs property set stonith-enabled=false
$ pcs property set no-quorum-policy=ignore
$ pcs property set start-failure-is-fatal=false
$ pcs resource defaults resource-stickiness=10


# 创建vip与haproxy资源，并添加限制，确保p_vip运行在haproxy服务正常的节点
$ pcs resource create p_vip ocf:heartbeat:IPaddr2 ip=ip4 cidr_netmask=24 op monitor interval=2s
$ pcs resource create p_haproxy systemd:haproxy op monitor interval=2s --clone
$ pcs constraint colocation add p_vip with p_haproxy-clone --force
$ pcs resource meta p_vip migration-threshold=3 failure-timeout=60s

# 下面是pacemaker+corosync+galera的安装过程
# 在此之前先创建集群，创建方法参考pacemaker+corosync+haproxy
# 安装必要的包
$ yum install -y MariaDB-server MariaDB-client percona-xtrabackup socat #MariaDB的版本不要是5.0.x  ，使用10.1.x,否则会报错
$ yum install -y pacemaker pcs psmisc policycoreutils-python
$ yum install -y openstack-resource-agent


# 创建用户wsrep_sst
$ mysql -e 'grant all on *.* to "wsrep_sst"@"%" identified by "DJ1JAs8z"'
$ mysql -e 'grant all on *.* to "wsrep_sst"@"localhost" identified by "DJ1JAs8z"'


# 写入配置 /etc/my.cnf.d/wsrep.cnf，注意后缀是cnf，完成garela集群的配置
[mysqld]
binlog_format=row
wsrep_on=ON
wsrep_cluster_address="gcomm://ip1,ip2,ip3?pc.wait_prim=no"  # 三台 MariaDB 的 IP
wsrep_provider=/usr/lib64/galera/libgalera_smm.so
wsrep_cluster_name="openstack"
wsrep_slave_threads=4
wsrep_sst_method=xtrabackup-v2
wsrep_sst_auth=wsrep_sst:DJ1JAs8z
wsrep_node_address=ip1      # 本机的地址
[xtrabackup]
parallel=2
[sst]
streamfmt=xbstream
transferfmt=socat
sockopt=,nodelay,sndbuf=1048576,rcvbuf=1048576


# 配置pacemaker管理Galera集群
$ pcs resource create p_mysql ocf:mysql-wss binary=/usr/sbin/mysqld test_passwd=DJ1JAs8z test_user=wsrep_sst socket=/var/lib/mysql/mysql.sock --disabled --clone
# 注意，这一步的mysql-wss可以在github上搜到，需要在/usr/lib/ocf/resource.d/放入mysql-wss
$ pcs resource update p_mysql op monitor interval=90 timeout=300
$ pcs resource update p_mysql op stop interval=0 timeout=120
$ pcs resource update p_mysql op start interval=0 timeout=300

# 启动数据库服务
$ pcs resource enable p_mysql

# 在galera 节点安装xinetd服务，并在/etc/xinetd.d/目录下创建galeracheck:


service galeracheck
{
        port            = 49000
        disable        = no
        socket_type    = stream
        protocol        = tcp
        wait            = no
        user            = nobody
        group          = nobody
        groups          = yes
        server          = /usr/bin/galeracheck
        bind            = 0.0.0.0
        only_from      = 0.0.0.0
        per_source      = UNLIMITED
        cps            = 512 10
        flags          = IPv4
        instances      = UNLIMITED
}

# galeracheck的来源及配置项：https://github.com/olafz/percona-clustercheck

# 最后 /etc/haproxy/haproxy.cfg:
listen mysql
  bind 0.0.0.0:3307
  mode tcp
  balance source
  option tcplog
  option clitcpka
  option srvtcpka
  option httpchk
  timeout client 48h
  timeout server 48h
  server mysql-2 192.168.0.7:3306 check port 49000 inter 5000 rise 2 fall 3 backup
  server mysql-3 192.168.0.8:3306 check port 49000 inter 5000 rise 2 fall 3 backup
  server mysql-1 192.168.0.6:3306 check port 49000 inter 10s fastinter 2s downinter 3s rise 3 fall 2s