orenoblog

エンジニアになりたいExcel方眼紙erの物語

HAProxyで実現するGalera Clusterノードのサービス脳筋的無停止メンテナンス

galera haproxy

HAProxyのエンドポイントとなるGalera Clusterをメンテモードにすることで、 Galera Clusterへのアクセスを停止することができ、サービス無停止でのメンテナンスが実現できる。

利用例

※パッケージもローリングアップデートでできるかな

図

f:id:buta9999:20150208143652p:plain

切替方法

  • ロードバランサの状態確認
[root@ip-192-168-0 ~]# echo "show stat"|socat stdio /var/lib/haproxy/haproxy.socket
# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
admin_page,FRONTEND,,,0,0,40000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,2,0,,,,0,0,0,0,,,,0,0,0,0,0,0,,0,0,0,,,0,0,0,0,,,,,,,,
admin_page,BACKEND,0,0,0,0,4000,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,313325,0,,1,2,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
haproxy-galera,FRONTEND,,,0,1,40000,1054,1409,4234,0,0,0,,,,,OPEN,,,,,,,,,1,3,0,,,,0,0,0,2,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
haproxy-galera,192.168.0.218,0,0,0,1,4000,348,461,1564,,0,,0,0,0,0,UP,100,1,0,1,1,310458,2867,128,1,3,1,,348,,2,0,,1,L7OK,200,8,,,,,,,0,,,,345,0,,,,,612,OK,,0,0,0,1,
haproxy-galera,192.168.0.219,0,0,0,1,4000,348,461,1276,,0,,0,0,0,0,UP,100,1,0,1,1,310406,2918,128,1,3,2,,348,,2,0,,1,L7OK,200,8,,,,,,,0,,,,345,0,,,,,12,OK,,0,0,0,1,
haproxy-galera,192.168.0.220,0,0,0,1,4000,348,487,1394,,0,,0,0,0,0,UP,100,1,0,1,1,310406,2917,128,1,3,3,,348,,2,0,,1,L7OK,200,8,,,,,,,0,,,,345,0,,,,,312,OK,,0,0,0,1,
haproxy-galera,BACKEND,0,0,0,1,4000,1054,1409,4234,0,0,,10,0,0,0,UP,300,3,0,,1,310458,2865,,1,3,0,,1044,,1,0,,2,,,,,,,,,,,,,,1035,0,0,0,0,0,12,,,0,0,0,1,
  • メンテナンスモードに変更
[root@ip-192-168-0 ~]# echo "disable server haproxy-galera/192.168.0.218"|socat stdio /var/lib/haproxy/haproxy.socket

[root@ip-192-168-0 ~]# echo "show stat"|socat stdio /var/lib/haproxy/haproxy.socket
# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
admin_page,FRONTEND,,,0,0,40000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,2,0,,,,0,0,0,0,,,,0,0,0,0,0,0,,0,0,0,,,0,0,0,0,,,,,,,,
admin_page,BACKEND,0,0,0,0,4000,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,313353,0,,1,2,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
haproxy-galera,FRONTEND,,,0,1,40000,1054,1409,4234,0,0,0,,,,,OPEN,,,,,,,,,1,3,0,,,,0,0,0,2,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
★haproxy-galera,192.168.0.218,0,0,0,1,4000,348,461,1564,,0,,0,0,0,0,MAINT,100,1,0,1,2,2,2869,128,1,3,1,,348,,2,0,,1,L7OK,200,8,,,,,,,0,,,,345,0,,,,,640,,,0,0,0,1,
haproxy-galera,192.168.0.219,0,0,0,1,4000,348,461,1276,,0,,0,0,0,0,UP,100,1,0,1,1,310434,2918,128,1,3,2,,348,,2,0,,1,L7OK,200,8,,,,,,,0,,,,345,0,,,,,40,OK,,0,0,0,1,
haproxy-galera,192.168.0.220,0,0,0,1,4000,348,487,1394,,0,,0,0,0,0,UP,100,1,0,1,1,310434,2917,128,1,3,3,,348,,2,0,,1,L7OK,200,8,,,,,,,0,,,,345,0,,,,,340,OK,,0,0,0,1,
haproxy-galera,BACKEND,0,0,0,1,4000,1054,1409,4234,0,0,,10,0,0,0,UP,200,2,0,,1,310486,2865,,1,3,0,,1044,,1,0,,2,,,,,,,,,,,,,,1035,0,0,0,0,0,40,,,0,0,0,1,
  • メンテナンスモードの確認

メンテ状態になったGalera Clusterへアクセスが発生しない

[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.219 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.220 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.219 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.220 |
+--------------------+---------------+
  • Galera Clusterのメンテナンス

HAProxyから外れたGalera Clusterを停止する。

Galera Clusterのインスタンスタイプを変更する

Galera Clusterを起動してメンバーノードとしてジョインさせる(MySQLを起動するとSSTにより自動でsyncされる)

  • HAProxyのメンテナンスモード解除
[root@ip-192-168-0 ~]# echo "enable server haproxy-galera/192.168.0.218"|socat stdio /var/lib/haproxy/haproxy.socket
[root@ip-192-168-0 ~]# echo "show stat"|socat stdio /var/lib/haproxy/haproxy.socket
# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
admin_page,FRONTEND,,,0,0,40000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,2,0,,,,0,0,0,0,,,,0,0,0,0,0,0,,0,0,0,,,0,0,0,0,,,,,,,,
admin_page,BACKEND,0,0,0,0,4000,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,313883,0,,1,2,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
haproxy-galera,FRONTEND,,,0,1,40000,1060,2299,12289,0,0,0,,,,,OPEN,,,,,,,,,1,3,0,,,,0,0,0,2,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
haproxy-galera,192.168.0.218,0,0,0,1,4000,348,461,1564,,0,,0,0,0,0,UP,7,1,0,1,2,4,3395,128,1,3,1,6,348,,2,0,,1,L7OK,200,8,,,,,,,0,,,,345,0,,,,,1170,OK,,0,0,0,1,
haproxy-galera,192.168.0.219,0,0,0,1,4000,351,1009,8451,,0,,0,0,0,0,UP,100,1,0,1,1,310964,2918,128,1,3,2,,351,,2,0,,1,L7OK,200,8,,,,,,,0,,,,345,0,,,,,135,OK,,0,0,0,38,
haproxy-galera,192.168.0.220,0,0,0,1,4000,351,829,2274,,0,,0,0,0,0,UP,100,1,0,1,1,310964,2917,128,1,3,3,,351,,2,0,,1,L7OK,200,8,,,,,,,0,,,,346,0,,,,,256,OK,,0,0,0,1,
haproxy-galera,BACKEND,0,0,0,1,4000,1060,2299,12289,0,0,,10,0,0,0,UP,207,3,0,,1,311016,2865,,1,3,0,,1050,,1,0,,2,,,,,,,,,,,,,,1036,0,0,0,0,0,135,,,0,0,0,38,
  • HAProxyのロードバランス確認
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.219 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.220 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.219 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.220 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.219 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.218 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.220 |
+--------------------+---------------+
[root@ip-192-168-0 ~]#  mysql -u root -p'<password>' -h 192.168.0.120 -P 33306 -e "show variables like 'wsrep_node_address'"
+--------------------+---------------+
| Variable_name      | Value         |
+--------------------+---------------+
| wsrep_node_address | 192.168.0.219 |
+--------------------+---------------+

以上です