1 //Here's the error log of it.
 2 2018-08-08T08:48:18.198487+01:00 0 [Note] WSREP: Initiating SST/IST transfer on DONOR side (wsrep_sst_xtrabackup-v2 --role 'donor' --address '192.168.1.102:4444/xtrabackup_sst//1' --socket '/tmp/mysql3308.sock' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix ''  --binlog '/data/mysql/mysql3308/logs/mysql-bin' --gtid 'bd5525ab-9a15-11e8-aa0f-4b830c783fc7:21')
 3 2018-08-08T08:48:18.200272+01:00 2 [Note] WSREP: DONOR thread signaled with 0
 4 which: no socat in (/usr/sbin:/sbin:/opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/mysql/bin:/usr/local/zabbix/bin:/usr/local/zabbix/sbin)
 5     2018-08-08T06:48:18.782344Z WSREP_SST: [ERROR] ******************* FATAL ERROR ********************** 
 6     2018-08-08T06:48:18.784043Z WSREP_SST: [ERROR] socat not found in path: /usr/sbin:/sbin:/opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/mysql/bin:/usr/local/zabbix/bin:/usr/local/zabbix/sbin
 7     2018-08-08T06:48:18.785926Z WSREP_SST: [ERROR] ****************************************************** 
 8 2018-08-08T08:48:18.786496+01:00 0 [ERROR] WSREP: Process completed with error: wsrep_sst_xtrabackup-v2 --role 'donor' --address '192.168.1.102:4444/xtrabackup_sst//1' --socket '/tmp/mysql3308.sock' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix ''  --binlog '/data/mysql/mysql3308/logs/mysql-bin' --gtid 'bd5525ab-9a15-11e8-aa0f-4b830c783fc7:21': 2 (No such file or directory)
 9 2018-08-08T08:48:18.786587+01:00 0 [ERROR] WSREP: Command did not run: wsrep_sst_xtrabackup-v2 --role 'donor' --address '192.168.1.102:4444/xtrabackup_sst//1' --socket '/tmp/mysql3308.sock' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix ''  --binlog '/data/mysql/mysql3308/logs/mysql-bin' --gtid 'bd5525ab-9a15-11e8-aa0f-4b830c783fc7:21'
10 2018-08-08T08:48:18.787610+01:00 0 [Warning] WSREP: 1.0 (zlm2): State transfer to 0.0 (zlm3) failed: -2 (No such file or directory)
11 
12 [root@zlm2 08:53:31 /data/mysql/mysql3308/data]
13 #rpm -qa|grep socat
14 
15 [root@zlm2 09:11:08 /data/mysql/mysql3308/data]
16 #yum install socat
17 ... //Omitted.
18 
19 Installed:
20   socat.x86_64 0:1.7.3.2-2.el7                                                                                                                       
21 
22 Complete!

**Startup
the mysqld on node zlm4.**

 

 

 

 

 

Disguise
the binlog server is a fake master.

 

    When we add a new node into PXC
structure,it will estimate the mothed(IST/SST) to transfer data to the
new node.It’s no wonder that sst will be automatically choosed.What if a
dropped node in PXC by accident want to rejoin the cluster?Especially it
has been a long period of time after it was dropped out.We should avoid
starting up the node directly.As the SST(either xtrabackup-v2 or rsync
mode) will be choosed to resynchronize the dropped node instead of
IST.Generally speaking,SST may cause to decline the perormance of the
whole cluster.It’s recommend to use another way to deal with that kind
of issue.That is,slave replication.

 

 

 

 

**Initialized a PXC node with
proper pxc relevant parameters below.**

 1 [root@zlm2 17:12:39 /data/backup/2018-07-29_16-32-33]
 2 #cat xtrabackup_binlog_info
 3 mysql-bin.000100    476    2a4b3562-2ab6-11e8-be7a-080027de0e0e:1-12715694
 4 
 5 zlm@192.168.56.101:3306 [sysbench]>reset master;
 6 Query OK, 0 rows affected (0.00 sec)
 7 
 8 zlm@192.168.56.101:3306 [sysbench]>set @@global.gtid_purged='2a4b3562-2ab6-11e8-be7a-080027de0e0e:1-12715694';
 9 Query OK, 0 rows affected (0.00 sec)
10 
11 zlm@192.168.56.101:3306 [sysbench]>start slave io_thread;
12 Query OK, 0 rows affected (0.00 sec)
13 
14 zlm@192.168.56.101:3306 [sysbench]>show slave status\G
15 *************************** 1. row ***************************
16                Slave_IO_State: 
17                   Master_Host: 192.168.56.102
18                   Master_User: repl
19                   Master_Port: 3306
20                 Connect_Retry: 60
21               Master_Log_File: 
22           Read_Master_Log_Pos: 4
23                Relay_Log_File: relay-bin.000001
24                 Relay_Log_Pos: 4
25         Relay_Master_Log_File: 
26              Slave_IO_Running: No
27             Slave_SQL_Running: No
28               Replicate_Do_DB: 
29           Replicate_Ignore_DB: 
30            Replicate_Do_Table: 
31        Replicate_Ignore_Table: 
32       Replicate_Wild_Do_Table: 
33   Replicate_Wild_Ignore_Table: 
34                    Last_Errno: 0
35                    Last_Error: 
36                  Skip_Counter: 0
37           Exec_Master_Log_Pos: 0
38               Relay_Log_Space: 154
39               Until_Condition: None
40                Until_Log_File: 
41                 Until_Log_Pos: 0
42            Master_SSL_Allowed: No
43            Master_SSL_CA_File: 
44            Master_SSL_CA_Path: 
45               Master_SSL_Cert: 
46             Master_SSL_Cipher: 
47                Master_SSL_Key: 
48         Seconds_Behind_Master: NULL
49 Master_SSL_Verify_Server_Cert: No
50                 Last_IO_Errno: 1236
51                 Last_IO_Error: Got fatal error 1236 from master when reading data from binary log: 'The slave is connecting using CHANGE MASTER TO MASTER_AUTO_POSITION = 1, but the master has purged binary logs containing GTIDs that the slave requires.'
52                Last_SQL_Errno: 0
53                Last_SQL_Error: 
54   Replicate_Ignore_Server_Ids: 
55              Master_Server_Id: 1023306
56                   Master_UUID: e00ef9f5-6c4b-11e8-8389-080027de0e0e
57              Master_Info_File: /data/mysql/mysql3306/data/master.info
58                     SQL_Delay: 0
59           SQL_Remaining_Delay: NULL
60       Slave_SQL_Running_State: 
61            Master_Retry_Count: 86400
62                   Master_Bind: 
63       Last_IO_Error_Timestamp: 180729 18:00:08
64      Last_SQL_Error_Timestamp: 
65                Master_SSL_Crl: 
66            Master_SSL_Crlpath: 
67            Retrieved_Gtid_Set: 
68             Executed_Gtid_Set: 2a4b3562-2ab6-11e8-be7a-080027de0e0e:1-12715694
69                 Auto_Position: 1
70          Replicate_Rewrite_DB: 
71                  Channel_Name: 
72            Master_TLS_Version: 
73 1 row in set (0.00 sec)

 

 

 

Restart
instance and check table.

    The error above was due to the
parameter of “innodb_log_group_home_dir”,after I
omitted it,there was no error message any more.

Hostname IP Port OS Version MySQL Version Xtrabackup version
zlm2 192.168.1.101 3308 CentOS 7.0 PXC 5.7.22 2.4.12 Linux (x86_64)
zlm3 192.168.1.102 3308 CentOS 7.0 PXC 5.7.22 2.4.12 Linux (x86_64)
zlm4 192.168.1.103 3308 CentOS 7.0 PXC 5.7.22 2.4.12 Linux (x86_64)

**Start mysqld of PXC of the
remaining nodes.**

 

    Yesterday I implemented a three-nodes
PXC,but there were some errors when proceeding it.So I decide to
research what’s wrong with it.The operations and evidences shows
below.

 

 1 [root@zlm2 07:56:03 ~]
 2 #mkdir -p /opt/mysql
 3 
 4 [root@zlm2 07:56:51 ~]
 5 #ls -l|grep Percona
 6 -rwxr-xr-x   1 root root 252395830 Aug  7 07:54 Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102.tar.gz
 7 
 8 [root@zlm2 07:57:01 ~]
 9 #gzip -d Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102.tar.gz 
10 
11 [root@zlm2 07:57:59 ~]
12 #ls -l|grep Percona
13 -rwxr-xr-x   1 root root 1035939840 Aug  7 07:54 Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102.tar
14 
15 [root@zlm2 07:59:36 ~]
16 #tar xf Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102.tar -C /opt/mysql
17 
18 [root@zlm2 08:02:29 ~]
19 #cd /usr/local
20 
21 [root@zlm2 08:02:22 /usr/local]
22 #ln -s /opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/ mysql
23 
24 [root@zlm2 08:02:28 /usr/local]
25 #ls -l
26 total 4
27 drwxr-xr-x. 2 root   root    105 May  7 10:44 bin
28 drwxr-xr-x. 2 root   root      6 Jun 10  2014 etc
29 drwxr-xr-x. 2 root   root      6 Jun 10  2014 games
30 drwxr-xr-x. 2 root   root      6 Jun 10  2014 include
31 drwxr-xr-x. 2 root   root      6 Jun 10  2014 lib
32 drwxr-xr-x. 3 root   root     18 May  7 09:37 lib64
33 drwxr-xr-x. 2 root   root      6 Jun 10  2014 libexec
34 lrwxrwxrwx  1 root   root     36 Apr 28 14:30 mysql -> mysql-5.7.21-linux-glibc2.12-x86_64/
35 drwxr-xr-x  9 root   root   4096 Aug  7 10:00 mysql-5.7.21-linux-glibc2.12-x86_64
36 drwxrwxr-x  5 root   root     94 May  7 09:28 mysql_performance_monitor-agent
37 drwxr-xr-x. 2 root   root     18 Apr 28 16:06 sbin
38 drwxr-xr-x. 6 root   root     58 May  7 09:37 share
39 drwxr-xr-x. 2 root   root      6 Jun 10  2014 src
40 drwxr-xr-x  8 zabbix zabbix   70 Apr 28 16:45 zabbix
41 
42 [root@zlm2 08:02:32 /usr/local]
43 #rm -f mysql
44 
45 [root@zlm2 08:02:37 /usr/local]
46 #ln -s /opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/ mysql
47 
48 [root@zlm2 08:03:04 /usr/local]
49 #ls -l
50 total 0
51 drwxr-xr-x. 2 root root  21 Jul  4 03:21 bin
52 drwxr-xr-x. 2 root root   6 Jun 10  2014 etc
53 drwxr-xr-x. 2 root root   6 Jun 10  2014 games
54 drwxr-xr-x. 2 root root   6 Jun 10  2014 include
55 drwxr-xr-x. 2 root root   6 Jun 10  2014 lib
56 drwxr-xr-x. 2 root root   6 Jun 10  2014 lib64
57 drwxr-xr-x. 2 root root   6 Jun 10  2014 libexec
58 lrwxrwxrwx  1 root root  75 Aug  7 08:23 mysql -> /opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/
59 drwxr-xr-x  9 root root 120 Aug  7 08:22 mysql-5.7.21-linux-glibc2.12-x86_64
60 drwxr-xr-x. 2 root root   6 Jun 10  2014 sbin
61 drwxr-xr-x. 7 root root  71 Jul  4 03:21 share
62 drwxr-xr-x. 2 root root   6 Jun 10  2014 src
63 
64 [root@zlm2 08:32:56 /usr/local]
65 #ldd mysql/bin/mysqld
66     linux-vdso.so.1 =>  (0x00007fff96f3f000)
67     libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f1119136000)
68     libaio.so.1 => /lib64/libaio.so.1 (0x00007f1118f34000)
69     libnuma.so.1 => /lib64/libnuma.so.1 (0x00007f1118d27000)
70     libcrypt.so.1 => /lib64/libcrypt.so.1 (0x00007f1118af0000)
71     libssl.so.10 => /lib64/libssl.so.10 (0x00007f1118883000)
72     libcrypto.so.10 => /lib64/libcrypto.so.10 (0x00007f111849b000)
73     libdl.so.2 => /lib64/libdl.so.2 (0x00007f1118297000)
74     libz.so.1 => /lib64/libz.so.1 (0x00007f1118081000)
75     librt.so.1 => /lib64/librt.so.1 (0x00007f1117e78000)
76     libstdc++.so.6 => /lib64/libstdc++.so.6 (0x00007f1117b71000)
77     libm.so.6 => /lib64/libm.so.6 (0x00007f111786f000)
78     libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007f1117658000)
79     libc.so.6 => /lib64/libc.so.6 (0x00007f1117297000)
80     /lib64/ld-linux-x86-64.so.2 (0x00007f1119359000)
81     libfreebl3.so => /lib64/libfreebl3.so (0x00007f1117094000)
82     libgssapi_krb5.so.2 => /lib64/libgssapi_krb5.so.2 (0x00007f1116e47000)
83     libkrb5.so.3 => /lib64/libkrb5.so.3 (0x00007f1116b64000)
84     libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00007f1116960000)
85     libk5crypto.so.3 => /lib64/libk5crypto.so.3 (0x00007f111672d000)
86     libkrb5support.so.0 => /lib64/libkrb5support.so.0 (0x00007f111651e000)
87     libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007f1116319000)
88     libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f11160ff000)
89     libselinux.so.1 => /lib64/libselinux.so.1 (0x00007f1115eda000)
90     libpcre.so.1 => /lib64/libpcre.so.1 (0x00007f1115c78000)
91     liblzma.so.5 => /lib64/liblzma.so.5 (0x00007f1115a53000)
92     
93 ... //Omitted the procedure of installation on the other nodes.
 1 [root@zlm1 16:56:54 /data/backup]
 2 #mysqladmin shutdown
 3 
 4 [root@zlm1 16:57:05 /data/backup]
 5 #ps aux|grep mysqld
 6 mysql     3804  0.0 20.7 1072396 210620 pts/0  Sl   15:52   0:02 mysqld --defaults-file=/data/mysql/mysql3306/my3306.cnf
 7 root      4512  0.0  0.0 112640   960 pts/2    R+   16:57   0:00 grep --color=auto mysqld
 8 
 9 [root@zlm1 16:57:10 /data/backup]
10 #ps aux|grep mysqld
11 mysql     3804  0.0 20.1 1038828 204780 pts/0  Sl   15:52   0:02 mysqld --defaults-file=/data/mysql/mysql3306/my3306.cnf
12 root      4518  0.0  0.0 112640   956 pts/2    R+   16:57   0:00 grep --color=auto mysqld
13 
14 [root@zlm1 16:57:15 /data/backup]
15 #mysqladmin shutdown
16 mysqladmin: connect to server at '192.168.56.100' failed
17 error: 'Can't connect to MySQL server on '192.168.56.100' (111)'
18 Check that mysqld is running on 192.168.56.100 and that the port is 3306.
19 You can check this by doing 'telnet 192.168.56.100 3306'
20 
21 [root@zlm1 16:57:23 /data/backup]
22 #ps aux|grep mysqld
23 root      4529  0.0  0.0 112640   960 pts/2    R+   16:57   0:00 grep --color=auto mysqld

  

**Check the PXC status at
first.**

  1 [root@zlm2 09:21:46 /data/mysql/mysql3308/data]
  2 #mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster &
  3 [1] 6251
  4 
  5 [root@zlm2 09:22:57 /data/mysql/mysql3308/data]
  6 #tail ../data/error.log |grep ERROR
  7 2018-08-07T09:22:57.471628+01:00 0 [ERROR] WSREP: wsrep_load(): dlopen(): /usr/local/mysql/lib/libgalera_smm.so: symbol SSL_COMP_free_compression_methods, version libssl.so.10 not defined in file libssl.so.10 with link time reference
  8 2018-08-07T09:22:57.471657+01:00 0 [ERROR] WSREP: wsrep_load() failed to load the provider('/usr/local/mysql/lib/libgalera_smm.so'): Invalid argument (22). Need to abort.
  9 2018-08-07T09:22:57.471661+01:00 0 [ERROR] Aborting
 10 [1]+  Exit 1                  mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf
 11 
 12 [root@zlm2 09:23:19 /data/mysql/mysql3308/data]
 13 #cd /usr/local/mysql/lib
 14 
 15 [root@zlm2 09:26:28 /usr/local/mysql/lib]
 16 #ls -l|grep libgalera_smm.so
 17 -rwxr-xr-x 1 root root 35086192 Jun 29 05:44 libgalera_smm.so
 18 
 19 [root@zlm2 09:26:38 /usr/local/mysql/lib]
 20 #rpm -qa|grep openssl
 21 openssl-libs-1.0.1e-42.el7.9.x86_64
 22 openssl-devel-1.0.1e-42.el7.9.x86_64
 23 openssl-1.0.1e-42.el7.9.x86_64
 24 
 25 [root@zlm2 09:32:25 /usr/local/mysql/lib]
 26 #cd /usr/lib64
 27 
 28 [root@zlm2 09:34:03 /usr/lib64]
 29 #ls -l|grep libssl
 30 -rwxr-xr-x.  1 root root  255888 Mar  6  2015 libssl3.so
 31 lrwxrwxrwx   1 root root      16 Jul 16  2015 libssl.so -> libssl.so.1.0.1e
 32 lrwxrwxrwx   1 root root      16 Jul 16  2015 libssl.so.10 -> libssl.so.1.0.1e
 33 -rwxr-xr-x   1 root root  449864 Jun 29  2015 libssl.so.1.0.1e
 34 
 35 [root@zlm2 09:41:02 /usr/lib64]
 36 #yum install openssl
 37 ... //Omitted.
 38 
 39 Updated:
 40   openssl.x86_64 1:1.0.2k-12.el7                                                                                                                     
 41 
 42 Dependency Updated:
 43   openssl-devel.x86_64 1:1.0.2k-12.el7                                      openssl-libs.x86_64 1:1.0.2k-12.el7                                     
 44 
 45 Complete!
 46 
 47 [root@zlm2 09:44:39 /usr/lib64]
 48 #tail /data/mysql/mysql3308/data/error.log |grep ERROR
 49 
 50 [root@zlm2 09:45:15 /usr/lib64]
 51 #ps aux|grep mysqld
 52 mysql     6345  2.9 20.1 1556468 204812 pts/0  Sl   09:44   0:01 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster
 53 root      6390  0.0  0.0 112640   960 pts/0    R+   09:45   0:00 grep --color=auto mysqld
 54 
 55 [root@zlm2 09:46:00 /usr/lib64]
 56 #cat /data/mysql/mysql3308/data/error.log |grep password
 57 2018-08-07T09:00:47.603271+01:00 1 [Note] A temporary password is generated for root@localhost: bY2jmV=k)ltM
 58 
 59 [root@zlm2 09:47:28 /usr/lib64]
 60 #mysql -uroot -p -S /tmp/mysql3308.sock
 61 Enter password: 
 62 ERROR 2003 (HY000): Can't connect to MySQL server on '192.168.1.101' (111)
 63 
 64 [root@zlm2 09:48:11 /usr/lib64]
 65 #ps aux|grep mysqld
 66 mysql     6345  0.6 20.1 1556468 204812 pts/0  Sl   09:44   0:01 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster
 67 root      6433  0.0  0.0 112640   960 pts/0    R+   09:48   0:00 grep --color=auto mysqld
 68 
 69 [root@zlm2 09:57:21 /usr/lib64]
 70 #mysql -uroot -p -hlocalhost -S /tmp/mysql3308.sock
 71 Enter password: 
 72 Welcome to the MySQL monitor.  Commands end with ; or \g.
 73 Your MySQL connection id is 4
 74 Server version: 5.7.22-22-29.26-log
 75 
 76 Copyright (c) 2009-2018 Percona LLC and/or its affiliates
 77 Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 78 
 79 Oracle is a registered trademark of Oracle Corporation and/or its
 80 affiliates. Other names may be trademarks of their respective
 81 owners.
 82 
 83 Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 84 
 85 (root@localhost mysql3308.sock)[(none)]>grant all privileges on *.* to zlm@'192.168.1.%';
 86 ERROR 1133 (42000): Can't find any matching row in the user table
 87 (root@localhost mysql3308.sock)[(none)]>grant all privileges on *.* to zlm@'192.168.1.%' identified by 'zlmzlm';
 88 Query OK, 0 rows affected, 1 warning (0.34 sec)
 89 
 90 (root@localhost mysql3308.sock)[(none)]>grant reload,lock tables,replication client on *.* to sst@localhost identified by 'zlmzlm';
 91 Query OK, 0 rows affected, 2 warnings (0.03 sec)
 92 
 93 (root@localhost mysql3308.sock)[(none)]>select user,host from mysql.user;
 94 +---------------+-------------+
 95 | user          | host        |
 96 +---------------+-------------+
 97 | zlm           | 192.168.1.% |
 98 | mysql.session | localhost   |
 99 | mysql.sys     | localhost   |
100 | root          | localhost   |
101 | sst           | localhost   |
102 +---------------+-------------+
103 5 rows in set (0.00 sec)
104 
105 (root@localhost mysql3308.sock)[(none)]>show global status like '%wsrep%';
106 +----------------------------------+--------------------------------------+
107 | Variable_name                    | Value                                |
108 +----------------------------------+--------------------------------------+
109 | wsrep_local_state_uuid           | bd5525ab-9a15-11e8-aa0f-4b830c783fc7 |
110 | wsrep_protocol_version           | 8                                    |
111 | wsrep_last_applied               | 2                                    |
112 | wsrep_last_committed             | 2                                    |
113 | wsrep_replicated                 | 2                                    |
114 | wsrep_replicated_bytes           | 432                                  |
115 | wsrep_repl_keys                  | 2                                    |
116 | wsrep_repl_keys_bytes            | 64                                   |
117 | wsrep_repl_data_bytes            | 232                                  |
118 | wsrep_repl_other_bytes           | 0                                    |
119 | wsrep_received                   | 2                                    |
120 | wsrep_received_bytes             | 141                                  |
121 | wsrep_local_commits              | 0                                    |
122 | wsrep_local_cert_failures        | 0                                    |
123 | wsrep_local_replays              | 0                                    |
124 | wsrep_local_send_queue           | 0                                    |
125 | wsrep_local_send_queue_max       | 1                                    |
126 | wsrep_local_send_queue_min       | 0                                    |
127 | wsrep_local_send_queue_avg       | 0.000000                             |
128 | wsrep_local_recv_queue           | 0                                    |
129 | wsrep_local_recv_queue_max       | 2                                    |
130 | wsrep_local_recv_queue_min       | 0                                    |
131 | wsrep_local_recv_queue_avg       | 0.500000                             |
132 | wsrep_local_cached_downto        | 1                                    |
133 | wsrep_flow_control_paused_ns     | 0                                    |
134 | wsrep_flow_control_paused        | 0.000000                             |
135 | wsrep_flow_control_sent          | 0                                    |
136 | wsrep_flow_control_recv          | 0                                    |
137 | wsrep_flow_control_interval      | [ 100, 100 ]                         |
138 | wsrep_flow_control_interval_low  | 100                                  |
139 | wsrep_flow_control_interval_high | 100                                  |
140 | wsrep_flow_control_status        | OFF                                  |
141 | wsrep_cert_deps_distance         | 1.000000                             |
142 | wsrep_apply_oooe                 | 0.000000                             |
143 | wsrep_apply_oool                 | 0.000000                             |
144 | wsrep_apply_window               | 1.000000                             |
145 | wsrep_commit_oooe                | 0.000000                             |
146 | wsrep_commit_oool                | 0.000000                             |
147 | wsrep_commit_window              | 1.000000                             |
148 | wsrep_local_state                | 4                                    |
149 | wsrep_local_state_comment        | Synced                               |
150 | wsrep_cert_index_size            | 1                                    |
151 | wsrep_cert_bucket_count          | 22                                   |
152 | wsrep_gcache_pool_size           | 1832                                 |
153 | wsrep_causal_reads               | 0                                    |
154 | wsrep_cert_interval              | 0.000000                             |
155 | wsrep_ist_receive_status         |                                      |
156 | wsrep_ist_receive_seqno_start    | 0                                    |
157 | wsrep_ist_receive_seqno_current  | 0                                    |
158 | wsrep_ist_receive_seqno_end      | 0                                    |
159 | wsrep_incoming_addresses         | 192.168.1.101:3308                   |
160 | wsrep_desync_count               | 0                                    |
161 | wsrep_evs_delayed                |                                      |
162 | wsrep_evs_evict_list             |                                      |
163 | wsrep_evs_repl_latency           | 0/0/0/0/0                            |
164 | wsrep_evs_state                  | OPERATIONAL                          |
165 | wsrep_gcomm_uuid                 | bd54497d-9a15-11e8-b386-2759969fcd01 |
166 | wsrep_cluster_conf_id            | 1                                    |
167 | wsrep_cluster_size               | 1                                    |
168 | wsrep_cluster_state_uuid         | bd5525ab-9a15-11e8-aa0f-4b830c783fc7 |
169 | wsrep_cluster_status             | Primary                              |
170 | wsrep_connected                  | ON                                   |
171 | wsrep_local_bf_aborts            | 0                                    |
172 | wsrep_local_index                | 0                                    |
173 | wsrep_provider_name              | Galera                               |
174 | wsrep_provider_vendor            | Codership Oy <info@codership.com>    |
175 | wsrep_provider_version           | 3.26(r)                              |
176 | wsrep_ready                      | ON                                   |
177 +----------------------------------+--------------------------------------+
178 68 rows in set (0.00 sec)
179 
180 //Check the below global status of PXC.
181 | wsrep_local_state                | 4                                    |
182 | wsrep_local_state_comment        | Synced                               |
183 | wsrep_cluster_status             | Primary                              |
184 | wsrep_connected                  | ON                                   |
185 
186 //It means that the PXC has been worked normally on node zlm2.
187 
188 //The partial error log of PXC which shows the success of starting up of it.
189 2018-08-07T09:44:40.631123+01:00 0 [Note] Event Scheduler: Loaded 0 events
190 2018-08-07T09:44:40.631550+01:00 0 [Note] mysqld: ready for connections.
191 Version: '5.7.22-22-29.26-log'  socket: '/tmp/mysql3308.sock'  port: 3308  Percona XtraDB Cluster binary (GPL) 5.7.22-29.26, Revision da86071, wsrep_29.26
192 2018-08-07T09:44:40.635469+01:00 2 [Note] WSREP: Initialized wsrep sidno 2
193 2018-08-07T09:44:40.635501+01:00 2 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 1) (Increment: 1 -> 1)
194 2018-08-07T09:44:40.635510+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
195 2018-08-07T09:44:40.635525+01:00 2 [Note] WSREP: Assign initial position for certification: 0, protocol version: 3
196 2018-08-07T09:44:40.635540+01:00 0 [Note] WSREP: Service thread queue flushed.
197 2018-08-07T09:44:40.635577+01:00 2 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> bd5525ab-9a15-11e8-aa0f-4b830c783fc7:0
198 2018-08-07T09:44:40.637360+01:00 2 [Note] WSREP: Synchronized with group, ready for connections 
199 2018-08-07T09:44:40.637369+01:00 2 [Note] WSREP: Setting wsrep_ready to true
200 2018-08-07T09:44:40.637372+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
201 
202 //Check the files in datadir.
203 [root@zlm2 10:38:25 /data/mysql/mysql3308/data]
204 #ls -l
205 total 556916
206 -rw-r----- 1 mysql mysql        56 Aug  7 09:00 auto.cnf
207 -rw------- 1 mysql mysql      1680 Aug  7 09:00 ca-key.pem
208 -rw-r--r-- 1 mysql mysql      1120 Aug  7 09:00 ca.pem
209 -rw-r--r-- 1 mysql mysql      1120 Aug  7 09:00 client-cert.pem
210 -rw------- 1 mysql mysql      1680 Aug  7 09:00 client-key.pem
211 -rw-r----- 1 mysql mysql     24663 Aug  7 10:30 error.log
212 -rw-r----- 1 mysql mysql 134219048 Aug  7 10:30 galera.cache
213 -rw-r----- 1 mysql mysql       113 Aug  7 10:30 grastate.dat
214 -rw-r----- 1 mysql mysql       170 Aug  7 10:30 gvwstate.dat
215 -rw-r----- 1 mysql mysql       423 Aug  7 09:00 ib_buffer_pool
216 -rw-r----- 1 mysql mysql 104857600 Aug  7 10:16 ibdata1
217 -rw-r----- 1 mysql mysql 104857600 Aug  7 10:16 ib_logfile0
218 -rw-r----- 1 mysql mysql 104857600 Aug  7 09:00 ib_logfile1
219 -rw-r----- 1 mysql mysql 104857600 Aug  7 09:00 ib_logfile2
220 -rw-r----- 1 mysql mysql  12582912 Aug  7 09:44 ibtmp1
221 drwxr-x--- 2 mysql mysql      4096 Aug  7 09:00 mysql
222 drwxr-x--- 2 mysql mysql      8192 Aug  7 09:00 performance_schema
223 -rw------- 1 mysql mysql      1676 Aug  7 09:00 private_key.pem
224 -rw-r--r-- 1 mysql mysql       452 Aug  7 09:00 public_key.pem
225 -rw-r--r-- 1 mysql mysql      1120 Aug  7 09:00 server-cert.pem
226 -rw------- 1 mysql mysql      1676 Aug  7 09:00 server-key.pem
227 -rw-r----- 1 mysql mysql       454 Aug  7 09:44 slow.log
228 drwxr-x--- 2 mysql mysql      8192 Aug  7 09:00 sys
229 -rw-r----- 1 mysql mysql   3932160 Aug  7 10:16 xb_doublewrite
230 -rw-r----- 1 mysql mysql         5 Aug  7 09:44 zlm2.pid
231 
232 [root@zlm2 10:38:27 /data/mysql/mysql3308/data]
233 #cat grastate.dat
234 # GALERA saved state
235 version: 2.1
236 uuid:    bd5525ab-9a15-11e8-aa0f-4b830c783fc7
237 seqno:   -1
238 safe_to_bootstrap: 1 //This means the node is the first one who starts the cluster.
Hostname IP/Port Identity OS Version MySQL Version GTID Mode Binlog Format
zlm1 192.168.56.100/3306 master CentOS 7.0 5.7.21 on row
zlm2 192.168.56.101/3306 slave CentOS 7.0 5.7.21 on row
zlm3 192.168.56.102/3306 binlog server CentOS 7.0 5.7.21 on row

 

 

 1 [root@zlm4 11:27:18 /data/mysql/mysql3308/data]
 2 #mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf initialize
 3 
 4 [root@zlm4 11:27:37 /data/mysql/mysql3308/data]
 5 #ls -l
 6 total 131092
 7 -rw-r----- 1 mysql mysql     13275 Aug  7 11:27 error.log
 8 -rw-r----- 1 mysql mysql 134219048 Aug  7 11:27 galera.cache
 9 -rw-r----- 1 mysql mysql         0 Aug  7 11:27 grastate.dat
10 
11 [root@zlm4 11:27:48 /data/mysql/mysql3308/data]
12 #cat error.log |grep ERROR
13     2018-08-07T09:27:24.770212Z WSREP_SST: [ERROR] ******************* FATAL ERROR ********************** 
14     2018-08-07T09:27:24.771275Z WSREP_SST: [ERROR] xtrabackup not in path: /usr/sbin:/sbin:/opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/bin:/root/perl5/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/mysql/bin:/usr/local/zabbix/bin:/usr/local/zabbix/sbin
15     2018-08-07T09:27:24.771951Z WSREP_SST: [ERROR] ****************************************************** 
16 2018-08-07T11:27:24.772254+01:00 0 [ERROR] WSREP: Failed to read 'ready <addr>' from: wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.103' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '4363' --binlog '/data/mysql/mysql3308/logs/mysql-bin' 
17 2018-08-07T11:27:24.772279+01:00 0 [ERROR] WSREP: Process completed with error: wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.103' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '4363' --binlog '/data/mysql/mysql3308/logs/mysql-bin' : 2 (No such file or directory)
18 2018-08-07T11:27:24.772317+01:00 1 [ERROR] WSREP: Failed to prepare for 'xtrabackup-v2' SST. Unrecoverable.
19 2018-08-07T11:27:24.772322+01:00 1 [ERROR] Aborting
20 
21 [root@zlm4 11:28:39 /data/mysql/mysql3308/data]
22 #innobackupex --help
23 -bash: innobackupex: command not found
24 
25 //It's due to absence of Xtrabackup tools on node zlm4.

Shutdown
instance and copy back.

  1 //Start the PCX on node zlm3.
  2 [root@zlm3 04:08:37 /data/mysql/mysql3308]
  3 #mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf &
  4 [1] 3893
  5 
  6 [root@zlm3 04:08:42 /data/mysql/mysql3308]
  7 #ps aux|grep mysqld
  8 mysql     3893  3.1 21.1 1574664 215668 pts/0  Sl   04:08   0:02 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf
  9 root      4537  0.0  0.0 112640   956 pts/0    R+   04:09   0:00 grep --color=auto mysqld
 10 
 11 //The intact error log of node zlm3 when joining the cluster.
 12 [root@zlm3 04:08:31 /data/mysql/mysql3308/data]
 13 #tail -f error.log 
 14 
 15 2018-08-09T04:08:43.084230+01:00 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
 16 2018-08-09T04:08:43.089848+01:00 0 [Warning] Insecure configuration for --secure-file-priv: Location is accessible to all OS users. Consider choosing a different directory.
 17 2018-08-09T04:08:43.089906+01:00 0 [Note] mysqld (mysqld 5.7.22-22-29.26-log) starting as process 3893 ...
 18 2018-08-09T04:08:43.109751+01:00 0 [Note] WSREP: Setting wsrep_ready to false
 19 2018-08-09T04:08:43.109768+01:00 0 [Note] WSREP: No pre-stored wsrep-start position found. Skipping position initialization.
 20 2018-08-09T04:08:43.109772+01:00 0 [Note] WSREP: wsrep_load(): loading provider library '/usr/local/mysql/lib/libgalera_smm.so'
 21 2018-08-09T04:08:43.129981+01:00 0 [Note] WSREP: wsrep_load(): Galera 3.26(r) by Codership Oy <info@codership.com> loaded successfully.
 22 2018-08-09T04:08:43.130124+01:00 0 [Note] WSREP: CRC-32C: using hardware acceleration.
 23 2018-08-09T04:08:43.130414+01:00 0 [Note] WSREP: Found saved state: 00000000-0000-0000-0000-000000000000:-1, safe_to_bootstrap: 1
 24 2018-08-09T04:08:43.141875+01:00 0 [Note] WSREP: Passing config to GCS: base_dir = /data/mysql/mysql3308/data/; base_host = 192.168.1.102; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.join_retrans_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 10; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.user_send_window = 4; evs.view_forget_timeout = PT24H; gcache.dir = /data/mysql/mysql3308/data/; gcache.freeze_purge_at_seqno = -1; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /data/mysql/mysql3308/data//galera.cache; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1; gcs.fc_limit = 100; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.segment = 0; gmcast.version = 0; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.npvo = false; pc.recovery = 1; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 8; socket.checksum = 2; socket.recv_buf_size = 212992; 
 25 2018-08-09T04:08:43.179573+01:00 0 [Note] WSREP: GCache history reset: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:0 -> 00000000-0000-0000-0000-000000000000:-1
 26 2018-08-09T04:08:43.183306+01:00 0 [Note] WSREP: Assign initial position for certification: -1, protocol version: -1
 27 2018-08-09T04:08:43.183338+01:00 0 [Note] WSREP: Preparing to initiate SST/IST
 28 2018-08-09T04:08:43.183342+01:00 0 [Note] WSREP: Starting replication
 29 2018-08-09T04:08:43.183355+01:00 0 [Note] WSREP: Setting initial position to 00000000-0000-0000-0000-000000000000:-1
 30 2018-08-09T04:08:43.183507+01:00 0 [Note] WSREP: Using CRC-32C for message checksums.
 31 2018-08-09T04:08:43.183568+01:00 0 [Note] WSREP: gcomm thread scheduling priority set to other:0 
 32 2018-08-09T04:08:43.183674+01:00 0 [Warning] WSREP: Fail to access the file (/data/mysql/mysql3308/data//gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown
 33 2018-08-09T04:08:43.183679+01:00 0 [Note] WSREP: Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown
 34 2018-08-09T04:08:43.184132+01:00 0 [Note] WSREP: GMCast version 0
 35 2018-08-09T04:08:43.184337+01:00 0 [Note] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') listening at tcp://0.0.0.0:4567
 36 2018-08-09T04:08:43.184344+01:00 0 [Note] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') multicast: , ttl: 1
 37 2018-08-09T04:08:43.184483+01:00 0 [Note] WSREP: EVS version 0
 38 2018-08-09T04:08:43.184957+01:00 0 [Note] WSREP: gcomm: connecting to group 'pxc_wubx', peer '192.168.1.101:,192.168.1.102:,192.168.103:'
 39 2018-08-09T04:08:43.186863+01:00 0 [Note] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') connection established to 23fb8f7a tcp://192.168.1.102:4567
 40 2018-08-09T04:08:43.186895+01:00 0 [Warning] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') address 'tcp://192.168.1.102:4567' points to own listening address, blacklisting
 41 2018-08-09T04:08:43.188042+01:00 0 [Note] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') connection established to 13eae368 tcp://192.168.1.101:4567
 42 2018-08-09T04:08:43.188190+01:00 0 [Note] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: 
 43 2018-08-09T04:08:43.686789+01:00 0 [Note] WSREP: declaring 13eae368 at tcp://192.168.1.101:4567 stable
 44 2018-08-09T04:08:43.687321+01:00 0 [Note] WSREP: Node 13eae368 state primary
 45 2018-08-09T04:08:43.690123+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
 46 view (view_id(PRIM,13eae368,2)
 47 memb {
 48     13eae368,0
 49     23fb8f7a,0
 50     }
 51 joined {
 52     }
 53 left {
 54     }
 55 partitioned {
 56     }
 57 )
 58 2018-08-09T04:08:43.690159+01:00 0 [Note] WSREP: Save the discovered primary-component to disk
 59 2018-08-09T04:08:43.690256+01:00 0 [Note] WSREP: discarding pending addr without UUID: tcp://192.168.0.103:4567
 60 2018-08-09T04:08:43.690263+01:00 0 [Note] WSREP: discarding pending addr proto entry 0x28b0b70
 61 2018-08-09T04:08:44.185955+01:00 0 [Note] WSREP: gcomm: connected
 62 2018-08-09T04:08:44.186036+01:00 0 [Note] WSREP: Shifting CLOSED -> OPEN (TO: 0)
 63 2018-08-09T04:08:44.186098+01:00 0 [Note] WSREP: Waiting for SST/IST to complete.
 64 2018-08-09T04:08:44.186339+01:00 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 1, memb_num = 2
 65 2018-08-09T04:08:44.186355+01:00 0 [Note] WSREP: STATE EXCHANGE: Waiting for state UUID.
 66 2018-08-09T04:08:44.186383+01:00 0 [Note] WSREP: STATE EXCHANGE: sent state msg: 24481910-9b79-11e8-b078-6a52474ad103
 67 2018-08-09T04:08:44.186388+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 24481910-9b79-11e8-b078-6a52474ad103 from 0 (zlm2)
 68 2018-08-09T04:08:44.189234+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 24481910-9b79-11e8-b078-6a52474ad103 from 1 (zlm3)
 69 2018-08-09T04:08:44.189263+01:00 0 [Note] WSREP: Quorum results:
 70     version    = 4,
 71     component  = PRIMARY,
 72     conf_id    = 1,
 73     members    = 1/2 (primary/total),
 74     act_id     = 22,
 75     last_appl. = -1,
 76     protocols  = 0/8/3 (gcs/repl/appl),
 77     group UUID = bd5525ab-9a15-11e8-aa0f-4b830c783fc7
 78 2018-08-09T04:08:44.189270+01:00 0 [Note] WSREP: Flow-control interval: [141, 141]
 79 2018-08-09T04:08:44.189273+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
 80 2018-08-09T04:08:44.189277+01:00 0 [Note] WSREP: Shifting OPEN -> PRIMARY (TO: 22)
 81 2018-08-09T04:08:44.189468+01:00 2 [Note] WSREP: State transfer required: 
 82     Group state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22
 83     Local state: 00000000-0000-0000-0000-000000000000:-1
 84 2018-08-09T04:08:44.189478+01:00 2 [Note] WSREP: REPL Protocols: 8 (3, 2)
 85 2018-08-09T04:08:44.189485+01:00 2 [Note] WSREP: New cluster view: global state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22, view# 2: Primary, number of nodes: 2, my index: 1, protocol version 3
 86 2018-08-09T04:08:44.189489+01:00 2 [Note] WSREP: Setting wsrep_ready to true
 87 2018-08-09T04:08:44.189491+01:00 2 [Warning] WSREP: Gap in state sequence. Need state transfer.
 88 2018-08-09T04:08:44.189493+01:00 2 [Note] WSREP: Setting wsrep_ready to false
 89 2018-08-09T04:08:44.192795+01:00 0 [Note] WSREP: Initiating SST/IST transfer on JOINER side (wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.102' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '3893' --binlog '/data/mysql/mysql3308/logs/mysql-bin' )
 90     2018-08-09T02:08:44.755132Z WSREP_SST: [WARNING] Found a stale sst_in_progress file: /data/mysql/mysql3308/data//sst_in_progress
 91 2018-08-09T04:08:44.870937+01:00 2 [Note] WSREP: Prepared SST/IST request: xtrabackup-v2|192.168.1.102:4444/xtrabackup_sst//1
 92 2018-08-09T04:08:44.871047+01:00 2 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 2) (Increment: 1 -> 2)
 93 2018-08-09T04:08:44.871090+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
 94 2018-08-09T04:08:44.871140+01:00 2 [Note] WSREP: Assign initial position for certification: 22, protocol version: 3
 95 2018-08-09T04:08:44.871282+01:00 0 [Note] WSREP: Service thread queue flushed.
 96 2018-08-09T04:08:44.871758+01:00 2 [Note] WSREP: Check if state gap can be serviced using IST
 97 2018-08-09T04:08:44.871782+01:00 2 [Note] WSREP: Local UUID: 00000000-0000-0000-0000-000000000000 != Group UUID: bd5525ab-9a15-11e8-aa0f-4b830c783fc7
 98 2018-08-09T04:08:44.871834+01:00 2 [Note] WSREP: State gap can't be serviced using IST. Switching to SST
 99 2018-08-09T04:08:44.871842+01:00 2 [Note] WSREP: Failed to prepare for incremental state transfer: Local state UUID (00000000-0000-0000-0000-000000000000) does not match group state UUID (bd5525ab-9a15-11e8-aa0f-4b830c783fc7): 1 (Operation not permitted)
100      at galera/src/replicator_str.cpp:prepare_for_IST():538. IST will be unavailable.
101 2018-08-09T04:08:44.872523+01:00 0 [Note] WSREP: Member 1.0 (zlm3) requested state transfer from '*any*'. Selected 0.0 (zlm2)(SYNCED) as donor.
102 2018-08-09T04:08:44.872545+01:00 0 [Note] WSREP: Shifting PRIMARY -> JOINER (TO: 22)
103 2018-08-09T04:08:44.872679+01:00 2 [Note] WSREP: Requesting state transfer: success, donor: 0
104 2018-08-09T04:08:44.872720+01:00 2 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22
105     2018-08-09T02:08:45.589512Z WSREP_SST: [INFO] WARNING: Stale temporary SST directory: /data/mysql/mysql3308/data//.sst from previous state transfer. Removing
106     2018-08-09T02:08:45.608370Z WSREP_SST: [INFO] Proceeding with SST.........
107 removed ‘/data/mysql/mysql3308/logs/mysql-bin.000015’
108     2018-08-09T02:08:45.637863Z WSREP_SST: [INFO] ............Waiting for SST streaming to complete!
109 2018-08-09T04:08:46.189014+01:00 0 [Note] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') turning message relay requesting off
110     2018-08-09T02:09:03.636108Z WSREP_SST: [INFO] Preparing the backup at /data/mysql/mysql3308/data//.sst
111 2018-08-09T04:09:03.649250+01:00 0 [Note] WSREP: 0.0 (zlm2): State transfer to 1.0 (zlm3) complete.
112 2018-08-09T04:09:03.659780+01:00 0 [Note] WSREP: Member 0.0 (zlm2) synced with group.
113     2018-08-09T02:09:21.887135Z WSREP_SST: [INFO] Moving the backup to /data/mysql/mysql3308/data/
114     2018-08-09T02:09:22.003073Z WSREP_SST: [INFO] Galera co-ords from recovery: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22
115 2018-08-09T04:09:22.015642+01:00 0 [Note] WSREP: SST complete, seqno: 22
116 2018-08-09T04:09:22.020459+01:00 0 [Warning] InnoDB: Using innodb_locks_unsafe_for_binlog is DEPRECATED. This option may be removed in future releases. Please use READ COMMITTED transaction isolation level instead; Please refer to http://dev.mysql.com/doc/refman/5.7/en/set-transaction.html
117 2018-08-09T04:09:22.020620+01:00 0 [Note] InnoDB: PUNCH HOLE support available
118 2018-08-09T04:09:22.020665+01:00 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins
119 2018-08-09T04:09:22.020674+01:00 0 [Note] InnoDB: Uses event mutexes
120 2018-08-09T04:09:22.020679+01:00 0 [Note] InnoDB: GCC builtin __atomic_thread_fence() is used for memory barrier
121 2018-08-09T04:09:22.020684+01:00 0 [Note] InnoDB: Compressed tables use zlib 1.2.7
122 2018-08-09T04:09:22.020688+01:00 0 [Note] InnoDB: Using Linux native AIO
123 2018-08-09T04:09:22.020708+01:00 0 [Note] InnoDB: Adjusting innodb_buffer_pool_instances from 4 to 1 since innodb_buffer_pool_size is less than 1024 MiB
124 2018-08-09T04:09:22.021332+01:00 0 [Note] InnoDB: Number of pools: 1
125 2018-08-09T04:09:22.021561+01:00 0 [Note] InnoDB: Using CPU crc32 instructions
126 2018-08-09T04:09:22.024636+01:00 0 [Note] InnoDB: Initializing buffer pool, total size = 100M, instances = 1, chunk size = 100M
127 2018-08-09T04:09:22.027579+01:00 0 [Note] InnoDB: Completed initialization of buffer pool
128 2018-08-09T04:09:22.031734+01:00 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().
129 2018-08-09T04:09:22.056548+01:00 0 [Note] InnoDB: Crash recovery did not find the parallel doublewrite buffer at /data/mysql/mysql3308/data/xb_doublewrite
130 2018-08-09T04:09:22.058938+01:00 0 [Note] InnoDB: Highest supported file format is Barracuda.
131 2018-08-09T04:09:22.176951+01:00 0 [Note] InnoDB: Created parallel doublewrite buffer at /data/mysql/mysql3308/data/xb_doublewrite, size 3932160 bytes
132 2018-08-09T04:09:22.311846+01:00 0 [Note] InnoDB: Removed temporary tablespace data file: "ibtmp1"
133 2018-08-09T04:09:22.311867+01:00 0 [Note] InnoDB: Creating shared tablespace for temporary tables
134 2018-08-09T04:09:22.311914+01:00 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...
135 2018-08-09T04:09:22.753228+01:00 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.
136 2018-08-09T04:09:22.753891+01:00 0 [Note] InnoDB: 96 redo rollback segment(s) found. 96 redo rollback segment(s) are active.
137 2018-08-09T04:09:22.753902+01:00 0 [Note] InnoDB: 32 non-redo rollback segment(s) are active.
138 2018-08-09T04:09:22.754607+01:00 0 [Note] InnoDB: Waiting for purge to start
139 2018-08-09T04:09:22.805815+01:00 0 [Note] InnoDB: Percona XtraDB (http://www.percona.com) 5.7.22-rel22 started; log sequence number 2605608
140 2018-08-09T04:09:22.806243+01:00 0 [Note] Plugin 'FEDERATED' is disabled.
141 2018-08-09T04:09:22.808992+01:00 0 [Note] InnoDB: Loading buffer pool(s) from /data/mysql/mysql3308/data/ib_buffer_pool
142 2018-08-09T04:09:22.854750+01:00 0 [Note] InnoDB: Buffer pool(s) load completed at 180809  4:09:22
143 2018-08-09T04:09:22.865970+01:00 0 [Note] Recovering after a crash using /data/mysql/mysql3308/logs/mysql-bin
144 2018-08-09T04:09:22.866002+01:00 0 [Note] WSREP: Before binlog recovery (wsrep position bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22)
145 2018-08-09T04:09:22.866021+01:00 0 [Note] Starting crash recovery...
146 2018-08-09T04:09:22.866046+01:00 0 [Note] Crash recovery finished.
147 2018-08-09T04:09:22.866057+01:00 0 [Note] WSREP: After binlog recovery (wsrep position bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22)
148 2018-08-09T04:09:22.875798+01:00 0 [Note] Salting uuid generator variables, current_pid: 3893, server_start_time: 1533780523, bytes_sent: 0, 
149 2018-08-09T04:09:22.875871+01:00 0 [Note] Generated uuid: '3ba41aa7-9b79-11e8-ad75-080027de0e0e', server_start_time: 1095782085868368224, bytes_sent: 46954208
150 2018-08-09T04:09:22.875894+01:00 0 [Warning] No existing UUID has been found, so we assume that this is the first time that this server has been started. Generating a new UUID: 3ba41aa7-9b79-11e8-ad75-080027de0e0e.
151 2018-08-09T04:09:23.579815+01:00 0 [Note] Auto generated SSL certificates are placed in data directory.
152 2018-08-09T04:09:23.580319+01:00 0 [Warning] CA certificate ca.pem is self signed.
153 2018-08-09T04:09:23.708603+01:00 0 [Note] Auto generated RSA key files are placed in data directory.
154 2018-08-09T04:09:23.708726+01:00 0 [Note] Server hostname (bind-address): '*'; port: 3308
155 2018-08-09T04:09:23.708750+01:00 0 [Note] IPv6 is available.
156 2018-08-09T04:09:23.708759+01:00 0 [Note]   - '::' resolves to '::';
157 2018-08-09T04:09:23.708775+01:00 0 [Note] Server socket created on IP: '::'.
158 2018-08-09T04:09:23.715377+01:00 0 [Warning] 'user' entry 'root@localhost' ignored in --skip-name-resolve mode.
159 2018-08-09T04:09:23.715477+01:00 0 [Warning] 'user' entry 'mysql.session@localhost' ignored in --skip-name-resolve mode.
160 2018-08-09T04:09:23.715490+01:00 0 [Warning] 'user' entry 'mysql.sys@localhost' ignored in --skip-name-resolve mode.
161 2018-08-09T04:09:23.715499+01:00 0 [Warning] 'user' entry 'sst@localhost' ignored in --skip-name-resolve mode.
162 2018-08-09T04:09:23.715539+01:00 0 [Warning] 'db' entry 'performance_schema mysql.session@localhost' ignored in --skip-name-resolve mode.
163 2018-08-09T04:09:23.715548+01:00 0 [Warning] 'db' entry 'sys mysql.sys@localhost' ignored in --skip-name-resolve mode.
164 2018-08-09T04:09:23.715559+01:00 0 [Warning] 'proxies_priv' entry '@ root@localhost' ignored in --skip-name-resolve mode.
165 2018-08-09T04:09:23.716731+01:00 0 [Warning] 'tables_priv' entry 'user mysql.session@localhost' ignored in --skip-name-resolve mode.
166 2018-08-09T04:09:23.716749+01:00 0 [Warning] 'tables_priv' entry 'sys_config mysql.sys@localhost' ignored in --skip-name-resolve mode.
167 2018-08-09T04:09:23.739950+01:00 0 [Note] Event Scheduler: Loaded 0 events
168 2018-08-09T04:09:23.740507+01:00 0 [Note] WSREP: Signalling provider to continue on SST completion.
169 2018-08-09T04:09:23.740531+01:00 0 [Note] WSREP: Initialized wsrep sidno 2
170 2018-08-09T04:09:23.740553+01:00 0 [Note] WSREP: SST received: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22
171 2018-08-09T04:09:23.740631+01:00 0 [Note] mysqld: ready for connections.
172 Version: '5.7.22-22-29.26-log'  socket: '/tmp/mysql3308.sock'  port: 3308  Percona XtraDB Cluster binary (GPL) 5.7.22-29.26, Revision da86071, wsrep_29.26
173 2018-08-09T04:09:23.741168+01:00 0 [Note] WSREP: 1.0 (zlm3): State transfer from 0.0 (zlm2) complete.
174 2018-08-09T04:09:23.741191+01:00 0 [Note] WSREP: SST leaving flow control
175 2018-08-09T04:09:23.741196+01:00 0 [Note] WSREP: Shifting JOINER -> JOINED (TO: 22)
176 2018-08-09T04:09:23.742029+01:00 0 [Note] WSREP: Member 1.0 (zlm3) synced with group.
177 2018-08-09T04:09:23.742059+01:00 0 [Note] WSREP: Shifting JOINED -> SYNCED (TO: 22)
178 2018-08-09T04:09:23.742077+01:00 2 [Note] WSREP: Synchronized with group, ready for connections
179 2018-08-09T04:09:23.742082+01:00 2 [Note] WSREP: Setting wsrep_ready to true
180 2018-08-09T04:09:23.742087+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
  1 //Create a replication user on node zlm3.
  2 (zlm@192.168.1.102 3308)[zlm]>grant all privileges on *.* to repl@'192.168.1.%' identified by 'repl4slave';
  3 ERROR 1045 (28000): Access denied for user 'zlm'@'192.168.1.%' (using password: YES)
  4 (zlm@192.168.1.102 3308)[zlm]>\q
  5 Bye
  6 
  7 [root@zlm3 10:11:28 /data/mysql/mysql3308/data]
  8 #mysql -uroot -pPassw0rd -hlocalhost -P3308
  9 mysql: [Warning] Using a password on the command line interface can be insecure.
 10 ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/tmp/mysql3306.sock' (2)
 11 
 12 [root@zlm3 10:11:48 /data/mysql/mysql3308/data]
 13 #mysql -uroot -pPassw0rd -hlocalhost -P3308 -S /tmp/mysql3308.sock
 14 mysql: [Warning] Using a password on the command line interface can be insecure.
 15 Welcome to the MySQL monitor.  Commands end with ; or \g.
 16 Your MySQL connection id is 8
 17 Server version: 5.7.22-22-29.26-log Percona XtraDB Cluster binary (GPL) 5.7.22-29.26, Revision da86071, wsrep_29.26
 18 
 19 Copyright (c) 2009-2018 Percona LLC and/or its affiliates
 20 Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 21 
 22 Oracle is a registered trademark of Oracle Corporation and/or its
 23 affiliates. Other names may be trademarks of their respective
 24 owners.
 25 
 26 Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 27 
 28 (root@localhost mysql3308.sock)[(none)]>grant all privileges on *.* to repl@'192.168.1.%' identified by 'repl4slave';
 29 Query OK, 0 rows affected, 1 warning (0.00 sec)
 30 
 31 (root@localhost mysql3308.sock)[(none)]>select user,host from mysql.user;
 32 +---------------+-------------+
 33 | user          | host        |
 34 +---------------+-------------+
 35 | repl          | 192.168.1.% |
 36 | zlm           | 192.168.1.% |
 37 | mysql.session | localhost   |
 38 | mysql.sys     | localhost   |
 39 | root          | localhost   |
 40 | sst           | localhost   |
 41 +---------------+-------------+
 42 6 rows in set (0.01 sec)
 43 
 44 //Execute "change master to ... " on node zlm4.
 45 [root@zlm4 10:14:37 /data/mysql/mysql3308]
 46 #mysql -uzlm -pzlmzlm -h192.168.1.103 -P3308
 47 mysql: [Warning] Using a password on the command line interface can be insecure.
 48 Welcome to the MySQL monitor.  Commands end with ; or \g.
 49 Your MySQL connection id is 2
 50 Server version: 5.7.22-22-29.26-log Percona XtraDB Cluster binary (GPL) 5.7.22-29.26, Revision da86071, wsrep_29.26
 51 
 52 Copyright (c) 2009-2018 Percona LLC and/or its affiliates
 53 Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 54 
 55 Oracle is a registered trademark of Oracle Corporation and/or its
 56 affiliates. Other names may be trademarks of their respective
 57 owners.
 58 
 59 Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 60 
 61 (zlm@192.168.1.103 3308)[(none)]>change master to master_host='192.168.1.102',master_port=3308,master_user='repl',master_password='repl4slave',master_auto_position=1;
 62 Query OK, 0 rows affected, 1 warning (0.02 sec)
 63 
 64 (zlm@192.168.1.103 3308)[(none)]>start slave;
 65 Query OK, 0 rows affected (0.02 sec)
 66 
 67 (zlm@192.168.1.103 3308)[(none)]>show slave status\G
 68 *************************** 1. row ***************************
 69                Slave_IO_State: Waiting for master to send event
 70                   Master_Host: 192.168.1.102
 71                   Master_User: repl
 72                   Master_Port: 3308
 73                 Connect_Retry: 60
 74               Master_Log_File: mysql-bin.000023
 75           Read_Master_Log_Pos: 1397
 76                Relay_Log_File: zlm4-relay-bin.000002
 77                 Relay_Log_Pos: 367
 78         Relay_Master_Log_File: mysql-bin.000023
 79              Slave_IO_Running: Yes
 80             Slave_SQL_Running: No
 81               Replicate_Do_DB: 
 82           Replicate_Ignore_DB: 
 83            Replicate_Do_Table: 
 84        Replicate_Ignore_Table: 
 85       Replicate_Wild_Do_Table: 
 86   Replicate_Wild_Ignore_Table: 
 87                    Last_Errno: 1007
 88                    Last_Error: Coordinator stopped because there were error(s) in the worker(s). The most recent failure being: Worker 1 failed executing transaction '42aada54-65ea-ee17-55f0-b47cf387c038:20' at master log mysql-bin.000023, end_log_pos 350. See error log and/or performance_schema.replication_applier_status_by_worker table for more details about this failure or others, if any.
 89                  Skip_Counter: 0
 90           Exec_Master_Log_Pos: 194
 91               Relay_Log_Space: 1776
 92               Until_Condition: None
 93                Until_Log_File: 
 94                 Until_Log_Pos: 0
 95            Master_SSL_Allowed: No
 96            Master_SSL_CA_File: 
 97            Master_SSL_CA_Path: 
 98               Master_SSL_Cert: 
 99             Master_SSL_Cipher: 
100                Master_SSL_Key: 
101         Seconds_Behind_Master: NULL
102 Master_SSL_Verify_Server_Cert: No
103                 Last_IO_Errno: 0
104                 Last_IO_Error: 
105                Last_SQL_Errno: 1007
106                Last_SQL_Error: Coordinator stopped because there were error(s) in the worker(s). The most recent failure being: Worker 1 failed executing transaction '42aada54-65ea-ee17-55f0-b47cf387c038:20' at master log mysql-bin.000023, end_log_pos 350. See error log and/or performance_schema.replication_applier_status_by_worker table for more details about this failure or others, if any.
107   Replicate_Ignore_Server_Ids: 
108              Master_Server_Id: 1023308
109                   Master_UUID: 3ba41aa7-9b79-11e8-ad75-080027de0e0e
110              Master_Info_File: /data/mysql/mysql3308/data/master.info
111                     SQL_Delay: 0
112           SQL_Remaining_Delay: NULL
113       Slave_SQL_Running_State: 
114            Master_Retry_Count: 86400
115                   Master_Bind: 
116       Last_IO_Error_Timestamp: 
117      Last_SQL_Error_Timestamp: 180809 10:18:58
118                Master_SSL_Crl: 
119            Master_SSL_Crlpath: 
120            Retrieved_Gtid_Set: 42aada54-65ea-ee17-55f0-b47cf387c038:20-24
121             Executed_Gtid_Set: 42aada54-65ea-ee17-55f0-b47cf387c038:1-19
122                 Auto_Position: 1
123          Replicate_Rewrite_DB: 
124                  Channel_Name: 
125            Master_TLS_Version: 
126 1 row in set (0.00 sec)
127 
128 //The error log shows below.
129 2018-08-09T10:15:45.368412+01:00 2 [Warning] Neither --relay-log nor --relay-log-index were used; so replication may break when this MySQL server acts as a slave and has his hostname changed!! Please use '--relay-log=zlm4-relay-bin' to avoid this problem.
130 2018-08-09T10:15:45.382500+01:00 2 [Note] 'CHANGE MASTER TO FOR CHANNEL '' executed'. Previous state master_host='', master_port= 3306, master_log_file='', master_log_pos= 4, master_bind=''. New state master_host='192.168.1.102', master_port= 3308, master_log_file='', master_log_pos= 4, master_bind=''.
131 2018-08-09T10:18:58.164370+01:00 3 [Warning] Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
132 2018-08-09T10:18:58.173589+01:00 3 [Note] Slave I/O thread for channel '': connected to master 'repl@192.168.1.102:3308',replication started in log 'FIRST' at position 4
133 2018-08-09T10:18:58.180721+01:00 4 [Warning] Slave SQL for channel '': If a crash happens this configuration does not guarantee that the relay log info will be consistent, Error_code: 0
134 2018-08-09T10:18:58.180761+01:00 4 [Note] Slave SQL thread for channel '' initialized, starting replication in log 'FIRST' at position 0, relay log './zlm4-relay-bin.000001' position: 4
135 2018-08-09T10:18:58.202347+01:00 5 [ERROR] Slave SQL for channel '': Worker 1 failed executing transaction '42aada54-65ea-ee17-55f0-b47cf387c038:20' at master log mysql-bin.000023, end_log_pos 350; Error 'Can't create database 'zlm'; database exists' on query. Default database: 'zlm'. Query: 'create database zlm', Error_code: 1007
136 2018-08-09T10:18:58.202651+01:00 4 [ERROR] Error running query, slave SQL thread aborted. Fix the problem, and restart the slave SQL thread with "SLAVE START". We stopped at log 'mysql-bin.000023' position 194
137 
138 //Check the "replication_applier_status_by_worker" table for detail.
139 (zlm@192.168.1.103 3308)[(none)]>select * from performance_schema.replication_applier_status_by_worker\G
140 *************************** 1. row ***************************
141          CHANNEL_NAME: 
142             WORKER_ID: 1
143             THREAD_ID: NULL
144         SERVICE_STATE: OFF
145 LAST_SEEN_TRANSACTION: 42aada54-65ea-ee17-55f0-b47cf387c038:20
146     LAST_ERROR_NUMBER: 1007
147    LAST_ERROR_MESSAGE: Worker 1 failed executing transaction '42aada54-65ea-ee17-55f0-b47cf387c038:20' at master log mysql-bin.000023, end_log_pos 350; Error 'Can't create database 'zlm'; database exists' on query. Default database: 'zlm'. Query: 'create database zlm'
148  LAST_ERROR_TIMESTAMP: 2018-08-09 16:18:58
149 *************************** 2. row ***************************
150          CHANNEL_NAME: 
151             WORKER_ID: 2
152             THREAD_ID: NULL
153         SERVICE_STATE: OFF
154 LAST_SEEN_TRANSACTION: 
155     LAST_ERROR_NUMBER: 0
156    LAST_ERROR_MESSAGE: 
157  LAST_ERROR_TIMESTAMP: 0000-00-00 00:00:00
158 *************************** 3. row ***************************
159          CHANNEL_NAME: 
160             WORKER_ID: 3
161             THREAD_ID: NULL
162         SERVICE_STATE: OFF
163 LAST_SEEN_TRANSACTION: 
164     LAST_ERROR_NUMBER: 0
165    LAST_ERROR_MESSAGE: 
166  LAST_ERROR_TIMESTAMP: 0000-00-00 00:00:00
167 *************************** 4. row ***************************
168          CHANNEL_NAME: 
169             WORKER_ID: 4
170             THREAD_ID: NULL
171         SERVICE_STATE: OFF
172 LAST_SEEN_TRANSACTION: 
173     LAST_ERROR_NUMBER: 0
174    LAST_ERROR_MESSAGE: 
175  LAST_ERROR_TIMESTAMP: 0000-00-00 00:00:00
176 4 rows in set (0.00 sec)
177 
178 //Check the target database and table.
179 (zlm@192.168.1.103 3308)[(none)]>show databases;
180 +--------------------+
181 | Database           |
182 +--------------------+
183 | information_schema |
184 | mysql              |
185 | performance_schema |
186 | sys                |
187 | zlm                |
188 +--------------------+
189 5 rows in set (0.00 sec)
190 
191 (zlm@192.168.1.103 3308)[(none)]>use zlm
192 Reading table information for completion of table and column names
193 You can turn off this feature to get a quicker startup with -A
194 
195 Database changed
196 (zlm@192.168.1.103 3308)[zlm]>show tables;
197 +---------------+
198 | Tables_in_zlm |
199 +---------------+
200 | t1            |
201 +---------------+
202 1 row in set (0.00 sec)
203 
204 (zlm@192.168.1.103 3308)[zlm]>select * from t1;
205 +----+------------+
206 | id | name       |
207 +----+------------+
208 |  1 | MySQL      |
209 |  2 | Oracle     |
210 |  3 | PostgreSQL |
211 +----+------------+
212 3 rows in set (0.01 sec)
213 
214 //The changes on node zlm3 has been replicated to zlm4.
215 
216 [root@zlm4 10:35:51 /data/mysql/mysql3308/logs]
217 #ls -l
218 total 16
219 -rw-r----- 1 mysql mysql 194 Aug  9 08:00 mysql-bin.000024
220 -rw-r----- 1 mysql mysql 217 Aug  9 09:02 mysql-bin.000025
221 -rw-r----- 1 mysql mysql 194 Aug  9 10:08 mysql-bin.000026
222 -rw-r----- 1 mysql mysql 132 Aug  9 10:08 mysql-bin.index
223 
224 //No mysql-bin.000023 was found.Check the relay-log file.
225 [root@zlm4 10:38:16 /data/mysql/mysql3308/data]
226 #ls -l|grep relay
227 -rw-r----- 1 mysql mysql        58 Aug  9 10:18 relay-log.info
228 -rw-r----- 1 mysql mysql        84 Aug  9 10:18 worker-relay-log.info.1
229 -rw-r----- 1 mysql mysql        84 Aug  9 10:18 worker-relay-log.info.2
230 -rw-r----- 1 mysql mysql        84 Aug  9 10:18 worker-relay-log.info.3
231 -rw-r----- 1 mysql mysql        84 Aug  9 10:18 worker-relay-log.info.4
232 -rw-r----- 1 mysql mysql       206 Aug  9 10:18 zlm4-relay-bin.000001
233 -rw-r----- 1 mysql mysql      1570 Aug  9 10:18 zlm4-relay-bin.000002
234 -rw-r----- 1 mysql mysql        48 Aug  9 10:18 zlm4-relay-bin.index
235 
236 [root@zlm4 10:38:30 /data/mysql/mysql3308/data]
237 #mysqlbinlog -v --base64-output=decode-rows zlm4-relay-bin.000002 > ~/02.log
238 
239 [root@zlm4 10:38:46 /data/mysql/mysql3308/data]
240 #cd
241 
242 [root@zlm4 10:38:49 ~]
243 #cat 02.log 
244 /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;
245 /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/;
246 DELIMITER /*!*/;
247 # at 4
248 #180809 10:18:58 server id 1033308  end_log_pos 123 CRC32 0x4367bb3e     Start: binlog v 4, server v 5.7.22-22-29.26-log created 180809 10:18:58
249 # This Format_description_event appears in a relay log and was generated by the slave thread.
250 # at 123
251 #180809 10:18:58 server id 1033308  end_log_pos 154 CRC32 0xf3605911     Previous-GTIDs
252 # [empty]
253 # at 154
254 #700101  1:00:00 server id 1023308  end_log_pos 0 CRC32 0xb5bcfbb7     Rotate to mysql-bin.000023  pos: 4
255 # at 201
256 #180809  4:09:22 server id 1023308  end_log_pos 123 CRC32 0x6f7e8565     Start: binlog v 4, server v 5.7.22-22-29.26-log created 180809  4:09:22 at startup
257 ROLLBACK/*!*/;
258 # at 320
259 #180809 10:18:58 server id 0  end_log_pos 367 CRC32 0xa757553d     Rotate to mysql-bin.000023  pos: 194
260 # at 367
261 #180809  9:07:52 server id 1023308  end_log_pos 259 CRC32 0x531076d3     GTID    last_committed=0    sequence_number=1    rbr_only=no
262 SET @@SESSION.GTID_NEXT= '42aada54-65ea-ee17-55f0-b47cf387c038:20'/*!*/;
263 # at 432
264 #180809  9:07:52 server id 1023308  end_log_pos 350 CRC32 0x9acb4e3f     Query    thread_id=5    exec_time=0    error_code=0
265 SET TIMESTAMP=1533798472/*!*/;
266 SET @@session.pseudo_thread_id=5/*!*/;
267 SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/;
268 SET @@session.sql_mode=1436549152/*!*/;
269 SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/;
270 /*!\C utf8 *//*!*/;
271 SET @@session.character_set_client=33,@@session.collation_connection=33,@@session.collation_server=33/*!*/;
272 SET @@session.lc_time_names=0/*!*/;
273 SET @@session.collation_database=DEFAULT/*!*/;
274 create database zlm
275 /*!*/;
276 # at 523
277 #180809  9:08:36 server id 1023308  end_log_pos 415 CRC32 0xde316a34     GTID    last_committed=1    sequence_number=2    rbr_only=no
278 SET @@SESSION.GTID_NEXT= '42aada54-65ea-ee17-55f0-b47cf387c038:21'/*!*/;
279 # at 588
280 #180809  9:08:36 server id 1023308  end_log_pos 557 CRC32 0xb29bd4ab     Query    thread_id=5    exec_time=0    error_code=0
281 use `zlm`/*!*/;
282 SET TIMESTAMP=1533798516/*!*/;
283 create table t1(
284 id int,
285 name char(10)
286 ) engine=innodb charset=utf8mb4
287 /*!*/;
288 # at 730
289 #180809  9:13:42 server id 1023308  end_log_pos 622 CRC32 0x2a5f6414     GTID    last_committed=2    sequence_number=3    rbr_only=no
290 SET @@SESSION.GTID_NEXT= '42aada54-65ea-ee17-55f0-b47cf387c038:22'/*!*/;
291 # at 795
292 #180809  9:13:42 server id 1023308  end_log_pos 728 CRC32 0xa803e3aa     Query    thread_id=5    exec_time=0    error_code=0
293 SET TIMESTAMP=1533798822/*!*/;
294 alter table t1 add primary key(id)
295 /*!*/;
296 # at 901
297 #180809  9:13:46 server id 1023308  end_log_pos 793 CRC32 0xfd677245     GTID    last_committed=3    sequence_number=4    rbr_only=yes
298 /*!50718 SET TRANSACTION ISOLATION LEVEL READ COMMITTED*//*!*/;
299 SET @@SESSION.GTID_NEXT= '42aada54-65ea-ee17-55f0-b47cf387c038:23'/*!*/;
300 # at 966
301 #180809  9:13:46 server id 1023308  end_log_pos 869 CRC32 0x7b568d5a     Query    thread_id=5    exec_time=0    error_code=0
302 SET TIMESTAMP=1533798826/*!*/;
303 SET @@session.auto_increment_increment=2, @@session.auto_increment_offset=2/*!*/;
304 BEGIN
305 /*!*/;
306 # at 1042
307 # at 1128
308 #180809  9:13:46 server id 1023308  end_log_pos 1002 CRC32 0x8b4e5e6b     Table_map: `zlm`.`t1` mapped to number 110
309 # at 1175
310 #180809  9:13:46 server id 1023308  end_log_pos 1076 CRC32 0x51750bc0     Write_rows: table id 110 flags: STMT_END_F
311 ### INSERT INTO `zlm`.`t1`
312 ### SET
313 ###   @1=1
314 ###   @2='MySQL'
315 ### INSERT INTO `zlm`.`t1`
316 ### SET
317 ###   @1=2
318 ###   @2='Oracle'
319 ### INSERT INTO `zlm`.`t1`
320 ### SET
321 ###   @1=3
322 ###   @2='PostgreSQL'
323 # at 1249
324 #180809  9:13:46 server id 1023308  end_log_pos 1107 CRC32 0x6033f0ee     Xid = 26
325 COMMIT/*!*/;
326 # at 1280
327 #180809 10:12:07 server id 1023308  end_log_pos 1172 CRC32 0x66f4de8c     GTID    last_committed=4    sequence_number=5    rbr_only=no
328 SET @@SESSION.GTID_NEXT= '42aada54-65ea-ee17-55f0-b47cf387c038:24'/*!*/;
329 # at 1345
330 #180809 10:12:07 server id 1023308  end_log_pos 1397 CRC32 0x86665c03     Query    thread_id=8    exec_time=0    error_code=0
331 SET TIMESTAMP=1533802327/*!*/;
332 SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/;
333 GRANT ALL PRIVILEGES ON *.* TO 'repl'@'192.168.1.%' IDENTIFIED WITH 'mysql_native_password' AS '*872ECE72A7EBAC6A183C90D7043D5F359BD85A9E'
334 /*!*/;
335 SET @@SESSION.GTID_NEXT= 'AUTOMATIC' /* added by mysqlbinlog */ /*!*/;
336 DELIMITER ;
337 # End of log file
338 /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;
339 /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;
340 
341 //Clean the value of variable "gtid_purged".
342 (zlm@192.168.1.103 3308)[(none)]>reset master;
343 Query OK, 0 rows affected (0.03 sec)
344 
345 (zlm@192.168.1.103 3308)[(none)]>set @@global.gtid_purged='42aada54-65ea-ee17-55f0-b47cf387c038:1-24';
346 Query OK, 0 rows affected (0.01 sec)
347 
348 (zlm@192.168.1.103 3308)[(none)]>start slave sql_thread;
349 Query OK, 0 rows affected (0.00 sec)
350 
351 (zlm@192.168.1.103 3308)[(none)]>show slave status\G
352 *************************** 1. row ***************************
353                Slave_IO_State: Waiting for master to send event
354                   Master_Host: 192.168.1.102
355                   Master_User: repl
356                   Master_Port: 3308
357                 Connect_Retry: 60
358               Master_Log_File: mysql-bin.000023
359           Read_Master_Log_Pos: 1397
360                Relay_Log_File: zlm4-relay-bin.000002
361                 Relay_Log_Pos: 1570
362         Relay_Master_Log_File: mysql-bin.000023
363              Slave_IO_Running: Yes
364             Slave_SQL_Running: Yes
365               Replicate_Do_DB: 
366           Replicate_Ignore_DB: 
367            Replicate_Do_Table: 
368        Replicate_Ignore_Table: 
369       Replicate_Wild_Do_Table: 
370   Replicate_Wild_Ignore_Table: 
371                    Last_Errno: 0
372                    Last_Error: 
373                  Skip_Counter: 0
374           Exec_Master_Log_Pos: 1397
375               Relay_Log_Space: 1776
376               Until_Condition: None
377                Until_Log_File: 
378                 Until_Log_Pos: 0
379            Master_SSL_Allowed: No
380            Master_SSL_CA_File: 
381            Master_SSL_CA_Path: 
382               Master_SSL_Cert: 
383             Master_SSL_Cipher: 
384                Master_SSL_Key: 
385         Seconds_Behind_Master: 0
386 Master_SSL_Verify_Server_Cert: No
387                 Last_IO_Errno: 0
388                 Last_IO_Error: 
389                Last_SQL_Errno: 0
390                Last_SQL_Error: 
391   Replicate_Ignore_Server_Ids: 
392              Master_Server_Id: 1023308
393                   Master_UUID: 3ba41aa7-9b79-11e8-ad75-080027de0e0e
394              Master_Info_File: /data/mysql/mysql3308/data/master.info
395                     SQL_Delay: 0
396           SQL_Remaining_Delay: NULL
397       Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates
398            Master_Retry_Count: 86400
399                   Master_Bind: 
400       Last_IO_Error_Timestamp: 
401      Last_SQL_Error_Timestamp: 
402                Master_SSL_Crl: 
403            Master_SSL_Crlpath: 
404            Retrieved_Gtid_Set: 42aada54-65ea-ee17-55f0-b47cf387c038:20-24
405             Executed_Gtid_Set: 42aada54-65ea-ee17-55f0-b47cf387c038:1-24
406                 Auto_Position: 1
407          Replicate_Rewrite_DB: 
408                  Channel_Name: 
409            Master_TLS_Version: 
410 1 row in set (0.00 sec)
  1 //Let's start the PXC on node zlm3.
  2 [root@zlm3 10:00:12 /data/mysql/mysql3308/data]
  3 #mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf &
  4 [1] 4181
  5 
  6 [root@zlm3 10:30:43 /data/mysql/mysql3308/data]
  7 #ps aux|grep mysqld
  8 mysql     4181 20.0 16.9 756016 172580 pts/0   Sl   10:30   0:01 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf
  9 root      4596  0.0  0.0 112640   960 pts/0    R+   10:30   0:00 grep --color=auto mysqld
 10 
 11 [root@zlm3 10:31:10 /data/mysql/mysql3308/data]
 12 #tail -1000 error.log 
 13 2018-08-07T09:54:08.825870+01:00 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
 14 2018-08-07T09:54:08.825960+01:00 0 [Warning] WSREP: Node is running in bootstrap/initialize mode. Disabling pxc_strict_mode checks
 15 2018-08-07T09:54:08.836945+01:00 0 [Warning] InnoDB: Using innodb_locks_unsafe_for_binlog is DEPRECATED. This option may be removed in future releases. Please use READ COMMITTED transaction isolation level instead; Please refer to http://dev.mysql.com/doc/refman/5.7/en/set-transaction.html
 16  100
 17  100
 18  100
 19  100
 20 2018-08-07T09:54:27.784997+01:00 0 [Warning] InnoDB: New log files created, LSN=45790
 21 2018-08-07T09:54:28.565405+01:00 0 [Warning] InnoDB: Creating foreign key constraint system tables.
 22 2018-08-07T09:54:29.043051+01:00 0 [Warning] No existing UUID has been found, so we assume that this is the first time that this server has been started. Generating a new UUID: 1ca6f9ae-9a17-11e8-9a7c-080027de0e0e.
 23 2018-08-07T09:54:29.054082+01:00 0 [Warning] Gtid table is not ready to be used. Table 'mysql.gtid_executed' cannot be opened.
 24 2018-08-07T09:54:29.471412+01:00 0 [Warning] CA certificate ca.pem is self signed.
 25 2018-08-07T09:54:29.536187+01:00 1 [Note] A temporary password is generated for root@localhost: ,M3=why-ok&d
 26 2018-08-07T09:55:01.556746+01:00 1 [Warning] 'user' entry 'root@localhost' ignored in --skip-name-resolve mode.
 27 2018-08-07T09:55:01.556802+01:00 1 [Warning] 'user' entry 'mysql.session@localhost' ignored in --skip-name-resolve mode.
 28 2018-08-07T09:55:01.556812+01:00 1 [Warning] 'user' entry 'mysql.sys@localhost' ignored in --skip-name-resolve mode.
 29 2018-08-07T09:55:01.556830+01:00 1 [Warning] 'db' entry 'performance_schema mysql.session@localhost' ignored in --skip-name-resolve mode.
 30 2018-08-07T09:55:01.556837+01:00 1 [Warning] 'db' entry 'sys mysql.sys@localhost' ignored in --skip-name-resolve mode.
 31 2018-08-07T09:55:01.556847+01:00 1 [Warning] 'proxies_priv' entry '@ root@localhost' ignored in --skip-name-resolve mode.
 32 2018-08-07T09:55:01.556920+01:00 1 [Warning] 'tables_priv' entry 'user mysql.session@localhost' ignored in --skip-name-resolve mode.
 33 2018-08-07T09:55:01.556931+01:00 1 [Warning] 'tables_priv' entry 'sys_config mysql.sys@localhost' ignored in --skip-name-resolve mode.
 34 
 35 //The starting mysqld operation begins here.
 36 2018-08-07T10:30:43.794296+01:00 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
 37 2018-08-07T10:30:43.801975+01:00 0 [Warning] Insecure configuration for --secure-file-priv: Location is accessible to all OS users. Consider choosing a different directory.
 38 2018-08-07T10:30:43.802040+01:00 0 [Note] mysqld (mysqld 5.7.22-22-29.26-log) starting as process 4181 ...
 39 2018-08-07T10:30:43.805420+01:00 0 [Note] WSREP: Setting wsrep_ready to false //Not ready.
 40 2018-08-07T10:30:43.805476+01:00 0 [Note] WSREP: No pre-stored wsrep-start position found. Skipping position initialization.
 41 2018-08-07T10:30:43.805486+01:00 0 [Note] WSREP: wsrep_load(): loading provider library '/usr/local/mysql/lib/libgalera_smm.so'
 42 2018-08-07T10:30:43.829026+01:00 0 [Note] WSREP: wsrep_load(): Galera 3.26(r) by Codership Oy <info@codership.com> loaded successfully.
 43 2018-08-07T10:30:43.829070+01:00 0 [Note] WSREP: CRC-32C: using hardware acceleration.
 44 2018-08-07T10:30:43.829385+01:00 0 [Warning] WSREP: Could not open state file for reading: '/data/mysql/mysql3308/data//grastate.dat'
 45 2018-08-07T10:30:43.829395+01:00 0 [Warning] WSREP: No persistent state found. Bootstraping with default state
 46 2018-08-07T10:30:43.829432+01:00 0 [Note] WSREP: Found saved state: 00000000-0000-0000-0000-000000000000:-1, safe_to_bootstrap: 1
 47 2018-08-07T10:30:43.837823+01:00 0 [Note] WSREP: Passing config to GCS: base_dir = /data/mysql/mysql3308/data/; base_host = 192.168.1.102; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.join_retrans_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 10; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.user_send_window = 4; evs.view_forget_timeout = PT24H; gcache.dir = /data/mysql/mysql3308/data/; gcache.freeze_purge_at_seqno = -1; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /data/mysql/mysql3308/data//galera.cache; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1; gcs.fc_limit = 100; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.segment = 0; gmcast.version = 0; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.npvo = false; pc.recovery = 1; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 8; socket.checksum = 2; socket.recv_buf_size = 212992; 
 48 2018-08-07T10:30:43.861834+01:00 0 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> 00000000-0000-0000-0000-000000000000:-1
 49 2018-08-07T10:30:43.866886+01:00 0 [Note] WSREP: Assign initial position for certification: -1, protocol version: -1
 50 2018-08-07T10:30:43.866919+01:00 0 [Note] WSREP: Preparing to initiate SST/IST //Begin to SST or IST.
 51 2018-08-07T10:30:43.866922+01:00 0 [Note] WSREP: Starting replication
 52 2018-08-07T10:30:43.866936+01:00 0 [Note] WSREP: Setting initial position to 00000000-0000-0000-0000-000000000000:-1
 53 2018-08-07T10:30:43.867102+01:00 0 [Note] WSREP: Using CRC-32C for message checksums.
 54 2018-08-07T10:30:43.867166+01:00 0 [Note] WSREP: gcomm thread scheduling priority set to other:0 
 55 2018-08-07T10:30:43.869814+01:00 0 [Warning] WSREP: Fail to access the file (/data/mysql/mysql3308/data//gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown
 56 2018-08-07T10:30:43.869842+01:00 0 [Note] WSREP: Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown
 57 2018-08-07T10:30:43.870450+01:00 0 [Note] WSREP: GMCast version 0
 58 2018-08-07T10:30:43.870721+01:00 0 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') listening at tcp://0.0.0.0:4567
 59 2018-08-07T10:30:43.870729+01:00 0 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') multicast: , ttl: 1
 60 2018-08-07T10:30:43.871017+01:00 0 [Note] WSREP: EVS version 0
 61 2018-08-07T10:30:43.871191+01:00 0 [Note] WSREP: gcomm: connecting to group 'pxc_wubx', peer '192.168.1.101:,192.168.1.102:,192.168.103:'
 62 2018-08-07T10:30:43.873170+01:00 0 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') connection established to 2cf33746 tcp://192.168.1.102:4567
 63 2018-08-07T10:30:43.873199+01:00 0 [Warning] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') address 'tcp://192.168.1.102:4567' points to own listening address, blacklisting
 64 2018-08-07T10:30:43.873754+01:00 0 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') connection established to bd54497d tcp://192.168.1.101:4567
 65 2018-08-07T10:30:43.873806+01:00 0 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: 
 66 2018-08-07T10:30:44.373051+01:00 0 [Note] WSREP: declaring bd54497d at tcp://192.168.1.101:4567 stable
 67 2018-08-07T10:30:44.373698+01:00 0 [Note] WSREP: Node bd54497d state primary
 68 2018-08-07T10:30:44.374205+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
 69 view (view_id(PRIM,2cf33746,2)
 70 memb {
 71     2cf33746,0 //This is node zlm2,first one started the cluster.
 72     bd54497d,0 //This is node zlm3.
 73     }
 74 joined {
 75     }
 76 left {
 77     }
 78 partitioned {
 79     }
 80 )
 81 2018-08-07T10:30:44.374227+01:00 0 [Note] WSREP: Save the discovered primary-component to disk
 82 2018-08-07T10:30:44.374354+01:00 0 [Note] WSREP: discarding pending addr without UUID: tcp://192.168.0.103:4567
 83 2018-08-07T10:30:44.374361+01:00 0 [Note] WSREP: discarding pending addr proto entry 0x378db50
 84 2018-08-07T10:30:44.872036+01:00 0 [Note] WSREP: gcomm: connected
 85 2018-08-07T10:30:44.872107+01:00 0 [Note] WSREP: Shifting CLOSED -> OPEN (TO: 0)
 86 2018-08-07T10:30:44.872173+01:00 0 [Note] WSREP: Waiting for SST/IST to complete.
 87 2018-08-07T10:30:44.872327+01:00 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 0, memb_num = 2
 88 2018-08-07T10:30:44.872840+01:00 0 [Note] WSREP: STATE_EXCHANGE: sent state UUID: 2d8c2e2a-9a1c-11e8-8388-a6fcb0b47ae5
 89 2018-08-07T10:30:44.873577+01:00 0 [Note] WSREP: STATE EXCHANGE: sent state msg: 2d8c2e2a-9a1c-11e8-8388-a6fcb0b47ae5
 90 2018-08-07T10:30:44.874140+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 2d8c2e2a-9a1c-11e8-8388-a6fcb0b47ae5 from 0 (zlm3)
 91 2018-08-07T10:30:44.874158+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 2d8c2e2a-9a1c-11e8-8388-a6fcb0b47ae5 from 1 (zlm2)
 92 2018-08-07T10:30:44.874169+01:00 0 [Note] WSREP: Quorum results:
 93     version    = 4,
 94     component  = PRIMARY,
 95     conf_id    = 1,
 96     members    = 1/2 (primary/total),
 97     act_id     = 18,
 98     last_appl. = -1,
 99     protocols  = 0/8/3 (gcs/repl/appl),
100     group UUID = bd5525ab-9a15-11e8-aa0f-4b830c783fc7
101 2018-08-07T10:30:44.874173+01:00 0 [Note] WSREP: Flow-control interval: [141, 141] //The scope of fc interval.
102 2018-08-07T10:30:44.874176+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
103 2018-08-07T10:30:44.874179+01:00 0 [Note] WSREP: Shifting OPEN -> PRIMARY (TO: 18)
104 2018-08-07T10:30:44.874224+01:00 2 [Note] WSREP: State transfer required: 
105     Group state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:18
106     Local state: 00000000-0000-0000-0000-000000000000:-1
107 2018-08-07T10:30:44.874231+01:00 2 [Note] WSREP: REPL Protocols: 8 (3, 2)
108 2018-08-07T10:30:44.874237+01:00 2 [Note] WSREP: New cluster view: global state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:18, view# 2: Primary, number of nodes: 2, my index: 0, protocol version 3
109 2018-08-07T10:30:44.874241+01:00 2 [Note] WSREP: Setting wsrep_ready to true
110 2018-08-07T10:30:44.874243+01:00 2 [Warning] WSREP: Gap in state sequence. Need state transfer.
111 2018-08-07T10:30:44.874245+01:00 2 [Note] WSREP: Setting wsrep_ready to false
112 2018-08-07T10:30:44.874315+01:00 0 [Note] WSREP: Initiating SST/IST transfer on JOINER side (wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.102' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '4181' --binlog '/data/mysql/mysql3308/logs/mysql-bin' )
113 which: no socat in (/usr/sbin:/sbin:/opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/mysql/bin:/usr/local/zabbix/bin:/usr/local/zabbix/sbin)
114     2018-08-07T08:30:45.523175Z WSREP_SST: [ERROR] ******************* FATAL ERROR ********************** 
115     2018-08-07T08:30:45.524742Z WSREP_SST: [ERROR] socat not found in path: /usr/sbin:/sbin:/opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/mysql/bin:/usr/local/zabbix/bin:/usr/local/zabbix/sbin
116     2018-08-07T08:30:45.525848Z WSREP_SST: [ERROR] ****************************************************** 
117 2018-08-07T10:30:45.526151+01:00 0 [ERROR] WSREP: Failed to read 'ready <addr>' from: wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.102' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '4181' --binlog '/data/mysql/mysql3308/logs/mysql-bin' 
118     Read: '(null)'
119 2018-08-07T10:30:45.526151+01:00 0 [ERROR] WSREP: Process completed with error: wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.102' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '4181' --binlog '/data/mysql/mysql3308/logs/mysql-bin' : 2 (No such file or directory)
120 2018-08-07T10:30:45.526201+01:00 2 [ERROR] WSREP: Failed to prepare for 'xtrabackup-v2' SST. Unrecoverable.
121 2018-08-07T10:30:45.526208+01:00 2 [ERROR] Aborting
122 
123 2018-08-07T10:30:45.526211+01:00 2 [Note] Giving 2 client threads a chance to die gracefully
124 2018-08-07T10:30:47.373430+01:00 0 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') turning message relay requesting off
125 2018-08-07T10:30:47.526391+01:00 2 [Note] WSREP: Closing send monitor...
126 2018-08-07T10:30:47.526431+01:00 2 [Note] WSREP: Closed send monitor.
127 2018-08-07T10:30:47.526455+01:00 2 [Note] WSREP: gcomm: terminating thread
128 2018-08-07T10:30:47.526465+01:00 2 [Note] WSREP: gcomm: joining thread
129 2018-08-07T10:30:47.526479+01:00 1 [Note] WSREP: rollbacker thread exiting
130 2018-08-07T10:30:47.526675+01:00 2 [Note] WSREP: gcomm: closing backend
131 2018-08-07T10:30:50.874663+01:00 2 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') connection to peer bd54497d with addr tcp://192.168.1.101:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout)
132 2018-08-07T10:30:50.874839+01:00 2 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: tcp://192.168.1.101:4567 
133 2018-08-07T10:30:51.877925+01:00 2 [Note] WSREP: (2cf33746, 'tcp://0.0.0.0:4567') reconnecting to bd54497d (tcp://192.168.1.101:4567), attempt 0
134 2018-08-07T10:30:53.026867+01:00 2 [Note] WSREP: declaring node with index 1 suspected, timeout PT5S (evs.suspect_timeout)
135 2018-08-07T10:30:53.027032+01:00 2 [Note] WSREP: evs::proto(2cf33746, LEAVING, view_id(REG,2cf33746,2)) suspecting node: bd54497d
136 2018-08-07T10:30:53.027042+01:00 2 [Note] WSREP: evs::proto(2cf33746, LEAVING, view_id(REG,2cf33746,2)) suspected node without join message, declaring inactive
137 2018-08-07T10:30:53.027063+01:00 2 [Note] WSREP: Current view of cluster as seen by this node
138 view (view_id(NON_PRIM,2cf33746,2)
139 memb {
140     2cf33746,0 //Only remains the first node zlm2.
141     }
142 joined {
143     }
144 left {
145     }
146 partitioned {
147     bd54497d,0 //Node zlm3 became a suspected node and be moved here.
148     }
149 )
150 2018-08-07T10:30:53.027092+01:00 2 [Note] WSREP: Current view of cluster as seen by this node
151 view ((empty))
152 2018-08-07T10:30:53.027370+01:00 2 [Note] WSREP: gcomm: closed
153 2018-08-07T10:30:53.027421+01:00 0 [Note] WSREP: New COMPONENT: primary = no, bootstrap = no, my_idx = 0, memb_num = 1
154 2018-08-07T10:30:53.027435+01:00 0 [Note] WSREP: Flow-control interval: [100, 100]
155 2018-08-07T10:30:53.027438+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
156 2018-08-07T10:30:53.027443+01:00 0 [Note] WSREP: Received NON-PRIMARY.
157 2018-08-07T10:30:53.027447+01:00 0 [Note] WSREP: Shifting PRIMARY -> OPEN (TO: 18)
158 2018-08-07T10:30:53.027458+01:00 0 [Note] WSREP: Received self-leave message.
159 2018-08-07T10:30:53.027463+01:00 0 [Note] WSREP: Flow-control interval: [0, 0]
160 2018-08-07T10:30:53.027466+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
161 2018-08-07T10:30:53.027469+01:00 0 [Note] WSREP: Received SELF-LEAVE. Closing connection.
162 2018-08-07T10:30:53.027472+01:00 0 [Note] WSREP: Shifting OPEN -> CLOSED (TO: 18)
163 2018-08-07T10:30:53.027477+01:00 0 [Note] WSREP: RECV thread exiting 0: Success
164 2018-08-07T10:30:53.027719+01:00 2 [Note] WSREP: recv_thread() joined.
165 2018-08-07T10:30:53.027732+01:00 2 [Note] WSREP: Closing replication queue.
166 2018-08-07T10:30:53.027738+01:00 2 [Note] WSREP: Closing slave action queue.
167 2018-08-07T10:30:53.027796+01:00 0 [Note] WSREP: Waiting for active wsrep applier to exit
168 2018-08-07T10:30:53.027804+01:00 0 [Note] WSREP: Service disconnected.
169 2018-08-07T10:30:53.027807+01:00 0 [Note] WSREP: Waiting to close threads......
170 2018-08-07T10:30:58.028048+01:00 0 [Note] WSREP: Some threads may fail to exit.
171 2018-08-07T10:30:58.028088+01:00 0 [Note] Binlog end
172 2018-08-07T10:30:58.028341+01:00 0 [Note] mysqld: Shutdown complete
173 
174 [root@zlm3 10:35:04 /data/mysql/mysql3308/data]
175 #ps aux|grep mysqld
176 root      4672  0.0  0.0 112640   960 pts/0    R+   10:35   0:00 grep --color=auto mysqld
177 
178 //Let's see the detail of ERROR message.
179 [root@zlm3 10:35:06 /data/mysql/mysql3308/data]
180 #cat error.log |grep ERROR
181     2018-08-07T08:30:45.523175Z WSREP_SST: [ERROR] ******************* FATAL ERROR ********************** 
182     2018-08-07T08:30:45.524742Z WSREP_SST: [ERROR] socat not found in path: /usr/sbin:/sbin:/opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/mysql/bin:/usr/local/zabbix/bin:/usr/local/zabbix/sbin
183     2018-08-07T08:30:45.525848Z WSREP_SST: [ERROR] ****************************************************** 
184 2018-08-07T10:30:45.526151+01:00 0 [ERROR] WSREP: Failed to read 'ready <addr>' from: wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.102' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '4181' --binlog '/data/mysql/mysql3308/logs/mysql-bin' 
185 2018-08-07T10:30:45.526151+01:00 0 [ERROR] WSREP: Process completed with error: wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.102' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '4181' --binlog '/data/mysql/mysql3308/logs/mysql-bin' : 2 (No such file or directory)
186 2018-08-07T10:30:45.526201+01:00 2 [ERROR] WSREP: Failed to prepare for 'xtrabackup-v2' SST. Unrecoverable.
187 2018-08-07T10:30:45.526208+01:00 2 [ERROR] Aborting
188 
189 //It says that we are absent of "socat" package.
190 
191 [root@zlm3 10:43:15 /data/mysql/mysql3308/data]
192 #yum install socat
193 ... //Omitted.
194 
195 Warning: RPMDB altered outside of yum.
196   Installing : socat-1.7.3.2-2.el7.x86_64                                                                                                        1/1 
197   Verifying  : socat-1.7.3.2-2.el7.x86_64                                                                                                        1/1 
198 
199 Installed:
200   socat.x86_64 0:1.7.3.2-2.el7                                                                                                                       
201 
202 Complete!
203 
204 //Start the PXC on zlm2 again.
205 [root@zlm3 10:35:58 /data/mysql/mysql3308/data]
206 #mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf &
207 [1] 4725
208 
209 [root@zlm3 10:53:02 /data/mysql/mysql3308/data]
210 #tail -f error.log 
211 2018-08-07T10:53:15.116919+01:00 0 [Note] WSREP: Member 0.0 (zlm3) requested state transfer from '*any*'. Selected 1.0 (zlm2)(SYNCED) as donor.
212 2018-08-07T10:53:15.117004+01:00 0 [Note] WSREP: Shifting PRIMARY -> JOINER (TO: 18)
213 2018-08-07T10:53:15.117035+01:00 2 [Note] WSREP: Requesting state transfer: success, donor: 1
214 2018-08-07T10:53:15.117051+01:00 2 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> bd5525ab-9a15-11e8-aa0f-4b830c783fc7:18
215 2018-08-07T10:53:15.696885+01:00 0 [Warning] WSREP: 1.0 (zlm2): State transfer to 0.0 (zlm3) failed: -2 (No such file or directory)
216 2018-08-07T10:53:15.696918+01:00 0 [ERROR] WSREP: gcs/src/gcs_group.cpp:gcs_group_handle_join_msg():766: Will never receive state. Need to abort.
217 2018-08-07T10:53:15.696940+01:00 0 [Note] WSREP: gcomm: terminating thread
218 2018-08-07T10:53:15.696949+01:00 0 [Note] WSREP: gcomm: joining thread
219 2018-08-07T10:53:15.697076+01:00 0 [Note] WSREP: gcomm: closing backend
220 2018-08-07T10:53:17.026257+01:00 0 [Note] WSREP: (5167e9c7, 'tcp://0.0.0.0:4567') turning message relay requesting off
221 2018-08-07T10:53:19.026327+01:00 0 [Note] WSREP: (5167e9c7, 'tcp://0.0.0.0:4567') connection to peer bd54497d with addr tcp://192.168.1.101:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout)
222 2018-08-07T10:53:19.026394+01:00 0 [Note] WSREP: (5167e9c7, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: tcp://192.168.1.101:4567 
223 2018-08-07T10:53:20.086590+01:00 0 [Note] WSREP: (5167e9c7, 'tcp://0.0.0.0:4567') reconnecting to bd54497d (tcp://192.168.1.101:4567), attempt 0
224 2018-08-07T10:53:20.786605+01:00 0 [Note] WSREP: declaring node with index 1 suspected, timeout PT5S (evs.suspect_timeout)
225 2018-08-07T10:53:20.786778+01:00 0 [Note] WSREP: evs::proto(5167e9c7, LEAVING, view_id(REG,5167e9c7,4)) suspecting node: bd54497d
226 2018-08-07T10:53:20.786826+01:00 0 [Note] WSREP: evs::proto(5167e9c7, LEAVING, view_id(REG,5167e9c7,4)) suspected node without join message, declaring inactive
227 2018-08-07T10:53:20.786941+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
228 view (view_id(NON_PRIM,5167e9c7,4)
229 memb {
230     5167e9c7,0
231     }
232 joined {
233     }
234 left {
235     }
236 partitioned {
237     bd54497d,0
238     }
239 )
240 2018-08-07T10:53:20.787061+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
241 view ((empty))
242 2018-08-07T10:53:20.787324+01:00 0 [Note] WSREP: gcomm: closed
243 2018-08-07T10:53:20.787362+01:00 0 [Note] WSREP: mysqld: Terminated.
244 ^C
245 
246 [root@zlm3 10:53:48 /data/mysql/mysql3308/data]
247 #cat error.log |grep ERROR
248 2018-08-07T10:53:15.696918+01:00 0 [ERROR] WSREP: gcs/src/gcs_group.cpp:gcs_group_handle_join_msg():766: Will never receive state. Need to abort.
249     
250 [root@zlm3 11:13:14 /data/mysql/mysql3308/data]
251 #ps aux|grep mysqld
252 root      6785  0.0  0.0 112640   960 pts/0    R+   11:20   0:00 grep --color=auto mysqld
253 [1]+  Aborted                 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf
254 
255 [root@zlm3 11:20:16 /data/mysql/mysql3308/data]
256 #cat error.log |grep ERROR
257 2018-08-07T11:13:15.619315+01:00 0 [ERROR] WSREP: gcs/src/gcs_group.cpp:gcs_group_handle_join_msg():766: Will never receive state. Need to abort.
258     2018-08-07T09:13:45.246252Z WSREP_SST: [ERROR] ******************* FATAL ERROR ********************** 
259     2018-08-07T09:13:45.247507Z WSREP_SST: [ERROR] Error while getting data from donor node:  exit codes: 255 0
260     2018-08-07T09:13:45.254423Z WSREP_SST: [ERROR] ****************************************************** 
261     2018-08-07T09:13:45.258735Z WSREP_SST: [ERROR] Cleanup after exit with status:32
262     2018-08-07T09:14:10.137993Z WSREP_SST: [ERROR] ******************* FATAL ERROR ********************** 
263     2018-08-07T09:14:10.139033Z WSREP_SST: [ERROR] Possible timeout in receving first data from donor in gtid/keyring stage
264     2018-08-07T09:14:10.140912Z WSREP_SST: [ERROR] ****************************************************** 
265     2018-08-07T09:14:10.141951Z WSREP_SST: [ERROR] Cleanup after exit with status:32
266     
267 //There's still one error which cause the mysqld process cannot be started.

    Eventually,In order to make the
gtid more clear,I executed “reset master” on binlog server(zlm3) and
then copyed those necessary binlogs which were transfered from original
master.Further more,I changed back the uuid of zlm3.Because I found that
there’s no need to change it at all.

Procedure

 

    The official website of
PXC:

 

 1 (zlm@192.168.1.101 3308)[(none)]>select user,host from mysql.user;
 2 +---------------+-------------+
 3 | user          | host        |
 4 +---------------+-------------+
 5 | zlm           | 192.168.1.% |
 6 | mysql.session | localhost   |
 7 | mysql.sys     | localhost   |
 8 | root          | localhost   |
 9 | sst           | localhost   |
10 +---------------+-------------+
11 5 rows in set (0.02 sec)
12 
13 (zlm@192.168.1.101 3308)[(none)]>show grants for sst@localhost;
14 +---------------------------------------------------------------------------+
15 | Grants for sst@localhost                                                  |
16 +---------------------------------------------------------------------------+
17 | GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'sst'@'localhost' |
18 +---------------------------------------------------------------------------+
19 1 row in set (0.00 sec)
20 
21 (zlm@192.168.1.101 3308)[(none)]>grant process on *.* to 'sst'@'localhost';
22 ERROR 1045 (28000): Access denied for user 'zlm'@'192.168.1.%' (using password: YES)
23 (zlm@192.168.1.101 3308)[(none)]>exit
24 Bye
25 
26 [root@zlm2 10:07:32 ~]
27 #mysql -uroot -pPassw0rd -hlocalhost -S /tmp/mysql3308.sock
28 mysql: [Warning] Using a password on the command line interface can be insecure.
29 Welcome to the MySQL monitor.  Commands end with ; or \g.
30 Your MySQL connection id is 9
31 Server version: 5.7.22-22-29.26-log Percona XtraDB Cluster binary (GPL) 5.7.22-29.26, Revision da86071, wsrep_29.26
32 
33 Copyright (c) 2009-2018 Percona LLC and/or its affiliates
34 Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
35 
36 Oracle is a registered trademark of Oracle Corporation and/or its
37 affiliates. Other names may be trademarks of their respective
38 owners.
39 
40 Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
41 
42 (root@localhost mysql3308.sock)[(none)]>grant process on *.* to sst@localhost;
43 Query OK, 0 rows affected, 1 warning (0.01 sec)
44 
45 (root@localhost mysql3308.sock)[(none)]>show grants for sst@localhost;
46 +------------------------------------------------------------------------------------+
47 | Grants for sst@localhost                                                           |
48 +------------------------------------------------------------------------------------+
49 | GRANT RELOAD, PROCESS, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'sst'@'localhost' |
50 +------------------------------------------------------------------------------------+
51 1 row in set (0.00 sec)
 1 [root@zlm3 09:01:42 /data/mysql/mysql3308/data]
 2 #tail -f error.log 
 3 2018-08-09T09:02:44.880772+01:00 0 [Note] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: tcp://192.168.1.103:4567 
 4 2018-08-09T09:02:44.880866+01:00 0 [Note] WSREP: declaring 13eae368 at tcp://192.168.1.101:4567 stable
 5 2018-08-09T09:02:44.880877+01:00 0 [Note] WSREP: forgetting 74f3db69 (tcp://192.168.1.103:4567)
 6 2018-08-09T09:02:44.880899+01:00 0 [Note] WSREP: (23fb8f7a, 'tcp://0.0.0.0:4567') turning message relay requesting off
 7 2018-08-09T09:02:44.884451+01:00 0 [Note] WSREP: Node 13eae368 state primary
 8 2018-08-09T09:02:44.887086+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
 9 view (view_id(PRIM,13eae368,12)
10 memb {
11     13eae368,0
12     23fb8f7a,0
13     }
14 joined {
15     }
16 left {
17     }
18 partitioned {
19     74f3db69,0
20     }
21 )
22 2018-08-09T09:02:44.887205+01:00 0 [Note] WSREP: Save the discovered primary-component to disk
23 2018-08-09T09:02:44.887402+01:00 0 [Note] WSREP: forgetting 74f3db69 (tcp://192.168.1.103:4567)
24 2018-08-09T09:02:44.887688+01:00 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 1, memb_num = 2
25 2018-08-09T09:02:44.887705+01:00 0 [Note] WSREP: STATE EXCHANGE: Waiting for state UUID.
26 2018-08-09T09:02:44.888444+01:00 0 [Note] WSREP: STATE EXCHANGE: sent state msg: 37401cc1-9ba2-11e8-87aa-2f9e83444bbe
27 2018-08-09T09:02:44.889150+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 37401cc1-9ba2-11e8-87aa-2f9e83444bbe from 0 (zlm2)
28 2018-08-09T09:02:44.889187+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 37401cc1-9ba2-11e8-87aa-2f9e83444bbe from 1 (zlm3)
29 2018-08-09T09:02:44.889198+01:00 0 [Note] WSREP: Quorum results:
30     version    = 4,
31     component  = PRIMARY,
32     conf_id    = 11,
33     members    = 2/2 (primary/total),
34     act_id     = 22,
35     last_appl. = 0,
36     protocols  = 0/8/3 (gcs/repl/appl),
37     group UUID = bd5525ab-9a15-11e8-aa0f-4b830c783fc7
38 2018-08-09T09:02:44.889206+01:00 0 [Note] WSREP: Flow-control interval: [141, 141]
39 2018-08-09T09:02:44.889210+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
40 2018-08-09T09:02:44.889304+01:00 2 [Note] WSREP: REPL Protocols: 8 (3, 2)
41 2018-08-09T09:02:44.889315+01:00 2 [Note] WSREP: New cluster view: global state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22, view# 12: Primary, number of nodes: 2, my index: 1, protocol version 3
42 2018-08-09T09:02:44.889319+01:00 2 [Note] WSREP: Setting wsrep_ready to true
43 2018-08-09T09:02:44.889324+01:00 2 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 2 -> 2) (Increment: 3 -> 2)
44 2018-08-09T09:02:44.889328+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
45 2018-08-09T09:02:44.889336+01:00 2 [Note] WSREP: Assign initial position for certification: 22, protocol version: 3
46 2018-08-09T09:02:44.889355+01:00 0 [Note] WSREP: Service thread queue flushed.
47 2018-08-09T09:02:50.000210+01:00 0 [Note] WSREP:  cleaning up 74f3db69 (tcp://192.168.1.103:4567)

 

 

[root@zlm2 10:11:10 ~]
#ps aux|grep mysqld
mysql     7055  0.1 22.2 1564952 225952 pts/3  Sl   09:30   0:04 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster
root      7661  0.0  0.0 112640   960 pts/0    R+   10:11   0:00 grep --color=auto mysqld

[root@zlm2 10:11:28 ~]
#mysqladmin -uzlm -pzlmzlm -h192.168.1.101 -P3308 shutdown
mysqladmin: [Warning] Using a password on the command line interface can be insecure.

[root@zlm2 10:11:50 ~]
#ps aux|grep mysqld
root      7672  0.0  0.0 112640   960 pts/0    R+   10:12   0:00 grep --color=auto mysqld

[root@zlm2 09:39:30 /data/mysql/mysql3308/data]
#mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster &
[2] 7690
[1]   Done                    mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster

[root@zlm2 10:12:44 /data/mysql/mysql3308/data]
#ps aux|grep mysqld
mysql     7690  5.8 20.6 1556488 210296 pts/3  Sl   10:12   0:01 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster
root      7729  0.0  0.0 112640   960 pts/3    R+   10:13   0:00 grep --color=auto mysqld

//No ERROR messages were found in error log.
 1 (zlm@192.168.1.101 3308)[(none)]>show global status like '%wsrep%';
 2 +----------------------------------+----------------------------------------------------------+
 3 | Variable_name                    | Value                                                    |
 4 +----------------------------------+----------------------------------------------------------+
 5 | wsrep_local_state_uuid           | bd5525ab-9a15-11e8-aa0f-4b830c783fc7                     |
 6 | wsrep_protocol_version           | 8                                                        |
 7 | wsrep_last_applied               | 22                                                       |
 8 | wsrep_last_committed             | 22                                                       |
 9 | wsrep_replicated                 | 0                                                        |
10 | wsrep_replicated_bytes           | 0                                                        |
11 | wsrep_repl_keys                  | 0                                                        |
12 | wsrep_repl_keys_bytes            | 0                                                        |
13 | wsrep_repl_data_bytes            | 0                                                        |
14 | wsrep_repl_other_bytes           | 0                                                        |
15 | wsrep_received                   | 21                                                       |
16 | wsrep_received_bytes             | 2733                                                     |
17 | wsrep_local_commits              | 0                                                        |
18 | wsrep_local_cert_failures        | 0                                                        |
19 | wsrep_local_replays              | 0                                                        |
20 | wsrep_local_send_queue           | 0                                                        |
21 | wsrep_local_send_queue_max       | 1                                                        |
22 | wsrep_local_send_queue_min       | 0                                                        |
23 | wsrep_local_send_queue_avg       | 0.000000                                                 |
24 | wsrep_local_recv_queue           | 0                                                        |
25 | wsrep_local_recv_queue_max       | 2                                                        |
26 | wsrep_local_recv_queue_min       | 0                                                        |
27 | wsrep_local_recv_queue_avg       | 0.047619                                                 |
28 | wsrep_local_cached_downto        | 0                                                        |
29 | wsrep_flow_control_paused_ns     | 0                                                        |
30 | wsrep_flow_control_paused        | 0.000000                                                 |
31 | wsrep_flow_control_sent          | 0                                                        |
32 | wsrep_flow_control_recv          | 0                                                        |
33 | wsrep_flow_control_interval      | [ 173, 173 ]                                             |
34 | wsrep_flow_control_interval_low  | 173                                                      |
35 | wsrep_flow_control_interval_high | 173                                                      |
36 | wsrep_flow_control_status        | OFF                                                      |
37 | wsrep_cert_deps_distance         | 0.000000                                                 |
38 | wsrep_apply_oooe                 | 0.000000                                                 |
39 | wsrep_apply_oool                 | 0.000000                                                 |
40 | wsrep_apply_window               | 0.000000                                                 |
41 | wsrep_commit_oooe                | 0.000000                                                 |
42 | wsrep_commit_oool                | 0.000000                                                 |
43 | wsrep_commit_window              | 0.000000                                                 |
44 | wsrep_local_state                | 4                                                        |
45 | wsrep_local_state_comment        | Synced                                                   |
46 | wsrep_cert_index_size            | 0                                                        |
47 | wsrep_cert_bucket_count          | 22                                                       |
48 | wsrep_gcache_pool_size           | 1712                                                     |
49 | wsrep_causal_reads               | 0                                                        |
50 | wsrep_cert_interval              | 0.000000                                                 |
51 | wsrep_ist_receive_status         |                                                          |
52 | wsrep_ist_receive_seqno_start    | 0                                                        |
53 | wsrep_ist_receive_seqno_current  | 0                                                        |
54 | wsrep_ist_receive_seqno_end      | 0                                                        |
55 | wsrep_incoming_addresses         | 192.168.1.101:3308,192.168.1.102:3308,192.168.1.103:3308 |
56 | wsrep_desync_count               | 0                                                        |
57 | wsrep_evs_delayed                |                                                          |
58 | wsrep_evs_evict_list             |                                                          |
59 | wsrep_evs_repl_latency           | 0/0/0/0/0                                                |
60 | wsrep_evs_state                  | OPERATIONAL                                              |
61 | wsrep_gcomm_uuid                 | 13eae368-9b79-11e8-9053-338307f4c6cc                     |
62 | wsrep_cluster_conf_id            | 11                                                       |
63 | wsrep_cluster_size               | 3                                                        | //It means I've got three nodes in PXC.
64 | wsrep_cluster_state_uuid         | bd5525ab-9a15-11e8-aa0f-4b830c783fc7                     |
65 | wsrep_cluster_status             | Primary                                                  |
66 | wsrep_connected                  | ON                                                       |
67 | wsrep_local_bf_aborts            | 0                                                        |
68 | wsrep_local_index                | 0                                                        |
69 | wsrep_provider_name              | Galera                                                   |
70 | wsrep_provider_vendor            | Codership Oy <info@codership.com>                        |
71 | wsrep_provider_version           | 3.26(r)                                                  |
72 | wsrep_ready                      | ON                                                       |
73 +----------------------------------+----------------------------------------------------------+
74 68 rows in set (0.00 sec)

 

 

Suplemented on August 9.

 

Preface

*Step
4: **Implement a slave with binlog server.***

  1 //Change the parameter "wsrep_sst_method" into "rysnc" in my3308.cnf file.
  2 #loose-wsrep_sst_method=xtrabackup-v2
  3 loose-wsrep_sst_method=rsync
  4 
  5 [root@zlm4 07:51:38 /data/mysql/mysql3308/data]
  6 #tail -f error.log 
  7 2018-08-09T07:52:21.038606+01:00 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
  8 2018-08-09T07:52:21.041387+01:00 0 [Warning] Insecure configuration for --secure-file-priv: Location is accessible to all OS users. Consider choosing a different directory.
  9 2018-08-09T07:52:21.041490+01:00 0 [Note] mysqld (mysqld 5.7.22-22-29.26-log) starting as process 5205 ...
 10 2018-08-09T07:52:21.044740+01:00 0 [Note] WSREP: Setting wsrep_ready to false
 11 2018-08-09T07:52:21.044761+01:00 0 [Note] WSREP: No pre-stored wsrep-start position found. Skipping position initialization.
 12 2018-08-09T07:52:21.044767+01:00 0 [Note] WSREP: wsrep_load(): loading provider library '/usr/local/mysql/lib/libgalera_smm.so'
 13 2018-08-09T07:52:21.050035+01:00 0 [Note] WSREP: wsrep_load(): Galera 3.26(r) by Codership Oy <info@codership.com> loaded successfully.
 14 2018-08-09T07:52:21.050186+01:00 0 [Note] WSREP: CRC-32C: using hardware acceleration.
 15 2018-08-09T07:52:21.050565+01:00 0 [Warning] WSREP: Could not open state file for reading: '/data/mysql/mysql3308/data//grastate.dat'
 16 2018-08-09T07:52:21.050585+01:00 0 [Warning] WSREP: No persistent state found. Bootstraping with default state
 17 2018-08-09T07:52:21.050637+01:00 0 [Note] WSREP: Found saved state: 00000000-0000-0000-0000-000000000000:-1, safe_to_bootstrap: 1
 18 2018-08-09T07:52:21.055209+01:00 0 [Note] WSREP: Passing config to GCS: base_dir = /data/mysql/mysql3308/data/; base_host = 192.168.1.103; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.join_retrans_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 10; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.user_send_window = 4; evs.view_forget_timeout = PT24H; gcache.dir = /data/mysql/mysql3308/data/; gcache.freeze_purge_at_seqno = -1; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /data/mysql/mysql3308/data//galera.cache; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1; gcs.fc_limit = 100; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.segment = 0; gmcast.version = 0; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.npvo = false; pc.recovery = 1; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 8; socket.checksum = 2; socket.recv_buf_size = 212992; 
 19 2018-08-09T07:52:21.069257+01:00 0 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> 00000000-0000-0000-0000-000000000000:-1
 20 2018-08-09T07:52:21.071550+01:00 0 [Note] WSREP: Assign initial position for certification: -1, protocol version: -1
 21 2018-08-09T07:52:21.071584+01:00 0 [Note] WSREP: Preparing to initiate SST/IST
 22 2018-08-09T07:52:21.071589+01:00 0 [Note] WSREP: Starting replication
 23 2018-08-09T07:52:21.071606+01:00 0 [Note] WSREP: Setting initial position to 00000000-0000-0000-0000-000000000000:-1
 24 2018-08-09T07:52:21.071812+01:00 0 [Note] WSREP: Using CRC-32C for message checksums.
 25 2018-08-09T07:52:21.071888+01:00 0 [Note] WSREP: gcomm thread scheduling priority set to other:0 
 26 2018-08-09T07:52:21.072021+01:00 0 [Warning] WSREP: Fail to access the file (/data/mysql/mysql3308/data//gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown
 27 2018-08-09T07:52:21.072030+01:00 0 [Note] WSREP: Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown
 28 2018-08-09T07:52:21.072800+01:00 0 [Note] WSREP: GMCast version 0
 29 2018-08-09T07:52:21.072942+01:00 0 [Note] WSREP: (61aad7a7, 'tcp://0.0.0.0:4567') listening at tcp://0.0.0.0:4567
 30 2018-08-09T07:52:21.072942+01:00 0 [Note] WSREP: (61aad7a7, 'tcp://0.0.0.0:4567') multicast: , ttl: 1
 31 2018-08-09T07:52:21.073357+01:00 0 [Note] WSREP: EVS version 0
 32 2018-08-09T07:52:21.073926+01:00 0 [Note] WSREP: gcomm: connecting to group 'pxc_wubx', peer '192.168.1.101:,192.168.1.102:,192.168.1.103:'
 33 2018-08-09T07:52:21.076189+01:00 0 [Note] WSREP: (61aad7a7, 'tcp://0.0.0.0:4567') connection established to 61aad7a7 tcp://192.168.1.103:4567
 34 2018-08-09T07:52:21.076220+01:00 0 [Warning] WSREP: (61aad7a7, 'tcp://0.0.0.0:4567') address 'tcp://192.168.1.103:4567' points to own listening address, blacklisting
 35 2018-08-09T07:52:21.078146+01:00 0 [Note] WSREP: (61aad7a7, 'tcp://0.0.0.0:4567') connection established to 13eae368 tcp://192.168.1.101:4567
 36 2018-08-09T07:52:21.078312+01:00 0 [Note] WSREP: (61aad7a7, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: 
 37 2018-08-09T07:52:21.080311+01:00 0 [Note] WSREP: (61aad7a7, 'tcp://0.0.0.0:4567') connection established to 23fb8f7a tcp://192.168.1.102:4567
 38 2018-08-09T07:52:21.583579+01:00 0 [Note] WSREP: declaring 13eae368 at tcp://192.168.1.101:4567 stable
 39 2018-08-09T07:52:21.583614+01:00 0 [Note] WSREP: declaring 23fb8f7a at tcp://192.168.1.102:4567 stable
 40 2018-08-09T07:52:21.584818+01:00 0 [Note] WSREP: Node 13eae368 state primary
 41 2018-08-09T07:52:21.589478+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
 42 view (view_id(PRIM,13eae368,9)
 43 memb {
 44     13eae368,0
 45     23fb8f7a,0
 46     61aad7a7,0
 47     }
 48 joined {
 49     }
 50 left {
 51     }
 52 partitioned {
 53     }
 54 )
 55 2018-08-09T07:52:21.589519+01:00 0 [Note] WSREP: Save the discovered primary-component to disk
 56 2018-08-09T07:52:22.075191+01:00 0 [Note] WSREP: gcomm: connected
 57 2018-08-09T07:52:22.075272+01:00 0 [Note] WSREP: Shifting CLOSED -> OPEN (TO: 0)
 58 2018-08-09T07:52:22.075348+01:00 0 [Note] WSREP: Waiting for SST/IST to complete.
 59 2018-08-09T07:52:22.075642+01:00 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 2, memb_num = 3
 60 2018-08-09T07:52:22.075658+01:00 0 [Note] WSREP: STATE EXCHANGE: Waiting for state UUID.
 61 2018-08-09T07:52:22.075689+01:00 0 [Note] WSREP: STATE EXCHANGE: sent state msg: 61f621e1-9b98-11e8-a772-bfae300f8d05
 62 2018-08-09T07:52:22.075697+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 61f621e1-9b98-11e8-a772-bfae300f8d05 from 0 (zlm2)
 63 2018-08-09T07:52:22.075704+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 61f621e1-9b98-11e8-a772-bfae300f8d05 from 1 (zlm3)
 64 2018-08-09T07:52:22.077812+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 61f621e1-9b98-11e8-a772-bfae300f8d05 from 2 (zlm4)
 65 2018-08-09T07:52:22.077845+01:00 0 [Note] WSREP: Quorum results:
 66     version    = 4,
 67     component  = PRIMARY,
 68     conf_id    = 8,
 69     members    = 2/3 (primary/total),
 70     act_id     = 22,
 71     last_appl. = -1,
 72     protocols  = 0/8/3 (gcs/repl/appl),
 73     group UUID = bd5525ab-9a15-11e8-aa0f-4b830c783fc7
 74 2018-08-09T07:52:22.077853+01:00 0 [Note] WSREP: Flow-control interval: [173, 173]
 75 2018-08-09T07:52:22.077858+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
 76 2018-08-09T07:52:22.077864+01:00 0 [Note] WSREP: Shifting OPEN -> PRIMARY (TO: 22)
 77 2018-08-09T07:52:22.077926+01:00 2 [Note] WSREP: State transfer required: 
 78     Group state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22
 79     Local state: 00000000-0000-0000-0000-000000000000:-1
 80 2018-08-09T07:52:22.077935+01:00 2 [Note] WSREP: REPL Protocols: 8 (3, 2)
 81 2018-08-09T07:52:22.077943+01:00 2 [Note] WSREP: New cluster view: global state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22, view# 9: Primary, number of nodes: 3, my index: 2, protocol version 3
 82 2018-08-09T07:52:22.077948+01:00 2 [Note] WSREP: Setting wsrep_ready to true
 83 2018-08-09T07:52:22.077951+01:00 2 [Warning] WSREP: Gap in state sequence. Need state transfer.
 84 2018-08-09T07:52:22.077954+01:00 2 [Note] WSREP: Setting wsrep_ready to false
 85 2018-08-09T07:52:22.078033+01:00 0 [Note] WSREP: Initiating SST/IST transfer on JOINER side (wsrep_sst_rsync --role 'joiner' --address '192.168.1.103' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '5205' --binlog '/data/mysql/mysql3308/logs/mysql-bin' )
 86 which: no lsof in (/usr/sbin:/sbin:/opt/mysql/Percona-XtraDB-Cluster-5.7.22-rel22-29.26.1.Linux.x86_64.ssl102/bin:/root/perl5/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/mysql/bin:/usr/local/zabbix/bin:/usr/local/zabbix/sbin)
 87 2018-08-09T07:52:22.254544+01:00 0 [ERROR] WSREP: Failed to read 'ready <addr>' from: wsrep_sst_rsync --role 'joiner' --address '192.168.1.103' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '5205' --binlog '/data/mysql/mysql3308/logs/mysql-bin' 
 88     Read: ''lsof' not found in PATH' //Here's the important message of error but has not "ERROR" keyword in the same line.So I've almost missed it.
 89 2018-08-09T07:52:22.254571+01:00 0 [ERROR] WSREP: Process completed with error: wsrep_sst_rsync --role 'joiner' --address '192.168.1.103' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '5205' --binlog '/data/mysql/mysql3308/logs/mysql-bin' : 2 (No such file or directory)
 90 2018-08-09T07:52:22.254631+01:00 2 [ERROR] WSREP: Failed to prepare for 'rsync' SST. Unrecoverable.
 91 2018-08-09T07:52:22.254639+01:00 2 [ERROR] Aborting
 92 
 93 2018-08-09T07:52:22.254643+01:00 2 [Note] Giving 2 client threads a chance to die gracefully
 94 2018-08-09T07:52:24.254907+01:00 2 [Note] WSREP: Closing send monitor...
 95 2018-08-09T07:52:24.254992+01:00 2 [Note] WSREP: Closed send monitor.
 96 2018-08-09T07:52:24.255028+01:00 2 [Note] WSREP: gcomm: terminating thread
 97 2018-08-09T07:52:24.255042+01:00 2 [Note] WSREP: gcomm: joining thread
 98 2018-08-09T07:52:24.255061+01:00 1 [Note] WSREP: rollbacker thread exiting
 99 2018-08-09T07:52:24.255377+01:00 2 [Note] WSREP: gcomm: closing backend
100 2018-08-09T07:52:24.260692+01:00 2 [Note] WSREP: Current view of cluster as seen by this node
101 view (view_id(NON_PRIM,13eae368,9)
102 memb {
103     61aad7a7,0
104     }
105 joined {
106     }
107 left {
108     }
109 partitioned {
110     13eae368,0
111     23fb8f7a,0
112     }
113 )
114 2018-08-09T07:52:24.260770+01:00 2 [Note] WSREP: (61aad7a7, 'tcp://0.0.0.0:4567') turning message relay requesting off
115 2018-08-09T07:52:24.260814+01:00 2 [Note] WSREP: Current view of cluster as seen by this node
116 view ((empty))
117 2018-08-09T07:52:24.261439+01:00 2 [Note] WSREP: gcomm: closed
118 2018-08-09T07:52:24.261690+01:00 0 [Note] WSREP: New COMPONENT: primary = no, bootstrap = no, my_idx = 0, memb_num = 1
119 2018-08-09T07:52:24.261741+01:00 0 [Note] WSREP: Flow-control interval: [100, 100]
120 2018-08-09T07:52:24.261746+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
121 2018-08-09T07:52:24.261753+01:00 0 [Note] WSREP: Received NON-PRIMARY.
122 2018-08-09T07:52:24.261758+01:00 0 [Note] WSREP: Shifting PRIMARY -> OPEN (TO: 22)
123 2018-08-09T07:52:24.261767+01:00 0 [Note] WSREP: Received self-leave message.
124 2018-08-09T07:52:24.261774+01:00 0 [Note] WSREP: Flow-control interval: [0, 0]
125 2018-08-09T07:52:24.261777+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
126 2018-08-09T07:52:24.261780+01:00 0 [Note] WSREP: Received SELF-LEAVE. Closing connection.
127 2018-08-09T07:52:24.261784+01:00 0 [Note] WSREP: Shifting OPEN -> CLOSED (TO: 22)
128 2018-08-09T07:52:24.261794+01:00 0 [Note] WSREP: RECV thread exiting 0: Success
129 2018-08-09T07:52:24.262101+01:00 2 [Note] WSREP: recv_thread() joined.
130 2018-08-09T07:52:24.262113+01:00 2 [Note] WSREP: Closing replication queue.
131 2018-08-09T07:52:24.262117+01:00 2 [Note] WSREP: Closing slave action queue.
132 2018-08-09T07:52:24.262162+01:00 0 [Note] WSREP: Waiting for active wsrep applier to exit
133 2018-08-09T07:52:24.262170+01:00 0 [Note] WSREP: Service disconnected.
134 2018-08-09T07:52:24.262173+01:00 0 [Note] WSREP: Waiting to close threads......
135 2018-08-09T07:52:29.262290+01:00 0 [Note] WSREP: Some threads may fail to exit.
136 2018-08-09T07:52:29.262337+01:00 0 [Note] Binlog end
137 2018-08-09T07:52:29.262596+01:00 0 [Note] mysqld: Shutdown complete
138 
139 //Install lsof tool.
140 [root@zlm4 07:58:19 /data/mysql/mysql3308]
141 #yum -y install lsof
142 ... //Omitted.
143 
144 Installed:
145   lsof.x86_64 0:4.87-5.el7                                                                                                                           
146 
147 Complete!
148 
149 //Start the node again.
150 [root@zlm4 07:59:47 /data/mysql/mysql3308]
151 #mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf &
152 [1] 5367
153 
154 [root@zlm4 08:00:02 /data/mysql/mysql3308]
155 #jobs
156 [1]+  Running                 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf &
157 
158 //Check the error log again.
159 [root@zlm4 08:59:38 /data/mysql/mysql3308/data]
160 #tail -f error.log 
161 2018-08-09T08:00:02.899495+01:00 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
162 2018-08-09T08:00:02.900253+01:00 0 [Warning] Insecure configuration for --secure-file-priv: Location is accessible to all OS users. Consider choosing a different directory.
163 2018-08-09T08:00:02.900326+01:00 0 [Note] mysqld (mysqld 5.7.22-22-29.26-log) starting as process 5367 ...
164 2018-08-09T08:00:02.902758+01:00 0 [Note] WSREP: Setting wsrep_ready to false
165 2018-08-09T08:00:02.902772+01:00 0 [Note] WSREP: No pre-stored wsrep-start position found. Skipping position initialization.
166 2018-08-09T08:00:02.902776+01:00 0 [Note] WSREP: wsrep_load(): loading provider library '/usr/local/mysql/lib/libgalera_smm.so'
167 2018-08-09T08:00:02.906503+01:00 0 [Note] WSREP: wsrep_load(): Galera 3.26(r) by Codership Oy <info@codership.com> loaded successfully.
168 2018-08-09T08:00:02.906567+01:00 0 [Note] WSREP: CRC-32C: using hardware acceleration.
169 2018-08-09T08:00:02.906904+01:00 0 [Note] WSREP: Found saved state: 00000000-0000-0000-0000-000000000000:-1, safe_to_bootstrap: 1
170 2018-08-09T08:00:02.911237+01:00 0 [Note] WSREP: Passing config to GCS: base_dir = /data/mysql/mysql3308/data/; base_host = 192.168.1.103; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.join_retrans_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 10; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.user_send_window = 4; evs.view_forget_timeout = PT24H; gcache.dir = /data/mysql/mysql3308/data/; gcache.freeze_purge_at_seqno = -1; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /data/mysql/mysql3308/data//galera.cache; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1; gcs.fc_limit = 100; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.segment = 0; gmcast.version = 0; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.npvo = false; pc.recovery = 1; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 8; socket.checksum = 2; socket.recv_buf_size = 212992; 
171 2018-08-09T08:00:02.921480+01:00 0 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> 00000000-0000-0000-0000-000000000000:-1
172 2018-08-09T08:00:02.923600+01:00 0 [Note] WSREP: Assign initial position for certification: -1, protocol version: -1
173 2018-08-09T08:00:02.923636+01:00 0 [Note] WSREP: Preparing to initiate SST/IST
174 2018-08-09T08:00:02.923639+01:00 0 [Note] WSREP: Starting replication
175 2018-08-09T08:00:02.923652+01:00 0 [Note] WSREP: Setting initial position to 00000000-0000-0000-0000-000000000000:-1
176 2018-08-09T08:00:02.923845+01:00 0 [Note] WSREP: Using CRC-32C for message checksums.
177 2018-08-09T08:00:02.923912+01:00 0 [Note] WSREP: gcomm thread scheduling priority set to other:0 
178 2018-08-09T08:00:02.923991+01:00 0 [Warning] WSREP: Fail to access the file (/data/mysql/mysql3308/data//gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown
179 2018-08-09T08:00:02.923996+01:00 0 [Note] WSREP: Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown
180 2018-08-09T08:00:02.924673+01:00 0 [Note] WSREP: GMCast version 0
181 2018-08-09T08:00:02.924779+01:00 0 [Note] WSREP: (74f3db69, 'tcp://0.0.0.0:4567') listening at tcp://0.0.0.0:4567
182 2018-08-09T08:00:02.924779+01:00 0 [Note] WSREP: (74f3db69, 'tcp://0.0.0.0:4567') multicast: , ttl: 1
183 2018-08-09T08:00:02.925300+01:00 0 [Note] WSREP: EVS version 0
184 2018-08-09T08:00:02.925433+01:00 0 [Note] WSREP: gcomm: connecting to group 'pxc_wubx', peer '192.168.1.101:,192.168.1.102:,192.168.1.103:'
185 2018-08-09T08:00:02.928124+01:00 0 [Note] WSREP: (74f3db69, 'tcp://0.0.0.0:4567') connection established to 74f3db69 tcp://192.168.1.103:4567
186 2018-08-09T08:00:02.928158+01:00 0 [Warning] WSREP: (74f3db69, 'tcp://0.0.0.0:4567') address 'tcp://192.168.1.103:4567' points to own listening address, blacklisting
187 2018-08-09T08:00:02.928505+01:00 0 [Note] WSREP: (74f3db69, 'tcp://0.0.0.0:4567') connection established to 13eae368 tcp://192.168.1.101:4567
188 2018-08-09T08:00:02.929670+01:00 0 [Note] WSREP: (74f3db69, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: 
189 2018-08-09T08:00:02.929755+01:00 0 [Note] WSREP: (74f3db69, 'tcp://0.0.0.0:4567') connection established to 23fb8f7a tcp://192.168.1.102:4567
190 2018-08-09T08:00:03.429892+01:00 0 [Note] WSREP: declaring 13eae368 at tcp://192.168.1.101:4567 stable
191 2018-08-09T08:00:03.429916+01:00 0 [Note] WSREP: declaring 23fb8f7a at tcp://192.168.1.102:4567 stable
192 2018-08-09T08:00:03.431850+01:00 0 [Note] WSREP: Node 13eae368 state primary
193 2018-08-09T08:00:03.434752+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
194 view (view_id(PRIM,13eae368,11)
195 memb {
196     13eae368,0
197     23fb8f7a,0
198     74f3db69,0
199     }
200 joined {
201     }
202 left {
203     }
204 partitioned {
205     }
206 )
207 2018-08-09T08:00:03.434779+01:00 0 [Note] WSREP: Save the discovered primary-component to disk
208 2018-08-09T08:00:03.927902+01:00 0 [Note] WSREP: gcomm: connected
209 2018-08-09T08:00:03.927980+01:00 0 [Note] WSREP: Shifting CLOSED -> OPEN (TO: 0)
210 2018-08-09T08:00:03.928042+01:00 0 [Note] WSREP: Waiting for SST/IST to complete.
211 2018-08-09T08:00:03.928251+01:00 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 2, memb_num = 3
212 2018-08-09T08:00:03.928261+01:00 0 [Note] WSREP: STATE EXCHANGE: Waiting for state UUID.
213 2018-08-09T08:00:03.928285+01:00 0 [Note] WSREP: STATE EXCHANGE: sent state msg: 753e350b-9b99-11e8-be59-b7e1945f65e8
214 2018-08-09T08:00:03.928290+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 753e350b-9b99-11e8-be59-b7e1945f65e8 from 0 (zlm2)
215 2018-08-09T08:00:03.928294+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 753e350b-9b99-11e8-be59-b7e1945f65e8 from 1 (zlm3)
216 2018-08-09T08:00:03.929335+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: 753e350b-9b99-11e8-be59-b7e1945f65e8 from 2 (zlm4)
217 2018-08-09T08:00:03.929361+01:00 0 [Note] WSREP: Quorum results:
218     version    = 4,
219     component  = PRIMARY,
220     conf_id    = 10,
221     members    = 2/3 (primary/total),
222     act_id     = 22,
223     last_appl. = -1,
224     protocols  = 0/8/3 (gcs/repl/appl),
225     group UUID = bd5525ab-9a15-11e8-aa0f-4b830c783fc7
226 2018-08-09T08:00:03.929366+01:00 0 [Note] WSREP: Flow-control interval: [173, 173]
227 2018-08-09T08:00:03.929369+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
228 2018-08-09T08:00:03.929372+01:00 0 [Note] WSREP: Shifting OPEN -> PRIMARY (TO: 22)
229 2018-08-09T08:00:03.929413+01:00 2 [Note] WSREP: State transfer required: 
230     Group state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22
231     Local state: 00000000-0000-0000-0000-000000000000:-1
232 2018-08-09T08:00:03.929419+01:00 2 [Note] WSREP: REPL Protocols: 8 (3, 2)
233 2018-08-09T08:00:03.929425+01:00 2 [Note] WSREP: New cluster view: global state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22, view# 11: Primary, number of nodes: 3, my index: 2, protocol version 3
234 2018-08-09T08:00:03.929428+01:00 2 [Note] WSREP: Setting wsrep_ready to true
235 2018-08-09T08:00:03.929430+01:00 2 [Warning] WSREP: Gap in state sequence. Need state transfer.
236 2018-08-09T08:00:03.929432+01:00 2 [Note] WSREP: Setting wsrep_ready to false
237 2018-08-09T08:00:03.929496+01:00 0 [Note] WSREP: Initiating SST/IST transfer on JOINER side (wsrep_sst_rsync --role 'joiner' --address '192.168.1.103' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '5367' --binlog '/data/mysql/mysql3308/logs/mysql-bin' )
238     2018-08-09T06:00:04.047845Z WSREP_SST: [INFO] Waiting for data-dir through rsync................ //Begin to transfer data using rsync mode.
239 2018-08-09T08:00:04.319479+01:00 2 [Note] WSREP: Prepared SST/IST request: rsync|192.168.1.103:4444/rsync_sst
240 2018-08-09T08:00:04.319504+01:00 2 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 3) (Increment: 1 -> 3)
241 2018-08-09T08:00:04.319510+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
242 2018-08-09T08:00:04.319527+01:00 2 [Note] WSREP: Assign initial position for certification: 22, protocol version: 3
243 2018-08-09T08:00:04.319546+01:00 0 [Note] WSREP: Service thread queue flushed.
244 2018-08-09T08:00:04.319558+01:00 2 [Note] WSREP: Check if state gap can be serviced using IST
245 2018-08-09T08:00:04.319566+01:00 2 [Note] WSREP: Local UUID: 00000000-0000-0000-0000-000000000000 != Group UUID: bd5525ab-9a15-11e8-aa0f-4b830c783fc7
246 2018-08-09T08:00:04.319598+01:00 2 [Note] WSREP: State gap can't be serviced using IST. Switching to SST
247 2018-08-09T08:00:04.319602+01:00 2 [Note] WSREP: Failed to prepare for incremental state transfer: Local state UUID (00000000-0000-0000-0000-000000000000) does not match group state UUID (bd5525ab-9a15-11e8-aa0f-4b830c783fc7): 1 (Operation not permitted)
248      at galera/src/replicator_str.cpp:prepare_for_IST():538. IST will be unavailable.
249 2018-08-09T08:00:04.321186+01:00 0 [Note] WSREP: Member 2.0 (zlm4) requested state transfer from '*any*'. Selected 0.0 (zlm2)(SYNCED) as donor.
250 2018-08-09T08:00:04.321208+01:00 0 [Note] WSREP: Shifting PRIMARY -> JOINER (TO: 22)
251 2018-08-09T08:00:04.321275+01:00 2 [Note] WSREP: Requesting state transfer: success, donor: 0
252 2018-08-09T08:00:04.321291+01:00 2 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22
253 2018-08-09T08:00:05.939711+01:00 0 [Note] WSREP: (74f3db69, 'tcp://0.0.0.0:4567') turning message relay requesting off
254 2018-08-09T08:00:26.343649+01:00 0 [Note] WSREP: 0.0 (zlm2): State transfer to 2.0 (zlm4) complete.
255 2018-08-09T08:00:26.344408+01:00 0 [Note] WSREP: Member 0.0 (zlm2) synced with group.
256     2018-08-09T06:00:26.399740Z WSREP_SST: [INFO] ..............rsync completed //Finished rsync transfer.
257 2018-08-09T08:00:26.929160+01:00 0 [Note] WSREP: SST complete, seqno: 22
258 2018-08-09T08:00:27.003561+01:00 0 [Warning] InnoDB: Using innodb_locks_unsafe_for_binlog is DEPRECATED. This option may be removed in future releases. Please use READ COMMITTED transaction isolation level instead; Please refer to http://dev.mysql.com/doc/refman/5.7/en/set-transaction.html
259 2018-08-09T08:00:27.003682+01:00 0 [Note] InnoDB: PUNCH HOLE support available
260 2018-08-09T08:00:27.003696+01:00 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins
261 2018-08-09T08:00:27.003701+01:00 0 [Note] InnoDB: Uses event mutexes
262 2018-08-09T08:00:27.003705+01:00 0 [Note] InnoDB: GCC builtin __atomic_thread_fence() is used for memory barrier
263 2018-08-09T08:00:27.003710+01:00 0 [Note] InnoDB: Compressed tables use zlib 1.2.7
264 2018-08-09T08:00:27.003715+01:00 0 [Note] InnoDB: Using Linux native AIO
265 2018-08-09T08:00:27.003734+01:00 0 [Note] InnoDB: Adjusting innodb_buffer_pool_instances from 4 to 1 since innodb_buffer_pool_size is less than 1024 MiB
266 2018-08-09T08:00:27.004212+01:00 0 [Note] InnoDB: Number of pools: 1
267 2018-08-09T08:00:27.004325+01:00 0 [Note] InnoDB: Using CPU crc32 instructions
268 2018-08-09T08:00:27.010477+01:00 0 [Note] InnoDB: Initializing buffer pool, total size = 100M, instances = 1, chunk size = 100M
269 2018-08-09T08:00:27.019767+01:00 0 [Note] InnoDB: Completed initialization of buffer pool
270 2018-08-09T08:00:27.023312+01:00 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().
271 2018-08-09T08:00:27.066815+01:00 0 [Note] InnoDB: Crash recovery did not find the parallel doublewrite buffer at /data/mysql/mysql3308/data/xb_doublewrite
272 2018-08-09T08:00:27.070460+01:00 0 [Note] InnoDB: Highest supported file format is Barracuda.
273 2018-08-09T08:00:27.071136+01:00 0 [Note] InnoDB: Log scan progressed past the checkpoint lsn 2605210
274 2018-08-09T08:00:27.071168+01:00 0 [Note] InnoDB: Doing recovery: scanned up to log sequence number 2605219
275 2018-08-09T08:00:27.071185+01:00 0 [Note] InnoDB: Database was not shutdown normally!
276 2018-08-09T08:00:27.071190+01:00 0 [Note] InnoDB: Starting crash recovery.
277 2018-08-09T08:00:27.306683+01:00 0 [Note] InnoDB: Created parallel doublewrite buffer at /data/mysql/mysql3308/data/xb_doublewrite, size 3932160 bytes
278 2018-08-09T08:00:27.558968+01:00 0 [Note] InnoDB: Last MySQL binlog file position 0 194, file name mysql-bin.000022
279 2018-08-09T08:00:27.575777+01:00 0 [Note] InnoDB: Creating shared tablespace for temporary tables
280 2018-08-09T08:00:27.575872+01:00 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...
281 2018-08-09T08:00:28.063310+01:00 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.
282 2018-08-09T08:00:28.064352+01:00 0 [Note] InnoDB: 96 redo rollback segment(s) found. 96 redo rollback segment(s) are active.
283 2018-08-09T08:00:28.064370+01:00 0 [Note] InnoDB: 32 non-redo rollback segment(s) are active.
284 2018-08-09T08:00:28.065740+01:00 0 [Note] InnoDB: Waiting for purge to start
285 2018-08-09T08:00:28.116905+01:00 0 [Note] InnoDB: Percona XtraDB (http://www.percona.com) 5.7.22-rel22 started; log sequence number 2605219
286 2018-08-09T08:00:28.117131+01:00 0 [Note] Plugin 'FEDERATED' is disabled.
287 2018-08-09T08:00:28.123701+01:00 0 [Note] InnoDB: Loading buffer pool(s) from /data/mysql/mysql3308/data/ib_buffer_pool
288 2018-08-09T08:00:28.123749+01:00 0 [ERROR] InnoDB: Cannot open '/data/mysql/mysql3308/data/ib_buffer_pool' for reading: No such file or directory
289 2018-08-09T08:00:28.126137+01:00 0 [Note] Recovering after a crash using /data/mysql/mysql3308/logs/mysql-bin
290 2018-08-09T08:00:28.126167+01:00 0 [Note] WSREP: Before binlog recovery (wsrep position bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22)
291 2018-08-09T08:00:28.128885+01:00 0 [Note] Starting crash recovery...
292 2018-08-09T08:00:28.128936+01:00 0 [Note] Crash recovery finished.
293 2018-08-09T08:00:28.128955+01:00 0 [Note] WSREP: After binlog recovery (wsrep position bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22)
294 2018-08-09T08:00:28.135097+01:00 0 [Note] Salting uuid generator variables, current_pid: 5367, server_start_time: 1533794402, bytes_sent: 0, 
295 2018-08-09T08:00:28.135163+01:00 0 [Note] Generated uuid: '83fac5b7-9b99-11e8-9f3a-080027de0e0e', server_start_time: 1510676201539890521, bytes_sent: 51827952
296 2018-08-09T08:00:28.135180+01:00 0 [Warning] No existing UUID has been found, so we assume that this is the first time that this server has been started. Generating a new UUID: 83fac5b7-9b99-11e8-9f3a-080027de0e0e.
297 2018-08-09T08:00:28.649143+01:00 0 [Note] Auto generated SSL certificates are placed in data directory.
298 2018-08-09T08:00:28.651737+01:00 0 [Warning] CA certificate ca.pem is self signed.
299 2018-08-09T08:00:28.812659+01:00 0 [Note] Auto generated RSA key files are placed in data directory.
300 2018-08-09T08:00:28.812836+01:00 0 [Note] Server hostname (bind-address): '*'; port: 3308
301 2018-08-09T08:00:28.812866+01:00 0 [Note] IPv6 is available.
302 2018-08-09T08:00:28.816928+01:00 0 [Note]   - '::' resolves to '::';
303 2018-08-09T08:00:28.821691+01:00 0 [Note] Server socket created on IP: '::'.
304 2018-08-09T08:00:28.846925+01:00 0 [Warning] 'user' entry 'root@localhost' ignored in --skip-name-resolve mode.
305 2018-08-09T08:00:28.846986+01:00 0 [Warning] 'user' entry 'mysql.session@localhost' ignored in --skip-name-resolve mode.
306 2018-08-09T08:00:28.847001+01:00 0 [Warning] 'user' entry 'mysql.sys@localhost' ignored in --skip-name-resolve mode.
307 2018-08-09T08:00:28.847028+01:00 0 [Warning] 'user' entry 'sst@localhost' ignored in --skip-name-resolve mode.
308 2018-08-09T08:00:28.847089+01:00 0 [Warning] 'db' entry 'performance_schema mysql.session@localhost' ignored in --skip-name-resolve mode.
309 2018-08-09T08:00:28.847097+01:00 0 [Warning] 'db' entry 'sys mysql.sys@localhost' ignored in --skip-name-resolve mode.
310 2018-08-09T08:00:28.847124+01:00 0 [Warning] 'proxies_priv' entry '@ root@localhost' ignored in --skip-name-resolve mode.
311 2018-08-09T08:00:28.858902+01:00 0 [Warning] 'tables_priv' entry 'user mysql.session@localhost' ignored in --skip-name-resolve mode.
312 2018-08-09T08:00:28.858929+01:00 0 [Warning] 'tables_priv' entry 'sys_config mysql.sys@localhost' ignored in --skip-name-resolve mode.
313 2018-08-09T08:00:28.873861+01:00 0 [Note] Event Scheduler: Loaded 0 events
314 2018-08-09T08:00:28.874109+01:00 0 [Note] WSREP: Signalling provider to continue on SST completion.
315 2018-08-09T08:00:28.874133+01:00 0 [Note] WSREP: Initialized wsrep sidno 2
316 2018-08-09T08:00:28.874156+01:00 0 [Note] WSREP: SST received: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22
317 2018-08-09T08:00:28.874266+01:00 0 [Note] mysqld: ready for connections.
318 Version: '5.7.22-22-29.26-log'  socket: '/tmp/mysql3308.sock'  port: 3308  Percona XtraDB Cluster binary (GPL) 5.7.22-29.26, Revision da86071, wsrep_29.26
319 2018-08-09T08:00:28.875367+01:00 0 [Note] WSREP: 2.0 (zlm4): State transfer from 0.0 (zlm2) complete.
320 2018-08-09T08:00:28.875409+01:00 0 [Note] WSREP: SST leaving flow control
321 2018-08-09T08:00:28.875418+01:00 0 [Note] WSREP: Shifting JOINER -> JOINED (TO: 22)
322 2018-08-09T08:00:28.876587+01:00 0 [Note] WSREP: Member 2.0 (zlm4) synced with group.
323 2018-08-09T08:00:28.876618+01:00 0 [Note] WSREP: Shifting JOINED -> SYNCED (TO: 22)
324 2018-08-09T08:00:28.876639+01:00 2 [Note] WSREP: Synchronized with group, ready for connections
325 2018-08-09T08:00:28.876645+01:00 2 [Note] WSREP: Setting wsrep_ready to true
326 2018-08-09T08:00:28.876649+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
327 
328 //It's okay now.No error is found this time.

Modify the
grastate.dat file on node zlm4(if not exist,touch a new file of
it).

Procedure

 

 1 [root@zlm3 10:27:10 ~]
 2 #ps aux|grep mysqld
 3 root      6694  0.0  0.0 112640   960 pts/1    R+   10:43   0:00 grep --color=auto mysqld
 4 [2]+  Exit 1                  mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf
 5 
 6 [root@zlm3 10:30:01 /data/mysql/mysql3308/data]
 7 #cat error.log |grep ERROR
 8     2018-08-08T08:27:19.819918Z WSREP_SST: [ERROR] Cleanup after exit with status:1
 9 2018-08-08T10:27:19.826527+01:00 0 [ERROR] WSREP: Process completed with error: wsrep_sst_xtrabackup-v2 --role 'joiner' --address '192.168.1.102' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix '' --parent '6021' --binlog '/data/mysql/mysql3308/logs/mysql-bin' : 1 (Operation not permitted)
10 2018-08-08T10:27:19.826557+01:00 0 [ERROR] WSREP: Failed to read uuid:seqno from joiner script.
11 2018-08-08T10:27:19.826565+01:00 0 [ERROR] WSREP: SST script aborted with error 1 (Operation not permitted)
12 2018-08-08T10:27:19.826592+01:00 0 [ERROR] WSREP: SST failed: 1 (Operation not permitted)
13 2018-08-08T10:27:19.826595+01:00 0 [ERROR] Aborting

 

 

**Flush
two logs on master.**

It
still show the ERROR output when node zlm3 startup.

 1 [root@zlm4 11:10:28 /data/mysql/mysql3308]
 2 #vim my3308.cnf 
 3 
 4 [root@zlm4 11:11:03 /data/mysql/mysql3308]
 5 #cat my3308.cnf |grep wsrep
 6 loose-wsrep_cluster_name=pxc_wubx 
 7 loose-wsrep_cluster_address=gcomm://192.168.1.101,192.168.1.102,192.168.1.103
 8 loose-wsrep_node_address=192.168.1.103
 9 loose-wsrep_provider=/usr/local/mysql/lib/libgalera_smm.so
10 #loose-wsrep_sst_method=xtrabackup-v2
11 loose-wsrep_sst_method=rsync
12 loose-wsrep_sst_auth=sst:zlmzlm
13 #loose-wsrep_debug=on
14 #loose-wsrep_provider_options="debug=on"

Start mysqld of PXC on node
zlm4.

Start
mysqld on binlog server.

  1 2018-08-08T10:54:47.539174+01:00 0 [Note] WSREP: (d4006db4, 'tcp://0.0.0.0:4567') connection established to b3de093b tcp://192.168.1.102:4567
  2 2018-08-08T10:54:47.539620+01:00 0 [Note] WSREP: (d4006db4, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: 
  3 2018-08-08T10:54:48.038167+01:00 0 [Note] WSREP: declaring b3de093b at tcp://192.168.1.102:4567 stable
  4 2018-08-08T10:54:48.038656+01:00 0 [Note] WSREP: Node d4006db4 state primary
  5 2018-08-08T10:54:48.039842+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
  6 view (view_id(PRIM,b3de093b,8)
  7 memb {
  8     b3de093b,0
  9     d4006db4,0
 10     }
 11 joined {
 12     }
 13 left {
 14     }
 15 partitioned {
 16     }
 17 )
 18 2018-08-08T10:54:48.039890+01:00 0 [Note] WSREP: Save the discovered primary-component to disk
 19 2018-08-08T10:54:48.040138+01:00 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 1, memb_num = 2
 20 2018-08-08T10:54:48.040154+01:00 0 [Note] WSREP: STATE EXCHANGE: Waiting for state UUID.
 21 2018-08-08T10:54:48.540351+01:00 0 [Note] WSREP: STATE EXCHANGE: sent state msg: b47737d3-9ae8-11e8-81e0-7bfd4d57e067
 22 2018-08-08T10:54:48.540644+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: b47737d3-9ae8-11e8-81e0-7bfd4d57e067 from 0 (zlm3)
 23 2018-08-08T10:54:48.541101+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: b47737d3-9ae8-11e8-81e0-7bfd4d57e067 from 1 (zlm2)
 24 2018-08-08T10:54:48.541140+01:00 0 [Note] WSREP: Quorum results:
 25     version    = 4,
 26     component  = PRIMARY,
 27     conf_id    = 7,
 28     members    = 1/2 (primary/total),
 29     act_id     = 22,
 30     last_appl. = 0,
 31     protocols  = 0/8/3 (gcs/repl/appl),
 32     group UUID = bd5525ab-9a15-11e8-aa0f-4b830c783fc7
 33 2018-08-08T10:54:48.541150+01:00 0 [Note] WSREP: Flow-control interval: [141, 141]
 34 2018-08-08T10:54:48.541155+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
 35 2018-08-08T10:54:48.541224+01:00 2 [Note] WSREP: REPL Protocols: 8 (3, 2)
 36 2018-08-08T10:54:48.541234+01:00 2 [Note] WSREP: New cluster view: global state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22, view# 8: Primary, number of nodes: 2, my index: 1, protocol version 3
 37 2018-08-08T10:54:48.541238+01:00 2 [Note] WSREP: Setting wsrep_ready to true
 38 2018-08-08T10:54:48.541241+01:00 2 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 2) (Increment: 1 -> 2)
 39 2018-08-08T10:54:48.541245+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
 40 2018-08-08T10:54:48.541252+01:00 2 [Note] WSREP: Assign initial position for certification: 22, protocol version: 3
 41 2018-08-08T10:54:48.541264+01:00 0 [Note] WSREP: Service thread queue flushed.
 42 2018-08-08T10:54:49.007421+01:00 0 [Note] WSREP: Member 0.0 (zlm3) requested state transfer from '*any*'. Selected 1.0 (zlm2)(SYNCED) as donor.
 43 2018-08-08T10:54:49.007441+01:00 0 [Note] WSREP: Shifting SYNCED -> DONOR/DESYNCED (TO: 22)
 44 2018-08-08T10:54:49.007480+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
 45 2018-08-08T10:54:49.007626+01:00 0 [Note] WSREP: Initiating SST/IST transfer on DONOR side (wsrep_sst_xtrabackup-v2 --role 'donor' --address '192.168.1.102:4444/xtrabackup_sst//1' --socket '/tmp/mysql3308.sock' --datadir '/data/mysql/mysql3308/data/' --defaults-file '/data/mysql/mysql3308/my3308.cnf' --defaults-group-suffix ''  --binlog '/data/mysql/mysql3308/logs/mysql-bin' --gtid 'bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22')
 46 2018-08-08T10:54:49.012384+01:00 2 [Note] WSREP: DONOR thread signaled with 0
 47 2018-08-08T10:54:50.911071+01:00 0 [Note] WSREP: (d4006db4, 'tcp://0.0.0.0:4567') turning message relay requesting off
 48     2018-08-08T08:54:59.535108Z WSREP_SST: [INFO] Streaming the backup to joiner at 192.168.1.102 4444
 49 2018-08-08T10:54:59.626864+01:00 8 [Note] Access denied for user 'sst'@'192.168.1.101' (using password: YES)
 50 2018-08-08T10:55:05.537365+01:00 0 [Note] WSREP: 1.0 (zlm2): State transfer to 0.0 (zlm3) complete.
 51 2018-08-08T10:55:05.537399+01:00 0 [Note] WSREP: Shifting DONOR/DESYNCED -> JOINED (TO: 22)
 52 2018-08-08T10:55:05.542864+01:00 0 [Note] WSREP: Member 1.0 (zlm2) synced with group.
 53 2018-08-08T10:55:05.542949+01:00 0 [Note] WSREP: Shifting JOINED -> SYNCED (TO: 22)
 54 2018-08-08T10:55:05.543015+01:00 2 [Note] WSREP: Synchronized with group, ready for connections
 55 2018-08-08T10:55:05.543022+01:00 2 [Note] WSREP: Setting wsrep_ready to true
 56 2018-08-08T10:55:05.543027+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
 57 2018-08-08T10:55:22.477724+01:00 0 [Note] WSREP: forgetting b3de093b (tcp://192.168.1.102:4567)
 58 2018-08-08T10:55:22.479542+01:00 0 [Note] WSREP: Node d4006db4 state primary
 59 2018-08-08T10:55:22.479612+01:00 0 [Note] WSREP: Current view of cluster as seen by this node
 60 view (view_id(PRIM,d4006db4,9)
 61 memb {
 62     d4006db4,0
 63     }
 64 joined {
 65     }
 66 left {
 67     }
 68 partitioned {
 69     b3de093b,0
 70     }
 71 )
 72 2018-08-08T10:55:22.479621+01:00 0 [Note] WSREP: Save the discovered primary-component to disk
 73 2018-08-08T10:55:22.479763+01:00 0 [Note] WSREP: forgetting b3de093b (tcp://192.168.1.102:4567)
 74 2018-08-08T10:55:22.479875+01:00 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 0, memb_num = 1
 75 2018-08-08T10:55:22.479976+01:00 0 [Note] WSREP: STATE_EXCHANGE: sent state UUID: c8aef280-9ae8-11e8-80c9-d74e514aa266
 76 2018-08-08T10:55:22.479976+01:00 0 [Note] WSREP: STATE EXCHANGE: sent state msg: c8aef280-9ae8-11e8-80c9-d74e514aa266
 77 2018-08-08T10:55:22.479976+01:00 0 [Note] WSREP: STATE EXCHANGE: got state msg: c8aef280-9ae8-11e8-80c9-d74e514aa266 from 0 (zlm2)
 78 2018-08-08T10:55:22.479978+01:00 0 [Note] WSREP: Quorum results:
 79     version    = 4,
 80     component  = PRIMARY,
 81     conf_id    = 8,
 82     members    = 1/1 (primary/total),
 83     act_id     = 22,
 84     last_appl. = 0,
 85     protocols  = 0/8/3 (gcs/repl/appl),
 86     group UUID = bd5525ab-9a15-11e8-aa0f-4b830c783fc7
 87 2018-08-08T10:55:22.479983+01:00 0 [Note] WSREP: Flow-control interval: [100, 100]
 88 2018-08-08T10:55:22.480000+01:00 0 [Note] WSREP: Trying to continue unpaused monitor
 89 2018-08-08T10:55:22.480032+01:00 2 [Note] WSREP: REPL Protocols: 8 (3, 2)
 90 2018-08-08T10:55:22.480037+01:00 2 [Note] WSREP: New cluster view: global state: bd5525ab-9a15-11e8-aa0f-4b830c783fc7:22, view# 9: Primary, number of nodes: 1, my index: 0, protocol version 3
 91 2018-08-08T10:55:22.480040+01:00 2 [Note] WSREP: Setting wsrep_ready to true
 92 2018-08-08T10:55:22.480043+01:00 2 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 2 -> 1) (Increment: 2 -> 1)
 93 2018-08-08T10:55:22.480046+01:00 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
 94 2018-08-08T10:55:22.480051+01:00 2 [Note] WSREP: Assign initial position for certification: 22, protocol version: 3
 95 2018-08-08T10:55:22.480060+01:00 0 [Note] WSREP: Service thread queue flushed.
 96 2018-08-08T10:55:26.085271+01:00 0 [Note] WSREP: (d4006db4, 'tcp://0.0.0.0:4567') connection established to b3de093b tcp://192.168.1.102:4567
 97 2018-08-08T10:55:26.085306+01:00 0 [Warning] WSREP: discarding established (time wait) b3de093b (tcp://192.168.1.102:4567) 
 98 2018-08-08T10:55:27.948973+01:00 0 [Note] WSREP:  cleaning up b3de093b (tcp://192.168.1.102:4567)
 99 
100 //There's no ERROR messages at all this time.

 

 

 1 [root@zlm2 16:39:39 /data/backup]
 2 #ps aux|grep mysqld
 3 mysql     3771  0.0 18.7 1071816 190784 pts/0  Sl   15:52   0:01 mysqld --defaults-file=/data/mysql/mysql3306/my3306.cnf
 4 root      4451  0.0  0.0 112640   956 pts/1    R+   16:43   0:00 grep --color=auto mysqld
 5 
 6 [root@zlm2 16:43:10 /data/backup]
 7 #mysqladmin shutdown
 8 
 9 [root@zlm2 16:43:15 /data/backup]
10 #ps aux|grep mysqld
11 root      4463  0.0  0.0 112640   956 pts/1    R+   16:43   0:00 grep --color=auto mysqld
12 
13 [root@zlm2 16:43:18 /data/backup]
14 #cd /data/mysql/mysql3306/data/
15 
16 [root@zlm2 16:43:36 /data/mysql/mysql3306/data]
17 #rm -rf *
18 
19 [root@zlm2 16:43:39 /data/mysql/mysql3306/data]
20 #cd ../logs
21 
22 [root@zlm2 16:43:50 /data/mysql/mysql3306/logs]
23 #rm -rf *
24 
25 [root@zlm2 16:43:53 /data/mysql/mysql3306/logs]
26 #innobackupex --defaults-file=/data/mysql/mysql3306/my3306.cnf -uroot -pPassw0rd --copy-back /data/backup/2018-07-29_16-32-33/
27 ... //Omitted.

 

**Copy the backup set to node
zlm4.**

Framework

澳门威呢斯人, 

Evidence on
node zlm2.

Preface

Hostname IP Port OS Version MySQL Version Xtrabackup version
zlm2 192.168.1.101 3308 CentOS 7.0 PXC 5.7.22 2.4.12 Linux (x86_64)
zlm3 192.168.1.102 3308 CentOS 7.0 PXC 5.7.22 2.4.12 Linux (x86_64)
zlm4 192.168.1.103 3308 CentOS 7.0 PXC 5.7.22 2.4.12 Linux (x86_64)

 

 1 [root@zlm2 09:21:38 /data/mysql/mysql3308/data]
 2 #mysqladmin -P3308 shutdown
 3 
 4 [root@zlm2 09:22:10 /data/mysql/mysql3308/data]
 5 #ps aux|grep mysqld
 6 mysql     6295  0.1 22.4 1565212 227920 pts/0  Sl   08:47   0:02 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster
 7 root      6835  0.0  0.0 112640   956 pts/3    R+   09:22   0:00 grep --color=auto mysqld
 8 
 9 [root@zlm2 09:22:15 /data/mysql/mysql3308/data]
10 #ps aux|grep mysqld
11 root      6841  0.0  0.0 112640   960 pts/3    R+   09:22   0:00 grep --color=auto mysqld
12 
13 [root@zlm2 09:30:04 /data/mysql/mysql3308/data]
14 #mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster &
15 [1] 7055
16 
17 [root@zlm2 09:30:20 /data/mysql/mysql3308/data]
18 #ps aux|grep mysqld
19 mysql     7055  0.4 22.1 1564952 224900 pts/3  Sl   09:30   0:02 mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --wsrep-new-cluster
20 root      7583  0.0  0.0 112640   956 pts/3    R+   09:39   0:00 grep --color=auto mysqld

Procedure

 1 [root@zlm2 08:51:49 ~]
 2 #mkdir -p /data/mysql/mysql3308/{data,logs,tmp}
 3 
 4 [root@zlm2 08:52:44 ~]
 5 #chown -R mysql.mysql /data
 6 
 7 [root@zlm2 08:53:27 ~]
 8 #cp /vagrant/my3308.cnf /data/mysql/mysql3308
 9 
10 [root@zlm2 08:54:09 ~]
11 #cd /data/mysql/mysql3308
12 
13 [root@zlm2 08:54:17 /data/mysql/mysql3308]
14 #ls -l
15 total 8
16 drwxr-xr-x 2 mysql mysql    6 Aug  7 08:52 data
17 drwxr-xr-x 2 mysql mysql    6 Aug  7 08:52 logs
18 -rwxr-xr-x 1 root  root  4522 Aug  7 08:54 my3308.cnf
19 drwxr-xr-x 2 mysql mysql    6 Aug  7 08:52 tmp
20 
21 [root@zlm2 08:54:30 /data/mysql/mysql3308]
22 #tail -20 my3308.cnf 
23 ...
24 #pxc
25 innodb_locks_unsafe_for_binlog=1
26 innodb_autoinc_lock_mode=2
27 loose-wsrep_cluster_name=pxc_wubx 
28 loose-wsrep_cluster_address=gcomm://192.168.1.101,192.168.1.102,192.168.103
29 loose-wsrep_node_address=192.168.1.101
30 loose-wsrep_provider=/usr/local/mysql/lib/libgalera_smm.so
31 loose-wsrep_sst_method=xtrabackup-v2
32 #loose-wsrep_sst_method=rsync
33 loose-wsrep_sst_auth=sst:zlmzlm
34 #loose-wsrep_debug=on
35 #loose-wsrep_provider_options="debug=on"
36 [root@zlm2 08:54:38 /data/mysql/mysql3308]
37 #mysqld --defaults-file=/data/mysql/mysql3308/my3308.cnf --initialize
38 
39 [root@zlm2 09:00:56 /data/mysql/mysql3308]
40 #tail ./data/error.log 
41 2018-08-07T09:00:47.560965+01:00 0 [Warning] CA certificate ca.pem is self signed.
42 2018-08-07T09:00:47.603271+01:00 1 [Note] A temporary password is generated for root@localhost: bY2jmV=k)ltM //This is the password we need when first login.
43 2018-08-07T09:00:48.459948+01:00 1 [Warning] 'user' entry 'root@localhost' ignored in --skip-name-resolve mode.
44 2018-08-07T09:00:48.459948+01:00 1 [Warning] 'user' entry 'mysql.session@localhost' ignored in --skip-name-resolve mode.
45 2018-08-07T09:00:48.459948+01:00 1 [Warning] 'user' entry 'mysql.sys@localhost' ignored in --skip-name-resolve mode.
46 2018-08-07T09:00:48.459948+01:00 1 [Warning] 'db' entry 'performance_schema mysql.session@localhost' ignored in --skip-name-resolve mode.
47 2018-08-07T09:00:48.459948+01:00 1 [Warning] 'db' entry 'sys mysql.sys@localhost' ignored in --skip-name-resolve mode.
48 2018-08-07T09:00:48.459948+01:00 1 [Warning] 'proxies_priv' entry '@ root@localhost' ignored in --skip-name-resolve mode.
49 2018-08-07T09:00:48.459948+01:00 1 [Warning] 'tables_priv' entry 'user mysql.session@localhost' ignored in --skip-name-resolve mode.
50 2018-08-07T09:00:48.459948+01:00 1 [Warning] 'tables_priv' entry 'sys_config mysql.sys@localhost' ignored in --skip-name-resolve mode.
51 
52 [root@zlm2 09:02:45 /data/mysql/mysql3308]
53 #cd data
54 
55 [root@zlm2 09:02:57 /data/mysql/mysql3308/data]
56 #ls -l
57 total 409676
58 -rw-r----- 1 mysql mysql        56 Aug  7 09:00 auto.cnf
59 -rw------- 1 mysql mysql      1680 Aug  7 09:00 ca-key.pem
60 -rw-r--r-- 1 mysql mysql      1120 Aug  7 09:00 ca.pem
61 -rw-r--r-- 1 mysql mysql      1120 Aug  7 09:00 client-cert.pem
62 -rw------- 1 mysql mysql      1680 Aug  7 09:00 client-key.pem
63 -rw-r----- 1 mysql mysql      2354 Aug  7 09:00 error.log
64 -rw-r----- 1 mysql mysql       423 Aug  7 09:00 ib_buffer_pool
65 -rw-r----- 1 mysql mysql 104857600 Aug  7 09:00 ibdata1
66 -rw-r----- 1 mysql mysql 104857600 Aug  7 09:00 ib_logfile0
67 -rw-r----- 1 mysql mysql 104857600 Aug  7 09:00 ib_logfile1
68 -rw-r----- 1 mysql mysql 104857600 Aug  7 09:00 ib_logfile2
69 drwxr-x--- 2 mysql mysql      4096 Aug  7 09:00 mysql
70 drwxr-x--- 2 mysql mysql      8192 Aug  7 09:00 performance_schema
71 -rw------- 1 mysql mysql      1676 Aug  7 09:00 private_key.pem
72 -rw-r--r-- 1 mysql mysql       452 Aug  7 09:00 public_key.pem
73 -rw-r--r-- 1 mysql mysql      1120 Aug  7 09:00 server-cert.pem
74 -rw------- 1 mysql mysql      1676 Aug  7 09:00 server-key.pem
75 -rw-r----- 1 mysql mysql       227 Aug  7 09:00 slow.log
76 drwxr-x--- 2 mysql mysql      8192 Aug  7 09:00 sys
77 
78 //Omitted the operations on the other nodes.Notice that there's no need to initialize the other node.

*Step 3: Rescue data.**奥门威斯,

 Evidence
on node zlm4
(using rsync mothed of sst)澳门威呢斯人赌场,.

 

 

 

Evidence on
node zlm3
(using xtrabackup-v2 mothed of sst).**

1 [root@zlm3 09:31:07 ~]
2 #scp -r /data/backup/2018-08-09_09-26-05/ zlm4:/data/backup/
3 root@zlm4's password: 
4 xtrabackup_logfile                                                                                                 100% 2560     2.5KB/s   00:00    
5 ibdata1                                                                                                            100%  100MB  33.3MB/s   00:03    
6 plugin.ibd                                                                                                         100%   96KB  96.0KB/s   00:00    
7 servers.ibd                                                                                                        100%   96KB  96.0KB/s   00:00
8 ... //Omitted.

Install Percona-XtraDB-Cluster
software on three nodes.

 1 [root@zlm3 16:50:00 /data/mysql/mysql3306/logs]
 2 #ps aux|grep mysqld
 3 root      4405  0.0  0.0 112640   960 pts/0    R+   16:50   0:00 grep --color=auto mysqld
 4 
 5 [root@zlm3 16:50:08 /data/mysql/mysql3306/logs]
 6 #sh /root/mysqld.sh
 7 
 8 [root@zlm3 16:50:13 /data/mysql/mysql3306/logs]
 9 #ps aux|grep mysqld
10 mysql     4411 16.5 15.6 498232 159408 pts/0   Rl   16:50   0:00 mysqld --defaults-file=/data/mysql/mysql3306/my3306.cnf
11 root      4429  0.0  0.0 112640   960 pts/0    R+   16:50   0:00 grep --color=auto mysqld
12 
13 [root@zlm3 16:51:03 /data/mysql/mysql3306/logs]
14 #ls -l
15 total 11216
16 -rw-r----- 1 root  root      4128 Jul 29 08:42 mysql-bin.000095
17 -rw-r----- 1 root  root       241 Jul 29 08:42 mysql-bin.000096
18 -rw-r----- 1 root  root  11461562 Jul 29 09:07 mysql-bin.000097
19 -rw-r----- 1 mysql mysql      154 Jul 29 16:50 mysql-bin.000098
20 -rw-r----- 1 mysql mysql       44 Jul 29 16:50 mysql-bin.index
21 
22 [root@zlm3 16:51:04 /data/mysql/mysql3306/logs]
23 #cat mysql-bin.index
24 /data/mysql/mysql3306/logs/mysql-bin.000098

Preface

 

    Today,I’m gonna implement a
PXC,Let’s see the procedure.

Insert one
record of “sbtest6” and drop it.

Author

发表评论

电子邮件地址不会被公开。 必填项已用*标注