Hello,
I'm new to this forum. I hope I've posted in the right location.
We have 3 mysql enterprise edition servers in group replication which are working on Oracle Linux 7.3 servers.
Almost daily (around midnight) one of the servers [random one] drops from the cluster with the following error:
[ERROR] Plugin group_replication reported: 'Member was expelled from the group due to network failures, changing member status to ERROR.'
After about 25 minutes we can see another error:
[ERROR] Plugin group_replication reported: 'Error, maximum number of retries exceeded when waiting for the internal server session state to be operating'
We have to stop the group replication and restart or stop the mysql service and restart
All 3 servers are virtual machines hosted on the same physical box so we shouldn't have any network issues.
This is our my.cnf config file:
[mysqld]
#
# Unique server-id within the Group/Cluster
#
server-id = 1001
datadir = /var/lib/mysql
socket = /var/lib/mysql/mysql.sock
pid-file = /var/run/mysqld/mysqld.pid
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
log-error = /var/log/mysqld.log
general_log_file = /var/log/mysql_general.log
slow_query_log_file = /var/log/mysql_slow_query.log
#
# Character Set
#
#character-set-server = utf8
#collation-server = utf8_unicode_ci
character-set-server=utf8mb4
collation-server=utf8mb4_general_ci
#
#max _allowed_packet
#
max_allowed_packet=64M
#
# Connections
#
max-connections = 500
#
# Plugin related
#
plugin-dir = /usr/lib64/mysql/plugin
#plugin-load = group_replication.so,thread_pool.so
#
# Replication related
#
binlog-rows-query-log-events = ON
log-bin-trust-function-creators = TRUE
log-bin
binlog-format = ROW
binlog-row-image = MINIMAL
log-slave-updates = ON
binlog-checksum = NONE
expire-logs-days = 30
max-binlog-size = 1G
relay-log-recovery = ON
#
# Engines and Transactions
#
disabled_storage_engines = 'MyISAM,BLACKHOLE,FEDERATED,ARCHIVE'
transaction-isolation = 'READ-COMMITTED'
transaction-write-set-extraction = XXHASH64
gtid-mode = ON
enforce-gtid-consistency = ON
sql_mode = NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
#
# InnoDB
#
innodb_flush_log_at_trx_commit = 1
innodb_buffer_pool_size = 4096M
innodb_flush_method = O_DIRECT
#
# MyISAM
#
key_buffer_size =128M
#
# Group Replication Settings
#
master-info-repository = TABLE
relay-log-info-repository = TABLE
group_replication_start_on_boot = ON
group_replication_bootstrap_group = OFF
group_replication_group_name = 'hidden'
group_replication_local_address = 'server1:33061'
group_replication_ip_whitelist = 'server1,server2,server3'
group_replication_group_seeds = 'server1:33061,server2:33061,server3:33061'
#group_replication_single_primary_mode = OFF
#group_replication_enforce_update_everywhere_checks = ON
#group_replication = FORCE_PLUS_PERMANENT
#For Zabbix monitoring
[client]
#for monitoring
user=zabbix
password=hidden
port=3306
# End Of File
Any help would be appreciated.
Thank you,
Andrei
I'm new to this forum. I hope I've posted in the right location.
We have 3 mysql enterprise edition servers in group replication which are working on Oracle Linux 7.3 servers.
Almost daily (around midnight) one of the servers [random one] drops from the cluster with the following error:
[ERROR] Plugin group_replication reported: 'Member was expelled from the group due to network failures, changing member status to ERROR.'
After about 25 minutes we can see another error:
[ERROR] Plugin group_replication reported: 'Error, maximum number of retries exceeded when waiting for the internal server session state to be operating'
We have to stop the group replication and restart or stop the mysql service and restart
All 3 servers are virtual machines hosted on the same physical box so we shouldn't have any network issues.
This is our my.cnf config file:
[mysqld]
#
# Unique server-id within the Group/Cluster
#
server-id = 1001
datadir = /var/lib/mysql
socket = /var/lib/mysql/mysql.sock
pid-file = /var/run/mysqld/mysqld.pid
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
log-error = /var/log/mysqld.log
general_log_file = /var/log/mysql_general.log
slow_query_log_file = /var/log/mysql_slow_query.log
#
# Character Set
#
#character-set-server = utf8
#collation-server = utf8_unicode_ci
character-set-server=utf8mb4
collation-server=utf8mb4_general_ci
#
#max _allowed_packet
#
max_allowed_packet=64M
#
# Connections
#
max-connections = 500
#
# Plugin related
#
plugin-dir = /usr/lib64/mysql/plugin
#plugin-load = group_replication.so,thread_pool.so
#
# Replication related
#
binlog-rows-query-log-events = ON
log-bin-trust-function-creators = TRUE
log-bin
binlog-format = ROW
binlog-row-image = MINIMAL
log-slave-updates = ON
binlog-checksum = NONE
expire-logs-days = 30
max-binlog-size = 1G
relay-log-recovery = ON
#
# Engines and Transactions
#
disabled_storage_engines = 'MyISAM,BLACKHOLE,FEDERATED,ARCHIVE'
transaction-isolation = 'READ-COMMITTED'
transaction-write-set-extraction = XXHASH64
gtid-mode = ON
enforce-gtid-consistency = ON
sql_mode = NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
#
# InnoDB
#
innodb_flush_log_at_trx_commit = 1
innodb_buffer_pool_size = 4096M
innodb_flush_method = O_DIRECT
#
# MyISAM
#
key_buffer_size =128M
#
# Group Replication Settings
#
master-info-repository = TABLE
relay-log-info-repository = TABLE
group_replication_start_on_boot = ON
group_replication_bootstrap_group = OFF
group_replication_group_name = 'hidden'
group_replication_local_address = 'server1:33061'
group_replication_ip_whitelist = 'server1,server2,server3'
group_replication_group_seeds = 'server1:33061,server2:33061,server3:33061'
#group_replication_single_primary_mode = OFF
#group_replication_enforce_update_everywhere_checks = ON
#group_replication = FORCE_PLUS_PERMANENT
#For Zabbix monitoring
[client]
#for monitoring
user=zabbix
password=hidden
port=3306
# End Of File
Any help would be appreciated.
Thank you,
Andrei