From 3647cb055078f83df7f7616ceaf400857988030c Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 13 Aug 2026 15:01:56 +0300 Subject: [PATCH] MDEV-38869 sequence conflicts with streaming replication Sequence access conflicts with streaming replication could cause server hanging, as shown in MDEV-38869 This commit avoids such deadlocks, by skipping fragment replication for sequence access. The commit has also new mtr test for testing two sequence/SR conflict scenarios: galera.galera_sequences_bf_kill_sr --- .../r/galera_sequences_bf_kill_sr.result | 140 +++++++++++++ .../galera/t/galera_sequences_bf_kill_sr.cnf | 12 ++ .../galera_sequences_bf_kill_sr.combinations | 5 + .../galera/t/galera_sequences_bf_kill_sr.test | 192 ++++++++++++++++++ sql/handler.cc | 13 +- 5 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/galera/r/galera_sequences_bf_kill_sr.result create mode 100644 mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.cnf create mode 100644 mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.combinations create mode 100644 mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.test diff --git a/mysql-test/suite/galera/r/galera_sequences_bf_kill_sr.result b/mysql-test/suite/galera/r/galera_sequences_bf_kill_sr.result new file mode 100644 index 0000000000000..e7527b81fffb0 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sequences_bf_kill_sr.result @@ -0,0 +1,140 @@ +connection node_2; +connection node_1; +connection node_1; +CREATE SEQUENCE s INCREMENT=0 CACHE=5 ENGINE=InnoDB; +CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0), (3, 0); +connection node_1; +START TRANSACTION; +INSERT INTO t1 VALUES (4, 0); +connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET SESSION wsrep_sync_wait=0; +# The transaction above must have replicated a fragment, otherwise +# this test degenerates into galera_sequences_bf_kill +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.wsrep_streaming_log; +EXPECT_1 +1 +# Block the applier on node #1 and issue a conflicting insert on node #2 +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_2; +INSERT INTO t1 VALUES (2, next value for s); +connection node_1a; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +# Block the commit monitor and let the local transaction reserve a +# new range of sequence values, which conflicts with the applier +SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync'; +connection node_1; +INSERT INTO t1 VALUES (5, next value for s); +connection node_1a; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +# Release the applier +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +# Release the local transaction +SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync'; +connection node_1; +COMMIT; +# Both nodes must have the same table contents. The sequence values +# are node specific, each node adjusts them to its own +# auto_increment_offset +connection node_1; +SELECT * FROM t1; +f1 f2 +1 0 +2 2 +3 0 +4 0 +5 1 +SELECT LASTVAL(s); +LASTVAL(s) +1 +SELECT next_not_cached_value FROM s; +next_not_cached_value +11 +connection node_2; +SELECT * FROM t1; +f1 f2 +1 0 +2 2 +3 0 +4 0 +5 1 +SELECT LASTVAL(s); +LASTVAL(s) +2 +SELECT next_not_cached_value FROM s; +next_not_cached_value +12 +# All fragments must have been removed on commit +connection node_1a; +SELECT COUNT(*) AS EXPECT_0 FROM mysql.wsrep_streaming_log; +EXPECT_0 +0 +connection node_1; +DROP SEQUENCE s; +DROP TABLE t1; +connection node_1; +CREATE SEQUENCE s INCREMENT=0 CACHE=5 ENGINE=InnoDB; +CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0), (3, 0); +connection node_1; +START TRANSACTION; +INSERT INTO t1 VALUES (4, 0); +SELECT * FROM t1 WHERE f1 > 0 FOR UPDATE; +f1 f2 +1 0 +3 0 +4 0 +# Block the applier on node #1 and issue a conflicting insert on node #2 +connection node_1a; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_2; +INSERT INTO t1 VALUES (2, next value for s); +connection node_1a; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +# Block the commit monitor and let the local transaction reserve a +# new range of sequence values, which conflicts with the applier +SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync'; +connection node_1; +INSERT INTO t1 VALUES (5, next value for s); +connection node_1a; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +# Release the applier +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +# Release the local transaction +SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync'; +# The streaming transaction is BF aborted and rolled back +connection node_1; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +ROLLBACK; +# The rollback must have removed the already replicated fragments +# from both nodes, so neither node has row 4 +connection node_1; +SELECT * FROM t1; +f1 f2 +1 0 +2 2 +3 0 +connection node_2; +SELECT * FROM t1; +f1 f2 +1 0 +2 2 +3 0 +connection node_1a; +SELECT COUNT(*) AS EXPECT_0 FROM mysql.wsrep_streaming_log; +EXPECT_0 +0 +wsrep_local_replays +1 +connection node_1; +DROP SEQUENCE s; +DROP TABLE t1; +disconnect node_1a; diff --git a/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.cnf b/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.cnf new file mode 100644 index 0000000000000..fd5f4edab8d66 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.cnf @@ -0,0 +1,12 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +wsrep-auto-increment-control=OFF +auto-increment-increment=2 +auto-increment-offset=1 +wsrep_trx_fragment_size=1 + +[mysqld.2] +wsrep-auto-increment-control=OFF +auto-increment-increment=2 +auto-increment-offset=2 diff --git a/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.combinations b/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.combinations new file mode 100644 index 0000000000000..cef98e75213f7 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.combinations @@ -0,0 +1,5 @@ +[binlogon] +log-bin +log-slave-updates + +[binlogoff] diff --git a/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.test b/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.test new file mode 100644 index 0000000000000..8b9ffa0322b18 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sequences_bf_kill_sr.test @@ -0,0 +1,192 @@ +# +# Test a conflict on an InnoDB sequence between a streaming replication +# transaction and an applier. +# +# The local transaction reserves a new range of sequence values, which +# writes the sequence table while holding SEQUENCE::mutex. At the same +# time an applier applies a sequence update originating from the other +# node, and needs the same mutex in SEQUENCE::set_value(). +# +# If the sequence table write replicates a streaming fragment while the +# mutex is held, that fragment waits for commit order behind the applier's +# write set, while the applier waits for the mutex. The node deadlocks and +# the BF abort of the local transaction can never be delivered. +# +# This is specific to streaming replication. Without it the write set is +# replicated at COMMIT, after SEQUENCE::mutex has been released. +# + +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_sequence.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc + +--disable_ps2_protocol + +# +# We create an InnoDB sequence with a small cache that is then used as +# value for a column in the table. The cache is exhausted immediately, so +# that the first NEXTVAL on each node has to write the sequence table. +# +--connection node_1 +--let $wsrep_local_replays_old = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'` +CREATE SEQUENCE s INCREMENT=0 CACHE=5 ENGINE=InnoDB; +CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0), (3, 0); + +--connection node_1 +START TRANSACTION; +INSERT INTO t1 VALUES (4, 0); # No conflict in cert + +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +SET SESSION wsrep_sync_wait=0; + +--echo # The transaction above must have replicated a fragment, otherwise +--echo # this test degenerates into galera_sequences_bf_kill +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.wsrep_streaming_log; + +--echo # Block the applier on node #1 and issue a conflicting insert on node #2 +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +# +# Send conflicting INSERT. This updates the sequence table on node #2, +# and the applier on node #1 will have to apply that update. +# +--connection node_2 +INSERT INTO t1 VALUES (2, next value for s); + +--connection node_1a +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc + +--echo # Block the commit monitor and let the local transaction reserve a +--echo # new range of sequence values, which conflicts with the applier +--let $galera_sync_point = commit_monitor_master_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_1 +--send INSERT INTO t1 VALUES (5, next value for s) + +--connection node_1a +--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_master_enter_sync +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc + +--echo # Release the applier +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc + +--echo # Release the local transaction +--let $galera_sync_point = commit_monitor_master_enter_sync +--source include/galera_signal_sync_point.inc + +--connection node_1 +--reap +COMMIT; + +--echo # Both nodes must have the same table contents. The sequence values +--echo # are node specific, each node adjusts them to its own +--echo # auto_increment_offset +--connection node_1 +SELECT * FROM t1; +SELECT LASTVAL(s); +SELECT next_not_cached_value FROM s; + +--connection node_2 +SELECT * FROM t1; +SELECT LASTVAL(s); +SELECT next_not_cached_value FROM s; + +--echo # All fragments must have been removed on commit +--connection node_1a +--let $wait_condition = SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log +--source include/wait_condition.inc +SELECT COUNT(*) AS EXPECT_0 FROM mysql.wsrep_streaming_log; + +--connection node_1 +DROP SEQUENCE s; +DROP TABLE t1; + +# +# Case 2: the same sequence conflict, but this time the applier also has +# to BF abort the local transaction over a gap lock. A streaming +# transaction cannot be replayed, so it is rolled back and the client +# gets ER_LOCK_DEADLOCK. +# +--connection node_1 +CREATE SEQUENCE s INCREMENT=0 CACHE=5 ENGINE=InnoDB; +CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0), (3, 0); + +--connection node_1 +START TRANSACTION; +INSERT INTO t1 VALUES (4, 0); +SELECT * FROM t1 WHERE f1 > 0 FOR UPDATE; # Should cause GAP lock between 1 and 3 + +--echo # Block the applier on node #1 and issue a conflicting insert on node #2 +--connection node_1a +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_2 +INSERT INTO t1 VALUES (2, next value for s); # BF aborts on the GAP lock + +--connection node_1a +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc + +--echo # Block the commit monitor and let the local transaction reserve a +--echo # new range of sequence values, which conflicts with the applier +--let $galera_sync_point = commit_monitor_master_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_1 +--send INSERT INTO t1 VALUES (5, next value for s) + +--connection node_1a +--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_master_enter_sync +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc + +--echo # Release the applier +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc + +--echo # Release the local transaction +--let $galera_sync_point = commit_monitor_master_enter_sync +--source include/galera_signal_sync_point.inc + +--echo # The streaming transaction is BF aborted and rolled back +--connection node_1 +--error ER_LOCK_DEADLOCK +--reap +ROLLBACK; + +--echo # The rollback must have removed the already replicated fragments +--echo # from both nodes, so neither node has row 4 +--connection node_1 +SELECT * FROM t1; + +--connection node_2 +SELECT * FROM t1; + +--connection node_1a +--let $wait_condition = SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log +--source include/wait_condition.inc +SELECT COUNT(*) AS EXPECT_0 FROM mysql.wsrep_streaming_log; + +# Streaming transactions are rolled back rather than replayed, so no +# replay happened in either case +--let $wsrep_local_replays_new = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_new - $wsrep_local_replays_old = 0 AS wsrep_local_replays; +--enable_query_log + +--connection node_1 +DROP SEQUENCE s; +DROP TABLE t1; + +--disconnect node_1a diff --git a/sql/handler.cc b/sql/handler.cc index 27b3e66823857..d0e28989f7ee6 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -7890,8 +7890,19 @@ int handler::ha_write_row(const uchar *buf) error= binlog_log_row(table, 0, buf, log_func); } + /* + Sequence tables are written with SEQUENCE::mutex held (see + SEQUENCE::next_value() and ha_sequence::write_row()). For a streaming + transaction wsrep_after_row() would replicate a fragment and block + waiting for certification and commit order, while an applier may be + waiting for the same mutex in SEQUENCE::set_value(). That deadlocks + the node, so skip the streaming step here. The row has already been + appended to the write set and will be replicated with the following + fragment, or at commit. + */ if (WSREP_NNULL(ha_thd()) && table_share->tmp_table == NO_TMP_TABLE && - ht->flags & HTON_WSREP_REPLICATION && !error) + ht->flags & HTON_WSREP_REPLICATION && !error && + !table_share->sequence) error= wsrep_after_row(ha_thd()); err: