From 3fe6463078c887ac1cac76c32607f611b3cc34c2 Mon Sep 17 00:00:00 2001 From: krylosov-aa Date: Sun, 11 Oct 2026 01:11:26 +0300 Subject: [PATCH] Fix flaky VACUUM progress tests VACUUM could leave the compact phase before FTS detected the stopped mirror, so the expected vacuum worker change did not happen. Keep VACUUM suspended until the mirror is marked down, and hold a snapshot so compacted segments stay visible until post-cleanup is checked. --- .../expected/vacuum_progress_column.out | 57 +++++---- .../expected/vacuum_progress_row.out | 112 +++++++++++------- .../isolation2/sql/vacuum_progress_column.sql | 25 ++-- .../isolation2/sql/vacuum_progress_row.sql | 50 ++++---- 4 files changed, 137 insertions(+), 107 deletions(-) diff --git a/src/test/isolation2/expected/vacuum_progress_column.out b/src/test/isolation2/expected/vacuum_progress_column.out index 40bce79477a..d308dc9f808 100644 --- a/src/test/isolation2/expected/vacuum_progress_column.out +++ b/src/test/isolation2/expected/vacuum_progress_column.out @@ -283,6 +283,15 @@ ABORT DELETE FROM vacuum_progress_ao_column where j % 2 = 0; DELETE 50000 +-- Keep the compacted segments visible until post-cleanup progress is checked. +3: BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN +3: SELECT count(*) FROM vacuum_progress_ao_column; + count +------- + 50000 +(1 row) + -- Suspend execution at the end of compact phase. 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault @@ -308,9 +317,9 @@ SET select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------------+---------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - 1 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33327 | 16621 - 0 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33462 | 16726 2 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33211 | 16653 + 0 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33462 | 16726 + 1 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33327 | 16621 (3 rows) select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples @@ -318,13 +327,7 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he vacuum_progress_ao_column | append-optimized compact | 75 | 27 | 51 | 0 | 100000 | 50000 (1 row) --- Resume execution of compact phase and block at syncrep on one segment. --- The suspend only takes effect the next time the walsender goes round its --- loop, so wait until it is really parked before letting the vacuum go on. --- Otherwise the compact phase can commit while the walsender is still --- streaming, syncrep is satisfied by the live mirror, the vacuum runs to --- completion on the same gang, no new vacuum worker ever takes over and the --- gp_wait_until_triggered_fault() below times out after ten minutes. +-- Keep VACUUM at the compact barrier until FTS updates the segment version. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; gp_inject_fault_infinite -------------------------- @@ -335,35 +338,41 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he ------------------------------- Success: (1 row) -2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: Success: Success: (3 rows) --- stop the mirror should turn off syncrep 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content = 1 AND role = 'm'; pg_ctl -------- OK (1 row) --- Resume walsender to detect mirror down and suspend at the beginning --- of post-cleanup taken over by a new vacuum worker. -2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +-- Let walsender detect the stopped mirror before releasing VACUUM. +2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; + gp_inject_fault_infinite +-------------------------- + Success: +(1 row) +2: SET statement_timeout = '180s'; +SET +2: SELECT wait_for_mirror_down(1::smallint, 120); + wait_for_mirror_down +---------------------- + t +(1 row) +2: RESET statement_timeout; +RESET +2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: Success: Success: (3 rows) --- resume walsender and let it exit so that mirror stop can be detected -2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; - gp_inject_fault_infinite --------------------------- - Success: -(1 row) -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -381,10 +390,10 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he ---------+------+----------------+------+-------- 2 | p | p | s | u 2 | m | m | s | u - 0 | p | p | s | u - 0 | m | m | s | u 1 | p | p | n | u 1 | m | m | n | d + 0 | p | p | s | u + 0 | m | m | s | u (6 rows) -- Resume execution and entering post_cleaup phase, suspend at the end of it. @@ -414,9 +423,9 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------------+---------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - 2 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 0 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 1 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 + 2 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 (3 rows) select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples @@ -424,6 +433,8 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 (1 row) +3: COMMIT; +COMMIT 2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- diff --git a/src/test/isolation2/expected/vacuum_progress_row.out b/src/test/isolation2/expected/vacuum_progress_row.out index 300f9ad8949..ed348e3ae18 100644 --- a/src/test/isolation2/expected/vacuum_progress_row.out +++ b/src/test/isolation2/expected/vacuum_progress_row.out @@ -343,6 +343,15 @@ ABORT DELETE FROM vacuum_progress_ao_row where j % 2 = 0; DELETE 50000 +-- Keep the compacted segments visible until post-cleanup progress is checked. +3: BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN +3: SELECT count(*) FROM vacuum_progress_ao_row; + count +------- + 50000 +(1 row) + -- Suspend execution at the end of compact phase. 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault @@ -368,9 +377,9 @@ SET select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------------+------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + 0 | vacuum_progress_ao_row | append-optimized compact | 56 | 19 | 37 | 0 | 33462 | 16725 1 | vacuum_progress_ao_row | append-optimized compact | 55 | 19 | 37 | 0 | 33327 | 16622 2 | vacuum_progress_ao_row | append-optimized compact | 55 | 19 | 37 | 0 | 33211 | 16653 - 0 | vacuum_progress_ao_row | append-optimized compact | 56 | 19 | 37 | 0 | 33462 | 16725 (3 rows) select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples @@ -378,13 +387,7 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he vacuum_progress_ao_row | append-optimized compact | 166 | 57 | 111 | 0 | 100000 | 50000 (1 row) --- Resume execution of compact phase and block at syncrep. --- The suspend only takes effect the next time the walsender goes round its --- loop, so wait until it is really parked before letting the vacuum go on. --- Otherwise the compact phase can commit while the walsender is still --- streaming, syncrep is satisfied by the live mirror, the vacuum runs to --- completion on the same gang, no new vacuum worker ever takes over and the --- gp_wait_until_triggered_fault() below times out after ten minutes. +-- Keep VACUUM at the compact barrier until FTS updates the segment version. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; gp_inject_fault_infinite -------------------------- @@ -395,35 +398,41 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he ------------------------------- Success: (1 row) -2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: Success: Success: (3 rows) --- stop the mirror should turn off syncrep 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm'; pg_ctl -------- OK (1 row) --- Resume walsender to detect mirror down and suspend at the beginning --- of post-cleanup taken over by a new vacuum worker. -2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +-- Let walsender detect the stopped mirror before releasing VACUUM. +2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; + gp_inject_fault_infinite +-------------------------- + Success: +(1 row) +2: SET statement_timeout = '180s'; +SET +2: SELECT wait_for_mirror_down(1::smallint, 120); + wait_for_mirror_down +---------------------- + t +(1 row) +2: RESET statement_timeout; +RESET +2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: Success: Success: (3 rows) --- resume walsender and let it exit so that mirror stop can be detected -2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; - gp_inject_fault_infinite --------------------------- - Success: -(1 row) -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -441,10 +450,10 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he ---------+------+----------------+------+-------- 2 | p | p | s | u 2 | m | m | s | u - 0 | p | p | s | u - 0 | m | m | s | u 1 | p | p | n | u 1 | m | m | n | d + 0 | p | p | s | u + 0 | m | m | s | u (6 rows) -- Resume execution and entering post_cleaup phase, suspend at the end of it. @@ -474,9 +483,9 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------------+------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - 0 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 1 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 2 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 + 0 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 (3 rows) select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples @@ -484,6 +493,8 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 (1 row) +3: COMMIT; +COMMIT 2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- @@ -558,6 +569,15 @@ ABORT DELETE FROM vacuum_progress_ao_row where j % 2 = 0; DELETE 50000 +-- Keep the compacted segments visible until post-cleanup progress is checked. +3: BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN +3: SELECT count(*) FROM vacuum_progress_ao_row; + count +------- + 50000 +(1 row) + -- Suspend execution at the end of compact phase. 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault @@ -583,9 +603,9 @@ SET select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------------+------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - 1 | vacuum_progress_ao_row | append-optimized compact | 55 | 19 | 37 | 0 | 33327 | 16622 2 | vacuum_progress_ao_row | append-optimized compact | 55 | 19 | 37 | 0 | 33211 | 16653 0 | vacuum_progress_ao_row | append-optimized compact | 56 | 19 | 37 | 0 | 33462 | 16725 + 1 | vacuum_progress_ao_row | append-optimized compact | 55 | 19 | 37 | 0 | 33327 | 16622 (3 rows) select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples @@ -593,13 +613,7 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he vacuum_progress_ao_row | append-optimized compact | 166 | 57 | 111 | 0 | 100000 | 50000 (1 row) --- Resume execution of compact phase and block at syncrep. --- The suspend only takes effect the next time the walsender goes round its --- loop, so wait until it is really parked before letting the vacuum go on. --- Otherwise the compact phase can commit while the walsender is still --- streaming, syncrep is satisfied by the live mirror, the vacuum runs to --- completion on the same gang, no new vacuum worker ever takes over and the --- gp_wait_until_triggered_fault() below times out after ten minutes. +-- Keep VACUUM at the compact barrier until FTS updates the segment version. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; gp_inject_fault_infinite -------------------------- @@ -610,35 +624,41 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he ------------------------------- Success: (1 row) -2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: Success: Success: (3 rows) --- stop the mirror should turn off syncrep 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm'; pg_ctl -------- OK (1 row) --- Resume walsender to detect mirror down and suspend at the beginning --- of post-cleanup taken over by a new vacuum worker. -2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +-- Let walsender detect the stopped mirror before releasing VACUUM. +2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; + gp_inject_fault_infinite +-------------------------- + Success: +(1 row) +2: SET statement_timeout = '180s'; +SET +2: SELECT wait_for_mirror_down(1::smallint, 120); + wait_for_mirror_down +---------------------- + t +(1 row) +2: RESET statement_timeout; +RESET +2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: Success: Success: (3 rows) --- resume walsender and let it exit so that mirror stop can be detected -2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; - gp_inject_fault_infinite --------------------------- - Success: -(1 row) -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -656,10 +676,10 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he ---------+------+----------------+------+-------- 2 | p | p | s | u 2 | m | m | s | u - 0 | p | p | s | u - 0 | m | m | s | u 1 | p | p | n | u 1 | m | m | n | d + 0 | p | p | s | u + 0 | m | m | s | u (6 rows) -- Resume execution and entering post_cleaup phase, suspend at the end of it. @@ -689,9 +709,9 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------------+------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - 0 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 - 2 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 1 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 + 2 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 + 0 | vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 (3 rows) select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples @@ -699,6 +719,8 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he vacuum_progress_ao_row | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 (1 row) +3: COMMIT; +COMMIT 2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- diff --git a/src/test/isolation2/sql/vacuum_progress_column.sql b/src/test/isolation2/sql/vacuum_progress_column.sql index 19bd474a0ae..a0c103a1492 100644 --- a/src/test/isolation2/sql/vacuum_progress_column.sql +++ b/src/test/isolation2/sql/vacuum_progress_column.sql @@ -124,6 +124,10 @@ CREATE INDEX on vacuum_progress_ao_column(j); 1: ABORT; DELETE FROM vacuum_progress_ao_column where j % 2 = 0; +-- Keep the compacted segments visible until post-cleanup progress is checked. +3: BEGIN ISOLATION LEVEL REPEATABLE READ; +3: SELECT count(*) FROM vacuum_progress_ao_column; + -- Suspend execution at the end of compact phase. 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; @@ -136,24 +140,18 @@ DELETE FROM vacuum_progress_ao_column where j % 2 = 0; select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; --- Resume execution of compact phase and block at syncrep on one segment. --- The suspend only takes effect the next time the walsender goes round its --- loop, so wait until it is really parked before letting the vacuum go on. --- Otherwise the compact phase can commit while the walsender is still --- streaming, syncrep is satisfied by the live mirror, the vacuum runs to --- completion on the same gang, no new vacuum worker ever takes over and the --- gp_wait_until_triggered_fault() below times out after ten minutes. +-- Keep VACUUM at the compact barrier until FTS updates the segment version. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; 2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; -2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- stop the mirror should turn off syncrep +2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content = 1 AND role = 'm'; --- Resume walsender to detect mirror down and suspend at the beginning --- of post-cleanup taken over by a new vacuum worker. -2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- resume walsender and let it exit so that mirror stop can be detected +-- Let walsender detect the stopped mirror before releasing VACUUM. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +2: SET statement_timeout = '180s'; +2: SELECT wait_for_mirror_down(1::smallint, 120); +2: RESET statement_timeout; +2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -171,6 +169,7 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; +3: COMMIT; 2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: diff --git a/src/test/isolation2/sql/vacuum_progress_row.sql b/src/test/isolation2/sql/vacuum_progress_row.sql index 8fde34ecddb..0c988b2606c 100644 --- a/src/test/isolation2/sql/vacuum_progress_row.sql +++ b/src/test/isolation2/sql/vacuum_progress_row.sql @@ -131,6 +131,10 @@ CREATE INDEX on vacuum_progress_ao_row(j); 1: ABORT; DELETE FROM vacuum_progress_ao_row where j % 2 = 0; +-- Keep the compacted segments visible until post-cleanup progress is checked. +3: BEGIN ISOLATION LEVEL REPEATABLE READ; +3: SELECT count(*) FROM vacuum_progress_ao_row; + -- Suspend execution at the end of compact phase. 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; @@ -143,24 +147,18 @@ DELETE FROM vacuum_progress_ao_row where j % 2 = 0; select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; --- Resume execution of compact phase and block at syncrep. --- The suspend only takes effect the next time the walsender goes round its --- loop, so wait until it is really parked before letting the vacuum go on. --- Otherwise the compact phase can commit while the walsender is still --- streaming, syncrep is satisfied by the live mirror, the vacuum runs to --- completion on the same gang, no new vacuum worker ever takes over and the --- gp_wait_until_triggered_fault() below times out after ten minutes. +-- Keep VACUUM at the compact barrier until FTS updates the segment version. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; 2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; -2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- stop the mirror should turn off syncrep +2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm'; --- Resume walsender to detect mirror down and suspend at the beginning --- of post-cleanup taken over by a new vacuum worker. -2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- resume walsender and let it exit so that mirror stop can be detected +-- Let walsender detect the stopped mirror before releasing VACUUM. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +2: SET statement_timeout = '180s'; +2: SELECT wait_for_mirror_down(1::smallint, 120); +2: RESET statement_timeout; +2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -178,6 +176,7 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; +3: COMMIT; 2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: @@ -212,6 +211,10 @@ CREATE INDEX on vacuum_progress_ao_row(j); 1: ABORT; DELETE FROM vacuum_progress_ao_row where j % 2 = 0; +-- Keep the compacted segments visible until post-cleanup progress is checked. +3: BEGIN ISOLATION LEVEL REPEATABLE READ; +3: SELECT count(*) FROM vacuum_progress_ao_row; + -- Suspend execution at the end of compact phase. 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; @@ -224,24 +227,18 @@ DELETE FROM vacuum_progress_ao_row where j % 2 = 0; select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; --- Resume execution of compact phase and block at syncrep. --- The suspend only takes effect the next time the walsender goes round its --- loop, so wait until it is really parked before letting the vacuum go on. --- Otherwise the compact phase can commit while the walsender is still --- streaming, syncrep is satisfied by the live mirror, the vacuum runs to --- completion on the same gang, no new vacuum worker ever takes over and the --- gp_wait_until_triggered_fault() below times out after ten minutes. +-- Keep VACUUM at the compact barrier until FTS updates the segment version. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; 2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; -2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- stop the mirror should turn off syncrep +2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm'; --- Resume walsender to detect mirror down and suspend at the beginning --- of post-cleanup taken over by a new vacuum worker. -2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- resume walsender and let it exit so that mirror stop can be detected +-- Let walsender detect the stopped mirror before releasing VACUUM. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +2: SET statement_timeout = '180s'; +2: SELECT wait_for_mirror_down(1::smallint, 120); +2: RESET statement_timeout; +2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -259,6 +256,7 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; +3: COMMIT; 2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: