Skip to content

Fix flaky VACUUM test #2051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
krylosov-aa wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix flaky VACUUM test #2051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 34 additions & 23 deletions src/test/isolation2/expected/vacuum_progress_column.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,15 @@ ABORT
DELETE FROM vacuum_progress_ao_column where j % 2 = 0;
DELETE 50000

-- Keep the compacted segments visible until post-cleanup progress is checked.
3: BEGIN ISOLATION LEVEL REPEATABLE READ;
BEGIN
3: SELECT count(*) FROM vacuum_progress_ao_column;
count
-------
50000
(1 row)

-- Suspend execution at the end of compact phase.
2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
gp_inject_fault
Expand All @@ -308,23 +317,17 @@ SET
select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1;
gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples
---------------+---------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+-----------------
1 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33327 | 16621
0 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33462 | 16726
2 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33211 | 16653
0 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33462 | 16726
1 | vacuum_progress_ao_column | append-optimized compact | 25 | 9 | 17 | 0 | 33327 | 16621
(3 rows)
select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary;
relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples
---------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+-----------------
vacuum_progress_ao_column | append-optimized compact | 75 | 27 | 51 | 0 | 100000 | 50000
(1 row)

-- Resume execution of compact phase and block at syncrep on one segment.
-- The suspend only takes effect the next time the walsender goes round its
-- loop, so wait until it is really parked before letting the vacuum go on.
-- Otherwise the compact phase can commit while the walsender is still
-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to
-- completion on the same gang, no new vacuum worker ever takes over and the
-- gp_wait_until_triggered_fault() below times out after ten minutes.
-- Keep VACUUM at the compact barrier until FTS updates the segment version.
2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
gp_inject_fault_infinite
--------------------------
Expand All @@ -335,35 +338,41 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he
-------------------------------
Success:
(1 row)
2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
gp_inject_fault
-----------------
Success:
Success:
Success:
(3 rows)
-- stop the mirror should turn off syncrep
2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content = 1 AND role = 'm';
pg_ctl
--------
OK
(1 row)

-- Resume walsender to detect mirror down and suspend at the beginning
-- of post-cleanup taken over by a new vacuum worker.
2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
-- Let walsender detect the stopped mirror before releasing VACUUM.
2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
gp_inject_fault_infinite
--------------------------
Success:
(1 row)
2: SET statement_timeout = '180s';
SET
2: SELECT wait_for_mirror_down(1::smallint, 120);
wait_for_mirror_down
----------------------
t
(1 row)
2: RESET statement_timeout;
RESET
2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
gp_inject_fault
-----------------
Success:
Success:
Success:
(3 rows)
-- resume walsender and let it exit so that mirror stop can be detected
2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
gp_inject_fault_infinite
--------------------------
Success:
(1 row)
-- Ensure we enter into the target logic which stops cumulative data but
-- initializes a new vacrelstats at the beginning of post-cleanup phase.
-- Also all segments should reach to the same "vacuum_worker_changed" point
Expand All @@ -381,10 +390,10 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he
---------+------+----------------+------+--------
2 | p | p | s | u
2 | m | m | s | u
0 | p | p | s | u
0 | m | m | s | u
1 | p | p | n | u
1 | m | m | n | d
0 | p | p | s | u
0 | m | m | s | u
(6 rows)

-- Resume execution and entering post_cleaup phase, suspend at the end of it.
Expand Down Expand Up @@ -414,16 +423,18 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he
select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1;
gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples
---------------+---------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+-----------------
2 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0
0 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0
1 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0
2 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0
(3 rows)
select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary;
relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples
---------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+-----------------
vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0
(1 row)

3: COMMIT;
COMMIT
2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
gp_inject_fault
-----------------
Expand Down
Loading