Fix flaky regression and isolation2 tests

contrib/pax_storage/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out

@@ -622,7 +622,11 @@ TRUNCATE
 -- end_ignore
 
 -- verify it
-1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 2);
+-- Each sample covers only ~300ms, i.e. about three 100ms CFS periods, so a
+-- capped group's reported usage jumps by a whole quota burst (~1/3 of the
+-- cap) depending on where the window falls.  Averaging five samples damps
+-- this but does not remove it, so allow a 30% relative error.
+1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 3);
  verify_cpu_usage 
 ------------------
  t                
@@ -797,12 +801,16 @@ TRUNCATE
 (1 row)
 -- end_ignore
 
-1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 2);
+-- Each sample covers only ~300ms, i.e. about three 100ms CFS periods, so a
+-- capped group's reported usage jumps by a whole quota burst (~1/3 of the
+-- cap) depending on where the window falls.  Averaging five samples damps
+-- this but does not remove it, so allow a 30% relative error.
+1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 3);
  verify_cpu_usage 
 ------------------
  t                
 (1 row)
-1:SELECT verify_cpu_usage('rg2_cpu_test', 20, 2);
+1:SELECT verify_cpu_usage('rg2_cpu_test', 20, 6);
  verify_cpu_usage 
 ------------------
  t                

contrib/pax_storage/src/test/isolation2/expected/resource_queue_multi_portal.out

@@ -150,6 +150,8 @@ DECLARE
 
 -- There should now only be one active statement, following the abort of session
 -- 1's transaction. The active statement is contributed by session 2.
+1<:  <... completed>
+ERROR:  canceling statement due to user request
 0:SELECT rsqcountlimit, rsqcountvalue FROM pg_resqueue_status WHERE rsqname = 'rq_multi_portal';
  rsqcountlimit | rsqcountvalue 
 ---------------+---------------
@@ -162,8 +164,6 @@ DECLARE
 (1 row)
 
 -- After ending the transactions, there should be 0 active statements.
-1<:  <... completed>
-ERROR:  canceling statement due to user request
 1:END;
 END
 2:END;

contrib/pax_storage/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql

@@ -314,7 +314,11 @@ ALTER RESOURCE GROUP rg2_cpu_test set cpu_max_percent 20;
 -- end_ignore
 
 -- verify it
-1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 2);
+-- Each sample covers only ~300ms, i.e. about three 100ms CFS periods, so a
+-- capped group's reported usage jumps by a whole quota burst (~1/3 of the
+-- cap) depending on where the window falls.  Averaging five samples damps
+-- this but does not remove it, so allow a 30% relative error.
+1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 3);
 
 -- start_ignore
 1:SELECT * FROM cancel_all;
@@ -385,8 +389,12 @@ ALTER RESOURCE GROUP rg2_cpu_test set cpu_max_percent 20;
 1:SELECT pg_sleep(1.7);
 -- end_ignore
 
-1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 2);
-1:SELECT verify_cpu_usage('rg2_cpu_test', 20, 2);
+-- Each sample covers only ~300ms, i.e. about three 100ms CFS periods, so a
+-- capped group's reported usage jumps by a whole quota burst (~1/3 of the
+-- cap) depending on where the window falls.  Averaging five samples damps
+-- this but does not remove it, so allow a 30% relative error.
+1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 3);
+1:SELECT verify_cpu_usage('rg2_cpu_test', 20, 6);
 
 -- start_ignore
 1:SELECT * FROM cancel_all;

contrib/pax_storage/src/test/isolation2/sql/resource_queue_multi_portal.sql

@@ -88,12 +88,12 @@
 
 -- There should now only be one active statement, following the abort of session
 -- 1's transaction. The active statement is contributed by session 2.
+1<:
 0:SELECT rsqcountlimit, rsqcountvalue FROM pg_resqueue_status WHERE rsqname = 'rq_multi_portal';
 0:SELECT query, state from pg_stat_activity
   WHERE query = 'DECLARE c2 CURSOR FOR SELECT 1;';
 
 -- After ending the transactions, there should be 0 active statements.
-1<:
 1:END;
 2:END;
 0:SELECT rsqcountlimit, rsqcountvalue FROM pg_resqueue_status WHERE rsqname = 'rq_multi_portal';

contrib/pax_storage/src/test/regress/expected/gp_tablespace.out

@@ -493,6 +493,5 @@ DROP TABLE tblspc_otherloc_heap;
 DROP TABLESPACE testspace_otherloc;
 CREATE TABLESPACE testspace_dir_empty LOCATION :'testtablespace';
 CREATE TABLE t_dir_empty(a int);
-\! rm -rf $PG_ABS_SRCDIR/testtablespace
 DROP TABLE IF EXISTS t_dir_empty;
 DROP TABLESPACE testspace_dir_empty;

contrib/pax_storage/src/test/regress/expected/index_constraint_naming_partition.out

@@ -177,10 +177,11 @@ CREATE FUNCTION recreate_two_level_table() RETURNS VOID
     );
 $fn$;
 -- validate that there are no constraints when we start
-SELECT * FROM partition_tables_show_all('r');
+-- call partition_tables() directly: inside partition_tables_show_all() the
+-- error depends on the join order picked by the planner
+SELECT * FROM partition_tables('r');
 ERROR:  relation "r" does not exist
 CONTEXT:  SQL function "partition_tables" statement 1
-SQL function "partition_tables_show_all" statement 1
 -- UNIQUE constraint: validate we correctly add it and can only drop from root
 SELECT recreate_two_level_table();
 NOTICE:  table "r" does not exist, skipping
@@ -872,10 +873,11 @@ CREATE FUNCTION recreate_three_level_table() RETURNS VOID
     );
 $fn$;
 -- validate that there are no constraints when we start
-SELECT * FROM partition_tables_show_all('r');
+-- call partition_tables() directly: inside partition_tables_show_all() the
+-- error depends on the join order picked by the planner
+SELECT * FROM partition_tables('r');
 ERROR:  relation "r" does not exist
 CONTEXT:  SQL function "partition_tables" statement 1
-SQL function "partition_tables_show_all" statement 1
 -- UNIQUE constraint: validate we correctly add it and can only drop from root
 SELECT recreate_three_level_table();
 NOTICE:  table "r" does not exist, skipping

contrib/pax_storage/src/test/regress/init_file

@@ -89,6 +89,13 @@ m/Distributed randomly/
 -- m/^HINT.*To uniquely identify a row within a distributed table.*/
 -- m/^NOTICE.*SELECT uses system-defined column.*/
 
+# When a QE errors out, the QD cancels the other QEs.  A QE that has already
+# committed locally (e.g. the first transaction of CREATE INDEX CONCURRENTLY)
+# and is waiting for the mirror ignores the cancel and warns about it.  Whether
+# that happens depends on timing, so ignore the warning and its detail.
+m/^WARNING:  ignoring query cancel request for synchronous replication to ensure cluster consistency/
+m/^DETAIL:  The transaction has already changed locally, it has to be replicated to standby\./
+
 -- end_matchignore
 
 -- start_matchsubs

contrib/pax_storage/src/test/regress/sql/disable_autovacuum.sql

@@ -4,5 +4,5 @@ select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg
 -- The reason for restarting cbdb here is that if the subsequent 
 -- vacuum ao test(uao*_catalog_tables/threshold) encounters an
 -- unfinished transaction, it will fail.
---\!gpstop -ari
+\!gpstop -ari
 -- end_ignore

contrib/pax_storage/src/test/regress/sql/gp_tablespace.sql

@@ -265,7 +265,6 @@ DROP TABLESPACE testspace_otherloc;
 
 CREATE TABLESPACE testspace_dir_empty LOCATION :'testtablespace';
 CREATE TABLE t_dir_empty(a int);
-\! rm -rf $PG_ABS_SRCDIR/testtablespace
 DROP TABLE IF EXISTS t_dir_empty;
 DROP TABLESPACE testspace_dir_empty;
 

contrib/pax_storage/src/test/regress/sql/index_constraint_naming_partition.sql

@@ -180,7 +180,9 @@ CREATE FUNCTION recreate_two_level_table() RETURNS VOID
 $fn$;
 
 -- validate that there are no constraints when we start
-SELECT * FROM partition_tables_show_all('r');
+-- call partition_tables() directly: inside partition_tables_show_all() the
+-- error depends on the join order picked by the planner
+SELECT * FROM partition_tables('r');
 
 -- UNIQUE constraint: validate we correctly add it and can only drop from root
 SELECT recreate_two_level_table();
@@ -331,7 +333,9 @@ CREATE FUNCTION recreate_three_level_table() RETURNS VOID
 $fn$;
 
 -- validate that there are no constraints when we start
-SELECT * FROM partition_tables_show_all('r');
+-- call partition_tables() directly: inside partition_tables_show_all() the
+-- error depends on the join order picked by the planner
+SELECT * FROM partition_tables('r');
 
 -- UNIQUE constraint: validate we correctly add it and can only drop from root
 SELECT recreate_three_level_table();

gpcontrib/gp_toolkit/expected/gp_toolkit.out

@@ -223,6 +223,26 @@ from gp_toolkit.gp_bloat_diag where bdirelid = 'toolkit_skew'::regclass;
  toolkit_skew | public     | toolkit_skew | t                   | t                    | f                   | t                    | significant amount of bloat suspected
 (1 row)
 
+-- Vacuum can only remove the deleted rows, and so truncate the table, if no
+-- other backend holds a snapshot older than the delete, e.g. an auto-analyze
+-- started after the cluster restart in the previous test. Wait for those to
+-- finish. Plain autovacuum VACUUMs do not hold back the horizon.
+DO $$
+BEGIN
+     FOR i in 1..600 loop
+	      IF NOT EXISTS (SELECT 1 FROM pg_stat_activity
+	                     WHERE pid <> pg_backend_pid()
+	                       AND (backend_xmin IS NOT NULL OR backend_xid IS NOT NULL)
+	                       AND NOT (backend_type = 'autovacuum worker'
+	                                AND query LIKE 'autovacuum: VACUUM %'
+	                                AND query NOT LIKE '%ANALYZE%')) THEN
+	         RETURN;
+	      END IF;
+	      PERFORM pg_sleep(0.1);
+	      PERFORM pg_stat_clear_snapshot();
+     END LOOP;
+     raise notice 'other backends still hold a snapshot';
+END$$;
 -- After vacuum, bloat should be gone
 vacuum toolkit_skew;
 select btdrelpages > 0 as btdrelpages_over_0,

gpcontrib/gp_toolkit/gp_toolkit--1.0.sql

@@ -252,7 +252,7 @@ CREATE EXTERNAL WEB TABLE gp_toolkit.__gp_log_master_ext
     logstack text
 )
 EXECUTE E'cat $GP_SEG_DATADIR/log/*.csv' ON COORDINATOR
-FORMAT 'CSV' (DELIMITER AS ',' NULL AS '' QUOTE AS '"');
+FORMAT 'CSV' (DELIMITER AS ',' NULL AS '' QUOTE AS '"' ESCAPE AS '\');
 
 REVOKE ALL ON TABLE gp_toolkit.__gp_log_master_ext FROM public;
 

gpcontrib/gp_toolkit/sql/gp_toolkit.sql

@@ -123,6 +123,26 @@ select bdirelid::regclass, bdinspname, bdirelname,
        bdidiag
 from gp_toolkit.gp_bloat_diag where bdirelid = 'toolkit_skew'::regclass;
 
+-- Vacuum can only remove the deleted rows, and so truncate the table, if no
+-- other backend holds a snapshot older than the delete, e.g. an auto-analyze
+-- started after the cluster restart in the previous test. Wait for those to
+-- finish. Plain autovacuum VACUUMs do not hold back the horizon.
+DO $$
+BEGIN
+     FOR i in 1..600 loop
+	      IF NOT EXISTS (SELECT 1 FROM pg_stat_activity
+	                     WHERE pid <> pg_backend_pid()
+	                       AND (backend_xmin IS NOT NULL OR backend_xid IS NOT NULL)
+	                       AND NOT (backend_type = 'autovacuum worker'
+	                                AND query LIKE 'autovacuum: VACUUM %'
+	                                AND query NOT LIKE '%ANALYZE%')) THEN
+	         RETURN;
+	      END IF;
+	      PERFORM pg_sleep(0.1);
+	      PERFORM pg_stat_clear_snapshot();
+     END LOOP;
+     raise notice 'other backends still hold a snapshot';
+END$$;
 -- After vacuum, bloat should be gone
 vacuum toolkit_skew;
 select btdrelpages > 0 as btdrelpages_over_0,

src/backend/catalog/heap.c

@@ -2162,40 +2162,14 @@ RemoveAttributeById(Oid relid, AttrNumber attnum)
 		/* Unset this so no one tries to look up the generation expression */
 		attStruct->attgenerated = '\0';
 
-		/* Update distribution policy for dropped distribution column */
-		if (GpPolicyIsHashPartitioned(rel->rd_cdbpolicy))
-		{
-			int            ia = 0;
-
-			for (ia = 0; ia < rel->rd_cdbpolicy->nattrs; ia++)
-			{
-				if (attnum == rel->rd_cdbpolicy->attrs[ia])
-				{
-					MemoryContext oldcontext;
-					GpPolicy *policy;
-
-					/* force a random distribution */
-					rel->rd_cdbpolicy->nattrs = 0;
-
-					oldcontext = MemoryContextSwitchTo(GetMemoryChunkContext(rel));
-					policy = GpPolicyCopy(rel->rd_cdbpolicy);
-					MemoryContextSwitchTo(oldcontext);
-
-					/*
-					 * replace policy first in catalog and then assign to
-					 * rd_cdbpolicy to make sure we have intended policy in relcache
-					 * even with relcache invalidation. Otherwise rd_cdbpolicy can
-					 * become invalid soon after assignment.
-					 */
-					GpPolicyReplace(RelationGetRelid(rel), policy);
-					rel->rd_cdbpolicy = policy;
-					if (Gp_role != GP_ROLE_EXECUTE)
-						ereport(NOTICE,
-								(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-									errmsg("dropping a column that is part of the distribution policy forces a random distribution policy")));
-				}
-			}
-		}
+		/*
+		 * The distribution policy was already switched to random by
+		 * set_random_distribution_if_drop_distkey() above.  Don't consult
+		 * rel->rd_cdbpolicy again here: a relcache rebuild in between (e.g.
+		 * after a sinval reset) reloads the not-yet-visible old policy, and
+		 * a second GpPolicyReplace() on the same gp_distribution_policy row
+		 * fails with "tuple already updated by self".
+		 */
 
 		/*
 		 * Change the column name to something that isn't likely to conflict

src/test/isolation2/expected/bitmap_index_crash.out

@@ -79,6 +79,14 @@ ALTER
 --------
  OK     
 (1 row)
+-- pg_ctl -w returns once crash recovery has started, not when seg1 accepts
+-- connections again. Gang creation retries while the segment is in recovery,
+-- so dispatch to it from a new session before connecting directly below.
+3:SELECT count(*) FROM bm;
+ count 
+-------
+ 5000  
+(1 row)
 3:SELECT gp_inject_fault_infinite('checkpoint', 'reset', dbid) FROM gp_segment_configuration WHERE role='p';
  gp_inject_fault_infinite 
 --------------------------

src/test/isolation2/expected/orphan_temp_table.out

@@ -60,6 +60,10 @@ BEGIN
 (0 rows)
 2q: ... <quitting>
 
+-- 2q only closes the connection; the backend drops its temp objects on
+-- exit asynchronously, so wait for that to finish (up to 60s).
+3: do $$ begin for i in 1..600 loop if (select count(*) from pg_class where relname = 'test_temp_table_cleanup') = 0 then return; end if; perform pg_sleep(0.1); end loop; end $$;
+DO
 3: select count(*) from pg_class where relname = 'test_temp_table_cleanup';
  count 
 -------

src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out

@@ -622,7 +622,11 @@ TRUNCATE
 -- end_ignore
 
 -- verify it
-1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 2);
+-- Each sample covers only ~300ms, i.e. about three 100ms CFS periods, so a
+-- capped group's reported usage jumps by a whole quota burst (~1/3 of the
+-- cap) depending on where the window falls.  Averaging five samples damps
+-- this but does not remove it, so allow a 30% relative error.
+1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 3);
  verify_cpu_usage 
 ------------------
  t                
@@ -797,12 +801,16 @@ TRUNCATE
 (1 row)
 -- end_ignore
 
-1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 2);
+-- Each sample covers only ~300ms, i.e. about three 100ms CFS periods, so a
+-- capped group's reported usage jumps by a whole quota burst (~1/3 of the
+-- cap) depending on where the window falls.  Averaging five samples damps
+-- this but does not remove it, so allow a 30% relative error.
+1:SELECT verify_cpu_usage('rg1_cpu_test', 10, 3);
  verify_cpu_usage 
 ------------------
  t                
 (1 row)
-1:SELECT verify_cpu_usage('rg2_cpu_test', 20, 2);
+1:SELECT verify_cpu_usage('rg2_cpu_test', 20, 6);
  verify_cpu_usage 
 ------------------
  t                

src/test/isolation2/expected/segwalrep/startup_rename_prepared_xlog.out

@@ -23,6 +23,12 @@
 !\retcode gpstop -rai;
 (exited with code 0)
 
+-- make sure the mirror is in sync, otherwise FTS will not promote it below
+1: select wait_until_segment_synchronized(0);
+ wait_until_segment_synchronized 
+---------------------------------
+ OK                              
+(1 row)
 1: create extension if not exists gp_inject_fault;
 CREATE
 1: create table t_rename1 (a int);