From 785a24eb06eb2d095a8f444b479ec4328e5aa3a7 Mon Sep 17 00:00:00 2001 From: Marta Kuczora Date: Tue, 18 Aug 2026 09:19:29 +0200 Subject: [PATCH] HIVE-29813: (ACID) Improve the query of duplicate deletion from COMPLETED_TXN_COMPONENTS by extending an index --- .../src/main/sql/derby/hive-schema-4.3.0.derby.sql | 2 +- .../src/main/sql/derby/upgrade-4.2.0-to-4.3.0.derby.sql | 3 +++ .../src/main/sql/mssql/hive-schema-4.3.0.mssql.sql | 2 +- .../src/main/sql/mssql/upgrade-4.2.0-to-4.3.0.mssql.sql | 3 +++ .../src/main/sql/mysql/hive-schema-4.3.0.mysql.sql | 2 +- .../src/main/sql/mysql/upgrade-4.2.0-to-4.3.0.mysql.sql | 3 +++ .../src/main/sql/oracle/hive-schema-4.3.0.oracle.sql | 2 +- .../src/main/sql/oracle/upgrade-4.2.0-to-4.3.0.oracle.sql | 3 +++ .../src/main/sql/postgres/hive-schema-4.3.0.postgres.sql | 2 +- .../src/main/sql/postgres/upgrade-4.2.0-to-4.3.0.postgres.sql | 3 +++ 10 files changed, 20 insertions(+), 5 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.3.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.3.0.derby.sql index 39d17cc88be7..06d536cf0acd 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.3.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.3.0.derby.sql @@ -556,7 +556,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_UPDATE_DELETE char(1) NOT NULL ); -CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); +CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE); CREATE TABLE TXN_LOCK_TBL ( TXN_LOCK bigint NOT NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.2.0-to-4.3.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.2.0-to-4.3.0.derby.sql index 4aa072134523..760dcdb07508 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.2.0-to-4.3.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.2.0-to-4.3.0.derby.sql @@ -3,5 +3,8 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD COLUMN MRL_CAT_NAME varchar(128) N CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID); +DROP INDEX COMPLETED_TXN_COMPONENTS_IDX; +CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE); + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1; diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.3.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.3.0.mssql.sql index ad2a45fc02b9..bb73cd85f238 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.3.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.3.0.mssql.sql @@ -1052,7 +1052,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS( ); -CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); +CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE); CREATE TABLE HIVE_LOCKS( HL_LOCK_EXT_ID bigint NOT NULL, diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.2.0-to-4.3.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.2.0-to-4.3.0.mssql.sql index 58a884a52c78..4481e70ff932 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.2.0-to-4.3.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.2.0-to-4.3.0.mssql.sql @@ -5,6 +5,9 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD MRL_CAT_NAME nvarchar(128) NOT NUL CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID); +DROP INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS; +CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0' AS MESSAGE; diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.3.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.3.0.mysql.sql index 03e37811f6cf..c768acf3350b 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.3.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.3.0.mysql.sql @@ -986,7 +986,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_UPDATE_DELETE char(1) NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1; -CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE; +CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE) USING BTREE; CREATE TABLE TXN_LOCK_TBL ( TXN_LOCK bigint NOT NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.2.0-to-4.3.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.2.0-to-4.3.0.mysql.sql index 5149dce33cff..42ca9cd0b260 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.2.0-to-4.3.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.2.0-to-4.3.0.mysql.sql @@ -5,6 +5,9 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD MRL_CAT_NAME varchar(128) NOT NUL CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID); +ALTER TABLE COMPLETED_TXN_COMPONENTS DROP INDEX COMPLETED_TXN_COMPONENTS_IDX; +CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE) USING BTREE ALGORITHM=INPLACE LOCK=NONE; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0' AS MESSAGE; diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.3.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.3.0.oracle.sql index 33958a17852f..04f3643be348 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.3.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.3.0.oracle.sql @@ -976,7 +976,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_UPDATE_DELETE CHAR(1) NOT NULL ) ROWDEPENDENCIES; -CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); +CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE); CREATE TABLE TXN_LOCK_TBL ( TXN_LOCK NUMBER(19) NOT NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.2.0-to-4.3.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.2.0-to-4.3.0.oracle.sql index bb210a6623c5..222b1ed17879 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.2.0-to-4.3.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.2.0-to-4.3.0.oracle.sql @@ -5,6 +5,9 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD (MRL_CAT_NAME VARCHAR2(128) DEFAUL CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID); +DROP INDEX COMPLETED_TXN_COMPONENTS_INDEX; +CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0' AS Status from dual; diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.3.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.3.0.postgres.sql index 10eb38923261..09f60d3c794c 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.3.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.3.0.postgres.sql @@ -1611,7 +1611,7 @@ CREATE TABLE "COMPLETED_TXN_COMPONENTS" ( "CTC_UPDATE_DELETE" char(1) NOT NULL ); -CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON "COMPLETED_TXN_COMPONENTS" USING btree ("CTC_DATABASE", "CTC_TABLE", "CTC_PARTITION"); +CREATE INDEX "COMPLETED_TXN_COMPONENTS_INDEX" ON "COMPLETED_TXN_COMPONENTS" USING btree ("CTC_DATABASE", "CTC_TABLE", "CTC_PARTITION", "CTC_WRITEID", "CTC_UPDATE_DELETE"); CREATE TABLE "TXN_LOCK_TBL" ( "TXN_LOCK" bigint NOT NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.2.0-to-4.3.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.2.0-to-4.3.0.postgres.sql index bdd28ce20298..201494d182fa 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.2.0-to-4.3.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.2.0-to-4.3.0.postgres.sql @@ -5,6 +5,9 @@ ALTER TABLE "MATERIALIZATION_REBUILD_LOCKS" ADD COLUMN "MRL_CAT_NAME" varchar(12 CREATE INDEX "MIN_HISTORY_WRITE_ID_IDX" ON "MIN_HISTORY_WRITE_ID" ("MH_DATABASE", "MH_TABLE", "MH_WRITEID"); +DROP INDEX COMPLETED_TXN_COMPONENTS_INDEX; +CREATE INDEX "COMPLETED_TXN_COMPONENTS_INDEX" ON "COMPLETED_TXN_COMPONENTS" USING btree ("CTC_DATABASE", "CTC_TABLE", "CTC_PARTITION", "CTC_WRITEID", "CTC_UPDATE_DELETE"); + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.3.0', "VERSION_COMMENT"='Hive release version 4.3.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0';