diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 70e50efe9c83..0d3943007c89 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2035,6 +2035,16 @@ public InlineElement getDescription() { "If true, it disables explicit type casting. For ex: it disables converting LONG type to INT type. " + "Users can enable this option to disable explicit type casting"); + public static final ConfigOption ADD_COLUMN_BEFORE_PARTITION = + ConfigOptions.key("add-column-before-partition") + .booleanType() + .defaultValue(false) + .withDescription( + "If true, when adding a new column without specifying a position, " + + "the column will be placed before the first partition column " + + "instead of at the end of the schema. " + + "This only takes effect for partitioned tables."); + public static final ConfigOption COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT = ConfigOptions.key("commit.strict-mode.last-safe-snapshot") .longType() diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java index e726b803268d..95e8b21a38e0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java @@ -293,6 +293,15 @@ public static TableSchema generateTableSchema( CoreOptions.DISABLE_EXPLICIT_TYPE_CASTING .defaultValue() .toString())); + + // boolean addColumnBeforePartition = + // Boolean.parseBoolean( + // oldOptions.getOrDefault( + // CoreOptions.ADD_COLUMN_BEFORE_PARTITION.key(), + // + // CoreOptions.ADD_COLUMN_BEFORE_PARTITION.defaultValue().toString())); + // List partitionKeys = oldTableSchema.partitionKeys(); + List newFields = new ArrayList<>(oldTableSchema.fields()); AtomicInteger highestFieldId = new AtomicInteger(oldTableSchema.highestFieldId()); String newComment = oldTableSchema.comment(); @@ -368,6 +377,20 @@ protected void updateLastColumn( throw new UnsupportedOperationException( "Unsupported move type: " + move.type()); } + // } else if (addColumnBeforePartition + // && !partitionKeys.isEmpty() + // && addColumn.fieldNames().length == 1) + // { + // int insertIndex = newFields.size(); + // for (int i = 0; i < newFields.size(); i++) + // { + // if + // (partitionKeys.contains(newFields.get(i).name())) { + // insertIndex = i; + // break; + // } + // } + // newFields.add(insertIndex, dataField); } else { newFields.add(dataField); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java index 9084b55d60e5..7b0552c76470 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java @@ -1578,4 +1578,203 @@ public void testDisableExplicitTypeCasting(String formatType) { sql("ALTER TABLE T MODIFY v INT"); assertThat(sql("SELECT * FROM T")).containsExactlyInAnyOrder(Row.of(1, 10), Row.of(2, 20)); } + + // @Test + // public void testAddColumnBeforePartitionEnabled() { + // sql( + // "CREATE TABLE T_PART (\n" + // + " user_id BIGINT,\n" + // + " item_id BIGINT,\n" + // + " behavior STRING,\n" + // + " dt STRING,\n" + // + " hh STRING\n" + // + ") PARTITIONED BY (dt, hh) WITH (\n" + // + " 'add-column-before-partition' = 'true'\n" + // + ")"); + // + // sql("INSERT INTO T_PART VALUES(1, 100, 'buy', '2024-01-01', '10')"); + // + // // Add column without specifying position + // sql("ALTER TABLE T_PART ADD score DOUBLE"); + // + // List result = sql("SHOW CREATE TABLE T_PART"); + // assertThat(result.toString()) + // .contains( + // "`user_id` BIGINT,\n" + // + " `item_id` BIGINT,\n" + // + " `behavior` VARCHAR(2147483647),\n" + // + " `score` DOUBLE,\n" + // + " `dt` VARCHAR(2147483647),\n" + // + " `hh` VARCHAR(2147483647)"); + // + // sql("INSERT INTO T_PART VALUES(2, 200, 'sell', 99.5, '2024-01-02', '11')"); + // result = sql("SELECT * FROM T_PART"); + // assertThat(result) + // .containsExactlyInAnyOrder( + // Row.of(1L, 100L, "buy", null, "2024-01-01", "10"), + // Row.of(2L, 200L, "sell", 99.5, "2024-01-02", "11")); + // } + // + // @Test + // public void testAddColumnBeforePartitionDisabledByDefault() { + // sql( + // "CREATE TABLE T_PART_DEFAULT (\n" + // + " user_id BIGINT,\n" + // + " item_id BIGINT,\n" + // + " dt STRING\n" + // + ") PARTITIONED BY (dt)"); + // + // // Add column without specifying position (default behavior) + // sql("ALTER TABLE T_PART_DEFAULT ADD score DOUBLE"); + // + // List result = sql("SHOW CREATE TABLE T_PART_DEFAULT"); + // // score should be appended at the end + // assertThat(result.toString()) + // .contains( + // "`user_id` BIGINT,\n" + // + " `item_id` BIGINT,\n" + // + " `dt` VARCHAR(2147483647),\n" + // + " `score` DOUBLE"); + // } + // + // @Test + // public void testAddColumnBeforePartitionWithExplicitPosition() { + // sql( + // "CREATE TABLE T_PART_POS (\n" + // + " user_id BIGINT,\n" + // + " item_id BIGINT,\n" + // + " dt STRING\n" + // + ") PARTITIONED BY (dt) WITH (\n" + // + " 'add-column-before-partition' = 'true'\n" + // + ")"); + // + // // Add column with explicit FIRST position, should respect explicit position + // sql("ALTER TABLE T_PART_POS ADD score DOUBLE FIRST"); + // + // List result = sql("SHOW CREATE TABLE T_PART_POS"); + // assertThat(result.toString()) + // .contains( + // "`score` DOUBLE,\n" + // + " `user_id` BIGINT,\n" + // + " `item_id` BIGINT,\n" + // + " `dt` VARCHAR(2147483647)"); + // } + // + // @Test + // public void testAddColumnBeforePartitionViaAlterOption() { + // sql( + // "CREATE TABLE T_PART_ALTER (\n" + // + " user_id BIGINT,\n" + // + " item_id BIGINT,\n" + // + " dt STRING\n" + // + ") PARTITIONED BY (dt)"); + // + // // First add column without config (default: append at end) + // sql("ALTER TABLE T_PART_ALTER ADD col1 INT"); + // List result = sql("SHOW CREATE TABLE T_PART_ALTER"); + // assertThat(result.toString()) + // .contains( + // "`user_id` BIGINT,\n" + // + " `item_id` BIGINT,\n" + // + " `dt` VARCHAR(2147483647),\n" + // + " `col1` INT"); + // + // // Enable config via ALTER TABLE SET + // sql("ALTER TABLE T_PART_ALTER SET ('add-column-before-partition' = 'true')"); + // + // // Now add another column, should go before partition column dt + // sql("ALTER TABLE T_PART_ALTER ADD col2 DOUBLE"); + // result = sql("SHOW CREATE TABLE T_PART_ALTER"); + // assertThat(result.toString()) + // .contains( + // "`user_id` BIGINT,\n" + // + " `item_id` BIGINT,\n" + // + " `col2` DOUBLE,\n" + // + " `dt` VARCHAR(2147483647),\n" + // + " `col1` INT"); + // } + // + // @Test + // public void testAddMultipleColumnsBeforePartition() { + // sql( + // "CREATE TABLE T_PART_MULTI (\n" + // + " user_id BIGINT,\n" + // + " item_id BIGINT,\n" + // + " dt STRING,\n" + // + " hh STRING\n" + // + ") PARTITIONED BY (dt, hh) WITH (\n" + // + " 'add-column-before-partition' = 'true'\n" + // + ")"); + // + // // Add first column + // sql("ALTER TABLE T_PART_MULTI ADD col1 INT"); + // // Add second column + // sql("ALTER TABLE T_PART_MULTI ADD ( col2 INT, col3 DOUBLE )"); + // + // List result = sql("SHOW CREATE TABLE T_PART_MULTI"); + // // Both new columns should be before partition columns dt and hh + // assertThat(result.toString()) + // .contains( + // "`user_id` BIGINT,\n" + // + " `item_id` BIGINT,\n" + // + " `col1` INT,\n" + // + " `col2` INT,\n" + // + " `col3` DOUBLE,\n" + // + " `dt` VARCHAR(2147483647),\n" + // + " `hh` VARCHAR(2147483647)"); + // } + // + // @Test + // public void testAddColumnBeforePartitionOnPrimaryKeyTable() { + // sql( + // "CREATE TABLE T_PK_PART (\n" + // + " user_id BIGINT,\n" + // + " item_id BIGINT,\n" + // + " behavior STRING,\n" + // + " dt STRING,\n" + // + " hh STRING,\n" + // + " PRIMARY KEY (dt, hh, user_id) NOT ENFORCED\n" + // + ") PARTITIONED BY (dt, hh) WITH (\n" + // + " 'add-column-before-partition' = 'true'\n" + // + ")"); + // + // sql("INSERT INTO T_PK_PART VALUES(1, 100, 'buy', '2024-01-01', '10')"); + // + // // Add single column + // sql("ALTER TABLE T_PK_PART ADD score DOUBLE"); + // + // List result = sql("SHOW CREATE TABLE T_PK_PART"); + // assertThat(result.toString()) + // .contains( + // "`user_id` BIGINT NOT NULL,\n" + // + " `item_id` BIGINT,\n" + // + " `behavior` VARCHAR(2147483647),\n" + // + " `score` DOUBLE,\n" + // + " `dt` VARCHAR(2147483647) NOT NULL,\n" + // + " `hh` VARCHAR(2147483647) NOT NULL"); + // + // // Add multiple columns + // sql("ALTER TABLE T_PK_PART ADD ( col1 INT, col2 DOUBLE )"); + // + // result = sql("SHOW CREATE TABLE T_PK_PART"); + // assertThat(result.toString()) + // .contains( + // "`user_id` BIGINT NOT NULL,\n" + // + " `item_id` BIGINT,\n" + // + " `behavior` VARCHAR(2147483647),\n" + // + " `score` DOUBLE,\n" + // + " `col1` INT,\n" + // + " `col2` DOUBLE,\n" + // + " `dt` VARCHAR(2147483647) NOT NULL,\n" + // + " `hh` VARCHAR(2147483647) NOT NULL"); + // + // // Verify data read/write still works + // sql("INSERT INTO T_PK_PART VALUES(2, 200, 'sell', 99.5, 10, 3.14, '2024-01-02', + // '11')"); + // result = sql("SELECT * FROM T_PK_PART"); + // assertThat(result) + // .containsExactlyInAnyOrder( + // Row.of(1L, 100L, "buy", null, null, null, "2024-01-01", "10"), + // Row.of(2L, 200L, "sell", 99.5, 10, 3.14, "2024-01-02", "11")); + // } }