Skip to content

[core] Supoort Option of adding columns before partition #7374

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
discivigour wants to merge 6 commits into
base: master
Choose a base branch
from
+232 −0
Draft

[core] Supoort Option of adding columns before partition #7374

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
02ad6ba
Supoort adding columns before partition Option.
Mar 9, 2026
9690ce4
test
Mar 9, 2026
6e9b6d2
test
Mar 9, 2026
635e2cf
Merge branch 'master' of github.com:apache/paimon into j/addColBefore…
Mar 9, 2026
f243b61
fix
Mar 9, 2026
177abd2
remove
Mar 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2035,6 +2035,16 @@ public InlineElement getDescription() {
"If true, it disables explicit type casting. For ex: it disables converting LONG type to INT type. "
+ "Users can enable this option to disable explicit type casting");

public static final ConfigOption<Boolean> ADD_COLUMN_BEFORE_PARTITION =
ConfigOptions.key("add-column-before-partition")
.booleanType()
.defaultValue(false)
.withDescription(
"If true, when adding a new column without specifying a position, "
+ "the column will be placed before the first partition column "
+ "instead of at the end of the schema. "
+ "This only takes effect for partitioned tables.");

public static final ConfigOption<Long> COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT =
ConfigOptions.key("commit.strict-mode.last-safe-snapshot")
.longType()
Expand Down
23 changes: 23 additions & 0 deletions paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,15 @@ public static TableSchema generateTableSchema(
CoreOptions.DISABLE_EXPLICIT_TYPE_CASTING
.defaultValue()
.toString()));

// boolean addColumnBeforePartition =
// Boolean.parseBoolean(
// oldOptions.getOrDefault(
// CoreOptions.ADD_COLUMN_BEFORE_PARTITION.key(),
//
// CoreOptions.ADD_COLUMN_BEFORE_PARTITION.defaultValue().toString()));
// List<String> partitionKeys = oldTableSchema.partitionKeys();

List<DataField> newFields = new ArrayList<>(oldTableSchema.fields());
AtomicInteger highestFieldId = new AtomicInteger(oldTableSchema.highestFieldId());
String newComment = oldTableSchema.comment();
Expand Down Expand Up @@ -368,6 +377,20 @@ protected void updateLastColumn(
throw new UnsupportedOperationException(
"Unsupported move type: " + move.type());
}
// } else if (addColumnBeforePartition
// && !partitionKeys.isEmpty()
// && addColumn.fieldNames().length == 1)
// {
// int insertIndex = newFields.size();
// for (int i = 0; i < newFields.size(); i++)
// {
// if
// (partitionKeys.contains(newFields.get(i).name())) {
// insertIndex = i;
// break;
// }
// }
// newFields.add(insertIndex, dataField);
} else {
newFields.add(dataField);
}
Expand Down
199 changes: 199 additions & 0 deletions ...n-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1578,4 +1578,203 @@ public void testDisableExplicitTypeCasting(String formatType) {
sql("ALTER TABLE T MODIFY v INT");
assertThat(sql("SELECT * FROM T")).containsExactlyInAnyOrder(Row.of(1, 10), Row.of(2, 20));
}

// @Test
// public void testAddColumnBeforePartitionEnabled() {
// sql(
// "CREATE TABLE T_PART (\n"
// + " user_id BIGINT,\n"
// + " item_id BIGINT,\n"
// + " behavior STRING,\n"
// + " dt STRING,\n"
// + " hh STRING\n"
// + ") PARTITIONED BY (dt, hh) WITH (\n"
// + " 'add-column-before-partition' = 'true'\n"
// + ")");
//
// sql("INSERT INTO T_PART VALUES(1, 100, 'buy', '2024-01-01', '10')");
//
// // Add column without specifying position
// sql("ALTER TABLE T_PART ADD score DOUBLE");
//
// List<Row> result = sql("SHOW CREATE TABLE T_PART");
// assertThat(result.toString())
// .contains(
// "`user_id` BIGINT,\n"
// + " `item_id` BIGINT,\n"
// + " `behavior` VARCHAR(2147483647),\n"
// + " `score` DOUBLE,\n"
// + " `dt` VARCHAR(2147483647),\n"
// + " `hh` VARCHAR(2147483647)");
//
// sql("INSERT INTO T_PART VALUES(2, 200, 'sell', 99.5, '2024-01-02', '11')");
// result = sql("SELECT * FROM T_PART");
// assertThat(result)
// .containsExactlyInAnyOrder(
// Row.of(1L, 100L, "buy", null, "2024-01-01", "10"),
// Row.of(2L, 200L, "sell", 99.5, "2024-01-02", "11"));
// }
//
// @Test
// public void testAddColumnBeforePartitionDisabledByDefault() {
// sql(
// "CREATE TABLE T_PART_DEFAULT (\n"
// + " user_id BIGINT,\n"
// + " item_id BIGINT,\n"
// + " dt STRING\n"
// + ") PARTITIONED BY (dt)");
//
// // Add column without specifying position (default behavior)
// sql("ALTER TABLE T_PART_DEFAULT ADD score DOUBLE");
//
// List<Row> result = sql("SHOW CREATE TABLE T_PART_DEFAULT");
// // score should be appended at the end
// assertThat(result.toString())
// .contains(
// "`user_id` BIGINT,\n"
// + " `item_id` BIGINT,\n"
// + " `dt` VARCHAR(2147483647),\n"
// + " `score` DOUBLE");
// }
//
// @Test
// public void testAddColumnBeforePartitionWithExplicitPosition() {
// sql(
// "CREATE TABLE T_PART_POS (\n"
// + " user_id BIGINT,\n"
// + " item_id BIGINT,\n"
// + " dt STRING\n"
// + ") PARTITIONED BY (dt) WITH (\n"
// + " 'add-column-before-partition' = 'true'\n"
// + ")");
//
// // Add column with explicit FIRST position, should respect explicit position
// sql("ALTER TABLE T_PART_POS ADD score DOUBLE FIRST");
//
// List<Row> result = sql("SHOW CREATE TABLE T_PART_POS");
// assertThat(result.toString())
// .contains(
// "`score` DOUBLE,\n"
// + " `user_id` BIGINT,\n"
// + " `item_id` BIGINT,\n"
// + " `dt` VARCHAR(2147483647)");
// }
//
// @Test
// public void testAddColumnBeforePartitionViaAlterOption() {
// sql(
// "CREATE TABLE T_PART_ALTER (\n"
// + " user_id BIGINT,\n"
// + " item_id BIGINT,\n"
// + " dt STRING\n"
// + ") PARTITIONED BY (dt)");
//
// // First add column without config (default: append at end)
// sql("ALTER TABLE T_PART_ALTER ADD col1 INT");
// List<Row> result = sql("SHOW CREATE TABLE T_PART_ALTER");
// assertThat(result.toString())
// .contains(
// "`user_id` BIGINT,\n"
// + " `item_id` BIGINT,\n"
// + " `dt` VARCHAR(2147483647),\n"
// + " `col1` INT");
//
// // Enable config via ALTER TABLE SET
// sql("ALTER TABLE T_PART_ALTER SET ('add-column-before-partition' = 'true')");
//
// // Now add another column, should go before partition column dt
// sql("ALTER TABLE T_PART_ALTER ADD col2 DOUBLE");
// result = sql("SHOW CREATE TABLE T_PART_ALTER");
// assertThat(result.toString())
// .contains(
// "`user_id` BIGINT,\n"
// + " `item_id` BIGINT,\n"
// + " `col2` DOUBLE,\n"
// + " `dt` VARCHAR(2147483647),\n"
// + " `col1` INT");
// }
//
// @Test
// public void testAddMultipleColumnsBeforePartition() {
// sql(
// "CREATE TABLE T_PART_MULTI (\n"
// + " user_id BIGINT,\n"
// + " item_id BIGINT,\n"
// + " dt STRING,\n"
// + " hh STRING\n"
// + ") PARTITIONED BY (dt, hh) WITH (\n"
// + " 'add-column-before-partition' = 'true'\n"
// + ")");
//
// // Add first column
// sql("ALTER TABLE T_PART_MULTI ADD col1 INT");
// // Add second column
// sql("ALTER TABLE T_PART_MULTI ADD ( col2 INT, col3 DOUBLE )");
//
// List<Row> result = sql("SHOW CREATE TABLE T_PART_MULTI");
// // Both new columns should be before partition columns dt and hh
// assertThat(result.toString())
// .contains(
// "`user_id` BIGINT,\n"
// + " `item_id` BIGINT,\n"
// + " `col1` INT,\n"
// + " `col2` INT,\n"
// + " `col3` DOUBLE,\n"
// + " `dt` VARCHAR(2147483647),\n"
// + " `hh` VARCHAR(2147483647)");
// }
//
// @Test
// public void testAddColumnBeforePartitionOnPrimaryKeyTable() {
// sql(
// "CREATE TABLE T_PK_PART (\n"
// + " user_id BIGINT,\n"
// + " item_id BIGINT,\n"
// + " behavior STRING,\n"
// + " dt STRING,\n"
// + " hh STRING,\n"
// + " PRIMARY KEY (dt, hh, user_id) NOT ENFORCED\n"
// + ") PARTITIONED BY (dt, hh) WITH (\n"
// + " 'add-column-before-partition' = 'true'\n"
// + ")");
//
// sql("INSERT INTO T_PK_PART VALUES(1, 100, 'buy', '2024-01-01', '10')");
//
// // Add single column
// sql("ALTER TABLE T_PK_PART ADD score DOUBLE");
//
// List<Row> result = sql("SHOW CREATE TABLE T_PK_PART");
// assertThat(result.toString())
// .contains(
// "`user_id` BIGINT NOT NULL,\n"
// + " `item_id` BIGINT,\n"
// + " `behavior` VARCHAR(2147483647),\n"
// + " `score` DOUBLE,\n"
// + " `dt` VARCHAR(2147483647) NOT NULL,\n"
// + " `hh` VARCHAR(2147483647) NOT NULL");
//
// // Add multiple columns
// sql("ALTER TABLE T_PK_PART ADD ( col1 INT, col2 DOUBLE )");
//
// result = sql("SHOW CREATE TABLE T_PK_PART");
// assertThat(result.toString())
// .contains(
// "`user_id` BIGINT NOT NULL,\n"
// + " `item_id` BIGINT,\n"
// + " `behavior` VARCHAR(2147483647),\n"
// + " `score` DOUBLE,\n"
// + " `col1` INT,\n"
// + " `col2` DOUBLE,\n"
// + " `dt` VARCHAR(2147483647) NOT NULL,\n"
// + " `hh` VARCHAR(2147483647) NOT NULL");
//
// // Verify data read/write still works
// sql("INSERT INTO T_PK_PART VALUES(2, 200, 'sell', 99.5, 10, 3.14, '2024-01-02',
// '11')");
// result = sql("SELECT * FROM T_PK_PART");
// assertThat(result)
// .containsExactlyInAnyOrder(
// Row.of(1L, 100L, "buy", null, null, null, "2024-01-01", "10"),
// Row.of(2L, 200L, "sell", 99.5, 10, 3.14, "2024-01-02", "11"));
// }
}
Loading