diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out index 49cff71970f1..88e5686f05a7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out @@ -357,19 +357,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"1\",\"added-data-files\":\"2\",\"deleted-data-files\":\"4\",\"added-records\":\"8\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"4\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"3\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"4\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"10\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### - numFiles 8 + numFiles 10 numPartitions 8 numRows 26 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 17 + snapshot-count 16 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -396,4 +396,3 @@ CompactionId Database Table Partition Type State Worker host Worker Enqueue Time #Masked# default ice_orc event_src_trunc=AAA/event_time_month=2024-09 MAJOR refused #Masked# manual default 0 0 0 --- #Masked# default ice_orc event_src_trunc=BBB/event_time_month=2024-07 MAJOR succeeded #Masked# manual default 0 0 0 --- #Masked# default ice_orc event_src_trunc=BBB/event_time_month=2024-08 MAJOR succeeded #Masked# manual default 0 0 0 --- -#Masked# default ice_orc --- MAJOR succeeded #Masked# manual default 0 0 0 --- diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out index c8706124ea51..aa622d876ca8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out @@ -311,19 +311,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"3\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"2\",\"added-data-files\":\"4\",\"deleted-data-files\":\"4\",\"removed-position-delete-files\":\"3\",\"removed-delete-files\":\"3\",\"added-records\":\"5\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"3\",\"changed-partition-count\":\"5\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"3\",\"manifests-kept\":\"5\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"1\",\"removed-position-delete-files\":\"1\",\"removed-delete-files\":\"1\",\"added-records\":\"1\",\"deleted-records\":\"2\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"13\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"3\",\"total-position-deletes\":\"3\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### numFiles 8 - numPartitions 4 + numPartitions 5 numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 15 + snapshot-count 14 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -350,4 +350,3 @@ CompactionId Database Table Partition Type State Worker host Worker Enqueue Time #Masked# default ice_orc company_id=100/dept_id=2 MAJOR succeeded #Masked# manual default 0 0 0 --- #Masked# default ice_orc company_id=100/dept_id=3 MAJOR succeeded #Masked# manual default 0 0 0 --- #Masked# default ice_orc company_id=100/dept_id=4 MAJOR refused #Masked# manual default 0 0 0 --- -#Masked# default ice_orc --- MAJOR succeeded #Masked# manual default 0 0 0 --- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java index adf4592f3dfc..6e08d7021991 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactOperation.java @@ -135,11 +135,11 @@ else if (desc.getPartitionSpec() != null) { compactionRequest, ServerUtils.hostname(), txnHandler, context.getConf()); parseCompactionResponse(compactionResponse, table, partitionMapEntry.getKey()); } - // If Iceberg table had partition evolution, it will create compaction request without partition specification, - // and it will compact all files from old partition specs, besides compacting partitions of current spec in parallel. + // If Iceberg table had partition evolution and the where filter is not supplied, it will create a compaction + // request without partition specification, and it will compact all files from old partition specs, + // besides compacting partitions of current spec in parallel. if (DDLUtils.isIcebergTable(table) && table.getStorageHandler().hasUndergonePartitionEvolution(table) && - (desc.getFilterExpr() == null || !table.getStorageHandler() - .getPartitionsByExpr(table, desc.getFilterExpr(), false).isEmpty())) { + desc.getFilterExpr() == null) { compactionRequest.setPartitionname(null); CompactionResponse compactionResponse = txnHandler.compact(compactionRequest); parseCompactionResponse(compactionResponse, table, compactionRequest.getPartitionname());