From f12fee8f46e1180a87e8119f900e3db818e38d1c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 17 Aug 2026 11:55:31 +0800 Subject: [PATCH 1/4] [fix](iceberg) Backport external write hardening to branch-4.1 --- .../operator/iceberg_sorter_reserve_memory.h | 132 +++++ be/src/exec/operator/operator.h | 4 + .../spill_iceberg_table_sink_operator.cpp | 97 ++-- .../spill_iceberg_table_sink_operator.h | 8 +- .../pipeline/pipeline_fragment_context.cpp | 135 ++--- .../exec/pipeline/pipeline_fragment_context.h | 2 + be/src/exec/pipeline/pipeline_task.cpp | 3 +- .../exec/pipeline/report_exec_status_size.h | 42 ++ be/src/exec/sink/viceberg_delete_sink.cpp | 28 +- be/src/exec/sink/viceberg_delete_sink.h | 1 + .../exec/sink/writer/async_result_writer.cpp | 74 ++- be/src/exec/sink/writer/async_result_writer.h | 24 +- .../writer/async_writer_queue_admission.h | 53 ++ .../writer/hive_multipart_compatibility.h | 29 ++ .../iceberg/iceberg_writer_compatibility.h | 35 ++ .../iceberg/viceberg_partition_writer.cpp | 32 +- .../writer/iceberg/viceberg_sort_writer.cpp | 130 +++-- .../writer/iceberg/viceberg_sort_writer.h | 9 + .../writer/iceberg/viceberg_table_writer.cpp | 201 ++++++-- .../writer/iceberg/viceberg_table_writer.h | 31 +- .../sink/writer/vhive_partition_writer.cpp | 18 +- .../exec/sink/writer/vhive_partition_writer.h | 1 + be/src/exec/sort/sorter.cpp | 70 ++- be/src/exec/sort/sorter.h | 20 + be/src/format/table/iceberg/schema.cpp | 29 +- be/src/format/table/iceberg/schema.h | 4 + be/src/format/table/iceberg_default_value.h | 51 ++ be/src/format/table/iceberg_scan_semantics.h | 6 + .../iceberg_partition_function.cpp | 60 ++- .../transformer/iceberg_partition_function.h | 4 + be/src/format_v2/column_data.h | 2 + be/src/format_v2/column_mapper.cpp | 79 +-- be/src/format_v2/column_mapper.h | 2 + ...eberg_position_delete_sys_table_reader.cpp | 6 + be/src/format_v2/table/iceberg_reader.cpp | 469 ++++++++++++++++-- be/src/format_v2/table/iceberg_reader.h | 6 + be/src/format_v2/table_reader.cpp | 3 + be/src/format_v2/table_reader.h | 241 ++++++++- be/src/io/fs/azure_obj_storage_client.cpp | 46 +- be/src/io/fs/azure_obj_storage_client.h | 1 + be/src/io/fs/obj_storage_client.h | 5 +- be/src/io/fs/s3_file_writer.cpp | 3 +- be/src/io/fs/s3_file_writer.h | 1 - .../runtime/memory/thread_mem_tracker_mgr.cpp | 40 ++ .../runtime/memory/thread_mem_tracker_mgr.h | 40 ++ be/src/runtime/runtime_state.cpp | 93 ++++ be/src/runtime/runtime_state.h | 40 +- be/test/core/value/merge_partitioner_test.cpp | 91 ++++ ...spill_iceberg_table_sink_operator_test.cpp | 146 ++++++ .../exec/sink/viceberg_delete_sink_test.cpp | 12 + .../exec/sink/viceberg_merge_sink_test.cpp | 38 +- .../sink/writer/async_result_writer_test.cpp | 185 +++++++ .../iceberg/iceberg_partition_writer_test.cpp | 64 ++- .../iceberg/iceberg_table_writer_test.cpp | 180 +++++++ ...partition_writer_report_lifecycle_test.cpp | 217 ++++++++ be/test/exec/sort/full_sort_test.cpp | 24 +- be/test/format/table/iceberg/schema_test.cpp | 31 ++ ..._position_delete_sys_table_reader_test.cpp | 25 + be/test/format_v2/table_reader_test.cpp | 221 +++++++++ be/test/io/client/s3_file_system_test.cpp | 13 +- .../io/fs/azure_obj_storage_client_test.cpp | 74 +++ .../memory/thread_mem_tracker_mgr_test.cpp | 33 ++ .../runtime_state_block_budget_test.cpp | 119 +++++ .../java/org/apache/doris/catalog/Type.java | 25 +- .../common/proc/IndexSchemaProcNode.java | 4 +- .../apache/doris/common/util/SqlUtils.java | 6 + .../doris/datasource/hive/HMSTransaction.java | 61 ++- .../iceberg/IcebergTransaction.java | 33 +- .../action/IcebergExecuteActionFactory.java | 7 +- .../IcebergRemoveOrphanFilesAction.java | 377 ++++++++++++++ .../iceberg/source/IcebergScanNode.java | 218 +++++++- .../apache/doris/fs/obj/AzureObjStorage.java | 27 +- .../translator/PhysicalPlanTranslator.java | 3 +- .../properties/DistributionSpecMerge.java | 17 +- .../insert/AbstractInsertExecutor.java | 13 +- .../BaseExternalTableInsertExecutor.java | 19 +- .../insert/InsertIntoTableCommand.java | 2 + .../physical/PhysicalIcebergMergeSink.java | 87 +++- .../doris/nereids/util/SqlLiteralUtils.java | 5 +- .../apache/doris/planner/DataPartition.java | 10 + .../apache/doris/planner/HiveTableSink.java | 3 + .../apache/doris/qe/AbstractJobProcessor.java | 11 +- .../java/org/apache/doris/qe/Coordinator.java | 188 ++++--- .../org/apache/doris/qe/JobProcessor.java | 2 +- .../apache/doris/qe/NereidsCoordinator.java | 4 +- .../org/apache/doris/qe/QeProcessorImpl.java | 60 ++- .../org/apache/doris/qe/SessionVariable.java | 3 + .../doris/qe/runtime/LoadProcessor.java | 47 +- .../runtime/SingleFragmentPipelineTask.java | 11 +- .../common/proc/IndexSchemaProcNodeTest.java | 24 +- .../hive/HMSTransactionPathTest.java | 37 ++ .../iceberg/IcebergTransactionTest.java | 43 +- .../IcebergRemoveOrphanFilesActionTest.java | 47 ++ .../iceberg/source/IcebergScanNodeTest.java | 33 ++ .../doris/fs/obj/AzureObjStorageTest.java | 15 + .../qe/QeProcessorImplReportAckTest.java | 190 +++++++ .../apache/doris/qe/SessionVariablesTest.java | 10 + .../SingleFragmentPipelineTaskTest.java | 15 + gensrc/thrift/DataSinks.thrift | 1 + gensrc/thrift/FrontendService.thrift | 2 + gensrc/thrift/PaloInternalService.thrift | 5 + gensrc/thrift/Partitions.thrift | 2 + .../test_iceberg_write_complex_evolution.out | 4 +- .../doris/regression/suite/Suite.groovy | 12 +- .../suite/SuiteJobLookupTest.groovy | 33 ++ ...est_iceberg_write_complex_evolution.groovy | 26 +- 106 files changed, 5055 insertions(+), 570 deletions(-) create mode 100644 be/src/exec/operator/iceberg_sorter_reserve_memory.h create mode 100644 be/src/exec/pipeline/report_exec_status_size.h create mode 100644 be/src/exec/sink/writer/async_writer_queue_admission.h create mode 100644 be/src/exec/sink/writer/hive_multipart_compatibility.h create mode 100644 be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h create mode 100644 be/src/format/table/iceberg_default_value.h create mode 100644 be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp create mode 100644 be/test/exec/sink/writer/async_result_writer_test.cpp create mode 100644 be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp create mode 100644 be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesAction.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesActionTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteJobLookupTest.groovy diff --git a/be/src/exec/operator/iceberg_sorter_reserve_memory.h b/be/src/exec/operator/iceberg_sorter_reserve_memory.h new file mode 100644 index 00000000000000..8b09a0af4dfbfc --- /dev/null +++ b/be/src/exec/operator/iceberg_sorter_reserve_memory.h @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +namespace doris { + +class Block; + +struct IcebergSorterReserveMemory { + size_t retained_growth = 0; + size_t retained_growth_trigger_bytes = 0; + size_t transient_workspace = 0; +}; + +inline size_t iceberg_saturating_add(size_t lhs, size_t rhs) { + return std::min(std::numeric_limits::max() - lhs, rhs) + lhs; +} + +inline size_t bounded_iceberg_reserve_size( + const std::vector& per_partition_reservations, + size_t incoming_rows = std::numeric_limits::max(), + size_t incoming_bytes = std::numeric_limits::max()) { + size_t transient_workspace = 0; + for (const auto& reservation : per_partition_reservations) { + transient_workspace = std::max(transient_workspace, reservation.transient_workspace); + } + + std::vector growth_candidates; + growth_candidates.reserve(per_partition_reservations.size()); + for (const auto& reservation : per_partition_reservations) { + if (reservation.retained_growth > 0) { + growth_candidates.push_back(&reservation); + } + } + + std::sort(growth_candidates.begin(), growth_candidates.end(), + [](const auto* lhs, const auto* rhs) { + return lhs->retained_growth > rhs->retained_growth; + }); + size_t row_bound = 0; + for (size_t i = 0; i < std::min(incoming_rows, growth_candidates.size()); ++i) { + row_bound = iceberg_saturating_add(row_bound, growth_candidates[i]->retained_growth); + } + + size_t byte_bound = 0; + std::vector positive_trigger_candidates; + positive_trigger_candidates.reserve(growth_candidates.size()); + for (const auto* reservation : growth_candidates) { + if (reservation->retained_growth_trigger_bytes == 0) { + byte_bound = iceberg_saturating_add(byte_bound, reservation->retained_growth); + } else { + positive_trigger_candidates.push_back(reservation); + } + } + std::sort(positive_trigger_candidates.begin(), positive_trigger_candidates.end(), + [](const auto* lhs, const auto* rhs) { + return static_cast(lhs->retained_growth) * + rhs->retained_growth_trigger_bytes > + static_cast(rhs->retained_growth) * + lhs->retained_growth_trigger_bytes; + }); + size_t remaining_bytes = incoming_bytes; + for (const auto* reservation : positive_trigger_candidates) { + if (reservation->retained_growth_trigger_bytes <= remaining_bytes) { + byte_bound = iceberg_saturating_add(byte_bound, reservation->retained_growth); + remaining_bytes -= reservation->retained_growth_trigger_bytes; + continue; + } + const auto numerator = + static_cast(reservation->retained_growth) * remaining_bytes + + reservation->retained_growth_trigger_bytes - 1; + const auto fractional_growth = + std::min(numerator / reservation->retained_growth_trigger_bytes, + std::numeric_limits::max()); + byte_bound = iceberg_saturating_add(byte_bound, static_cast(fractional_growth)); + break; + } + + // A block's rows and bytes are divided across partition sorters. The two fractional-relaxation + // bounds avoid charging the complete input block to every active partition while remaining safe. + const size_t retained_growth = std::min(row_bound, byte_bound); + return iceberg_saturating_add(retained_growth, transient_workspace); +} + +inline size_t iceberg_reserve_size( + const std::vector& per_partition_reservations, + size_t incoming_block_reserve, size_t incoming_rows = std::numeric_limits::max(), + size_t incoming_bytes = std::numeric_limits::max()) { + size_t sorter_reserve = + bounded_iceberg_reserve_size(per_partition_reservations, incoming_rows, incoming_bytes); + // The incoming block creates cold partition writers before they can appear in the published snapshot. + return iceberg_saturating_add(sorter_reserve, incoming_block_reserve); +} + +size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_workspace_bytes); + +inline size_t iceberg_spill_merge_workspace(size_t spill_file_count, size_t spill_buffer_bytes, + size_t merge_limit_bytes) { + if (spill_file_count == 0 || spill_buffer_bytes == 0) { + return 0; + } + const size_t max_fan_in = std::max(2, merge_limit_bytes / spill_buffer_bytes); + const size_t input_count = std::min(spill_file_count, max_fan_in); + const size_t max_size = std::numeric_limits::max(); + const size_t input_bytes = input_count > max_size / spill_buffer_bytes + ? max_size + : input_count * spill_buffer_bytes; + // VSortedRunMerger materializes one block per input cursor plus the block being emitted. + return input_bytes > max_size - spill_buffer_bytes ? max_size + : input_bytes + spill_buffer_bytes; +} + +} // namespace doris diff --git a/be/src/exec/operator/operator.h b/be/src/exec/operator/operator.h index e193d57163b364..72b2cbced48b5a 100644 --- a/be/src/exec/operator/operator.h +++ b/be/src/exec/operator/operator.h @@ -652,6 +652,10 @@ class DataSinkOperatorXBase : public OperatorBase { [[nodiscard]] virtual size_t get_reserve_mem_size(RuntimeState* state, bool eos) { return state->minimum_operator_memory_required_bytes(); } + [[nodiscard]] virtual size_t get_reserve_mem_size(RuntimeState* state, bool eos, + const Block* block) { + return get_reserve_mem_size(state, eos); + } bool is_blockable(RuntimeState* state) const override { return state->get_sink_local_state()->is_blockable(); } diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 20b4eea954a599..8ad3b86f132c7a 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -18,6 +18,7 @@ #include "exec/operator/spill_iceberg_table_sink_operator.h" #include "common/status.h" +#include "core/block/block.h" #include "exec/operator/iceberg_table_sink_operator.h" #include "exec/operator/spill_utils.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" @@ -26,12 +27,27 @@ namespace doris { #include "common/compile_check_begin.h" +size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_workspace_bytes) { + const size_t block_bytes = block.allocated_bytes(); + const size_t row_index_bytes = + std::min(std::numeric_limits::max() / sizeof(size_t), block.rows()) * + sizeof(size_t); + const size_t dispatch_copies = block_bytes > std::numeric_limits::max() / 4 + ? std::numeric_limits::max() + : block_bytes * 4; + size_t reserve = iceberg_saturating_add(writer_workspace_bytes, dispatch_copies); + // Transform, selected blocks, and retained sorters coexist during high-cardinality dispatch. + return iceberg_saturating_add(reserve, row_index_bytes); +} + SpillIcebergTableSinkLocalState::SpillIcebergTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) : Base(parent, state) {} Status SpillIcebergTableSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); + // Admission samples async sorter state, so wait until the prior append has published it. + _writer->wait_for_processing_before_next_sink(); SCOPED_TIMER(exec_time_counter()); SCOPED_TIMER(_init_timer); @@ -53,30 +69,49 @@ bool SpillIcebergTableSinkLocalState::is_blockable() const { return true; } -size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos) { +size_t SpillIcebergTableSinkLocalState::get_reserve_mem_size(RuntimeState* state, bool eos, + const Block* block) { if (!_writer) { return 0; } - auto current_writer = _writer->current_writer(); - auto* sort_writer = dynamic_cast(current_writer.get()); - if (!sort_writer) { - return 0; + std::vector per_partition_reservations; + const size_t incoming_rows = block == nullptr ? 0 : block->rows(); + const size_t incoming_bytes = block == nullptr ? 0 : block->allocated_bytes(); + auto active_writers = _writer->active_writers(); + per_partition_reservations.reserve(active_writers->size()); + for (const auto& writer : *active_writers) { + if (auto* sort_writer = dynamic_cast(writer.get())) { + auto reservation = sort_writer->get_reserve_mem_size_components( + state, eos, incoming_rows, incoming_bytes); + per_partition_reservations.push_back( + {.retained_growth = reservation.retained_growth, + .retained_growth_trigger_bytes = reservation.retained_growth_trigger_bytes, + .transient_workspace = reservation.transient_workspace}); + } } - - return sort_writer->get_reserve_mem_size(state, eos); + // Column growth remains in every touched sorter, while sorting workspace is reused by serial dispatch. + // The final queued item may contain rows and also owns the reservation used by async finish(). + const size_t incoming_reserve = + block == nullptr ? state->minimum_operator_memory_required_bytes() + : iceberg_cold_writer_reserve_size( + *block, state->minimum_operator_memory_required_bytes()); + return iceberg_reserve_size(per_partition_reservations, incoming_reserve, incoming_rows, + incoming_bytes); } size_t SpillIcebergTableSinkLocalState::get_revocable_mem_size(RuntimeState* state) const { if (!_writer) { return 0; } - auto current_writer = _writer->current_writer(); - auto* sort_writer = dynamic_cast(current_writer.get()); - if (!sort_writer) { - return 0; + size_t revocable_size = 0; + // Retain the published container while the async writer may replace the current snapshot. + auto active_writers = _writer->active_writers(); + for (const auto& writer : *active_writers) { + if (auto* sort_writer = dynamic_cast(writer.get())) { + revocable_size += sort_writer->data_size(); + } } - - return sort_writer->data_size(); + return revocable_size; } Status SpillIcebergTableSinkLocalState::revoke_memory(RuntimeState* state) { @@ -84,20 +119,25 @@ Status SpillIcebergTableSinkLocalState::revoke_memory(RuntimeState* state) { if (!_writer) { return Status::OK(); } - auto current_writer = _writer->current_writer(); - auto* sort_writer = dynamic_cast(current_writer.get()); - if (!sort_writer) { - return Status::OK(); + std::shared_ptr largest_writer; + size_t largest_size = 0; + // Retain the snapshot while the async writer may publish a replacement. + auto active_writers = _writer->active_writers(); + for (const auto& writer : *active_writers) { + if (auto* sort_writer = dynamic_cast(writer.get())) { + size_t size = sort_writer->data_size(); + if (size > largest_size) { + largest_size = size; + largest_writer = writer; + } + } } - - auto exception_catch_func = [current_writer, sort_writer]() { - auto status = [&]() { - RETURN_IF_CATCH_EXCEPTION({ return sort_writer->trigger_spill(); }); - }(); - return status; - }; - - return run_spill_task(state, exception_catch_func); + if (largest_writer != nullptr) { + // Drain one largest partition per revocation to avoid launching O(P) spill tasks. + auto* sort_writer = dynamic_cast(largest_writer.get()); + RETURN_IF_CATCH_EXCEPTION({ RETURN_IF_ERROR(sort_writer->trigger_spill()); }); + } + return Status::OK(); } SpillIcebergTableSinkOperatorX::SpillIcebergTableSinkOperatorX( @@ -127,9 +167,10 @@ Status SpillIcebergTableSinkOperatorX::sink_impl(RuntimeState* state, Block* in_ return local_state.sink(state, in_block, eos); } -size_t SpillIcebergTableSinkOperatorX::get_reserve_mem_size(RuntimeState* state, bool eos) { +size_t SpillIcebergTableSinkOperatorX::get_reserve_mem_size(RuntimeState* state, bool eos, + const Block* block) { auto& local_state = get_local_state(state); - return local_state.get_reserve_mem_size(state, eos); + return local_state.get_reserve_mem_size(state, eos, block); } size_t SpillIcebergTableSinkOperatorX::revocable_mem_size(RuntimeState* state) const { diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.h b/be/src/exec/operator/spill_iceberg_table_sink_operator.h index 7e6a037d2f55ed..c1c8e07798b720 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.h +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.h @@ -18,7 +18,9 @@ #pragma once #include +#include +#include "exec/operator/iceberg_sorter_reserve_memory.h" #include "exec/operator/operator.h" #include "exec/sink/writer/iceberg/viceberg_table_writer.h" @@ -42,7 +44,7 @@ class SpillIcebergTableSinkLocalState final Status open(RuntimeState* state) override; bool is_blockable() const override; - [[nodiscard]] size_t get_reserve_mem_size(RuntimeState* state, bool eos); + [[nodiscard]] size_t get_reserve_mem_size(RuntimeState* state, bool eos, const Block* block); Status revoke_memory(RuntimeState* state); size_t get_revocable_mem_size(RuntimeState* state) const; @@ -66,7 +68,7 @@ class SpillIcebergTableSinkOperatorX final Status sink_impl(RuntimeState* state, Block* in_block, bool eos) override; - size_t get_reserve_mem_size(RuntimeState* state, bool eos) override; + size_t get_reserve_mem_size(RuntimeState* state, bool eos, const Block* block) override; size_t revocable_mem_size(RuntimeState* state) const override; @@ -89,4 +91,4 @@ class SpillIcebergTableSinkOperatorX final }; #include "common/compile_check_end.h" -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 4226bf51b17436..7e16f1c92d3060 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -26,6 +26,7 @@ #include #include + // IWYU pragma: no_include #include #include @@ -117,6 +118,7 @@ #include "exec/operator/union_source_operator.h" #include "exec/pipeline/dependency.h" #include "exec/pipeline/pipeline_task.h" +#include "exec/pipeline/report_exec_status_size.h" #include "exec/pipeline/task_scheduler.h" #include "exec/runtime_filter/runtime_filter_mgr.h" #include "exec/sort/topn_sorter.h" @@ -443,6 +445,8 @@ Status PipelineFragmentContext::_build_pipeline_tasks_for_instance( _params.query_options, _query_ctx->query_globals, _exec_env, _query_ctx.get()); { // Initialize runtime state for this task + task_runtime_state->set_external_file_report_state( + _runtime_state->external_file_report_state()); task_runtime_state->set_query_mem_tracker(_query_ctx->query_mem_tracker()); task_runtime_state->set_task_execution_context(shared_from_this()); @@ -1990,6 +1994,15 @@ std::string PipelineFragmentContext::_to_http_path(const std::string& file_name) return url.str(); } +void PipelineFragmentContext::_append_external_file_commit_data( + const ReportStatusRequest& req, TReportExecStatusParams* params) const { + // External-file cleanup remains BE-owned until the final report transfers commit metadata. + req.runtime_state->append_external_file_commit_data(params, req.done); + for (auto* rs : req.runtime_states) { + rs->append_external_file_commit_data(params, req.done); + } +} + void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& req) { DBUG_EXECUTE_IF("FragmentMgr::coordinator_callback.report_delay", { int random_seconds = req.status.is() ? 8 : 2; @@ -2001,6 +2014,11 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r DCHECK(req.status.ok() || req.done); // if !status.ok() => done if (req.coord_addr.hostname == "external") { // External query (flink/spark read tablets) not need to report to FE. + if (req.done) { + // Without a coordinator acknowledgement no external-write file may escape rollback. + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); + } return; } int callback_retries = 10; @@ -2021,6 +2039,10 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r static_cast(req.cancel_fn(Status::InternalError( "query_id: {}, couldn't get a client for {}, reason is {}", uid.to_string(), PrintThriftNetworkAddress(req.coord_addr), coord_status.to_string()))); + if (req.done) { + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); + } return; } @@ -2144,59 +2166,7 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } } } - if (auto hpu = req.runtime_state->hive_partition_updates(); !hpu.empty()) { - params.__isset.hive_partition_updates = true; - params.hive_partition_updates.insert(params.hive_partition_updates.end(), hpu.begin(), - hpu.end()); - } else if (!req.runtime_states.empty()) { - for (auto* rs : req.runtime_states) { - if (auto rs_hpu = rs->hive_partition_updates(); !rs_hpu.empty()) { - params.__isset.hive_partition_updates = true; - params.hive_partition_updates.insert(params.hive_partition_updates.end(), - rs_hpu.begin(), rs_hpu.end()); - } - } - } - if (auto icd = req.runtime_state->iceberg_commit_datas(); !icd.empty()) { - params.__isset.iceberg_commit_datas = true; - params.iceberg_commit_datas.insert(params.iceberg_commit_datas.end(), icd.begin(), - icd.end()); - } else if (!req.runtime_states.empty()) { - for (auto* rs : req.runtime_states) { - if (auto rs_icd = rs->iceberg_commit_datas(); !rs_icd.empty()) { - params.__isset.iceberg_commit_datas = true; - params.iceberg_commit_datas.insert(params.iceberg_commit_datas.end(), - rs_icd.begin(), rs_icd.end()); - } - } - } - - if (auto mcd = req.runtime_state->mc_commit_datas(); !mcd.empty()) { - params.__isset.mc_commit_datas = true; - params.mc_commit_datas.insert(params.mc_commit_datas.end(), mcd.begin(), mcd.end()); - } else if (!req.runtime_states.empty()) { - for (auto* rs : req.runtime_states) { - if (auto rs_mcd = rs->mc_commit_datas(); !rs_mcd.empty()) { - params.__isset.mc_commit_datas = true; - params.mc_commit_datas.insert(params.mc_commit_datas.end(), rs_mcd.begin(), - rs_mcd.end()); - } - } - } - - if (auto pcm = req.runtime_state->paimon_commit_messages(); !pcm.empty()) { - params.__isset.paimon_commit_messages = true; - params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), pcm.begin(), - pcm.end()); - } else if (!req.runtime_states.empty()) { - for (auto* rs : req.runtime_states) { - if (auto rs_pcm = rs->paimon_commit_messages(); !rs_pcm.empty()) { - params.__isset.paimon_commit_messages = true; - params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), - rs_pcm.begin(), rs_pcm.end()); - } - } - } + _append_external_file_commit_data(req, ¶ms); req.runtime_state->get_unreported_errors(&(params.error_log)); params.__isset.error_log = (!params.error_log.empty()); @@ -2205,8 +2175,20 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r params.__set_backend_id(_exec_env->cluster_info()->backend_id); } + Status report_size_status = validate_report_exec_status_size( + params, req.runtime_state->coordinator_thrift_message_limit()); + if (!report_size_status.ok()) { + if (req.done) { + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); + } + req.cancel_fn(report_size_status); + return; + } + TReportExecStatusResult res; Status rpc_status; + bool report_outcome_ambiguous = false; VLOG_DEBUG << "reportExecStatus params is " << apache::thrift::ThriftDebugString(params).c_str(); @@ -2235,14 +2217,40 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r rpc_status = Status::create(res.status); } catch (apache::thrift::TException& e) { + report_outcome_ambiguous = true; rpc_status = Status::InternalError("ReportExecStatus() to {} failed: {}", PrintThriftNetworkAddress(req.coord_addr), e.what()); } + // Only Iceberg keeps BE rollback callbacks after close; the other vectors remain compatible + // with coordinators that acknowledge acceptance through the RPC status alone. + const bool requires_external_file_ack = params.__isset.iceberg_commit_datas; + if (rpc_status.ok() && requires_external_file_ack && + (!res.__isset.external_file_commit_data_accepted || + !res.external_file_commit_data_accepted)) { + rpc_status = Status::InternalError( + "Coordinator did not accept ownership of the external-file report"); + } + if (!rpc_status.ok()) { + if (req.done && !report_outcome_ambiguous) { + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); + } else if (req.done) { + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::AMBIGUOUS); + } LOG_INFO("Going to cancel query {} since report exec status got rpc failed: {}", print_id(req.query_id), rpc_status.to_string()); req.cancel_fn(rpc_status); + } else if (req.done && req.status.ok()) { + // Files remain rollback-owned until the coordinator has acknowledged the final metadata report. + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::ACKNOWLEDGED); + } else if (req.done) { + // An acknowledged error report confirms that FE will not publish this write's files. + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); } } @@ -2289,13 +2297,20 @@ Status PipelineFragmentContext::send_report(bool done) { .first_error_msg = first_error_msg, .cancel_fn = [this](const Status& reason) { cancel(reason); }}; auto ctx = std::dynamic_pointer_cast(shared_from_this()); - return _exec_env->fragment_mgr()->get_thread_pool()->submit_func([this, req, ctx]() { - SCOPED_ATTACH_TASK(ctx->get_query_ctx()->query_mem_tracker()); - _coordinator_callback(req); - if (!req.done) { - ctx->refresh_next_report_time(); - } - }); + Status submit_status = + _exec_env->fragment_mgr()->get_thread_pool()->submit_func([this, req, ctx]() { + SCOPED_ATTACH_TASK(ctx->get_query_ctx()->query_mem_tracker()); + _coordinator_callback(req); + if (!req.done) { + ctx->refresh_next_report_time(); + } + }); + if (!submit_status.ok() && req.done) { + // A rejected final callback can never transfer ownership to the coordinator. + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::REJECTED); + } + return submit_status; } size_t PipelineFragmentContext::get_revocable_size(bool* has_running_task) const { diff --git a/be/src/exec/pipeline/pipeline_fragment_context.h b/be/src/exec/pipeline/pipeline_fragment_context.h index f8a1bfea229765..3ac90abe2c43bf 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.h +++ b/be/src/exec/pipeline/pipeline_fragment_context.h @@ -150,6 +150,8 @@ class PipelineFragmentContext : public TaskExecutionContext { private: void _coordinator_callback(const ReportStatusRequest& req); + void _append_external_file_commit_data(const ReportStatusRequest& req, + TReportExecStatusParams* params) const; std::string _to_http_path(const std::string& file_name) const; void _release_resource(); diff --git a/be/src/exec/pipeline/pipeline_task.cpp b/be/src/exec/pipeline/pipeline_task.cpp index c898024c91590d..26d2c407c150dc 100644 --- a/be/src/exec/pipeline/pipeline_task.cpp +++ b/be/src/exec/pipeline/pipeline_task.cpp @@ -658,7 +658,8 @@ Status PipelineTask::execute(bool* done) { ->task_controller() ->is_enable_reserve_memory() && workload_group && !(_wake_up_early || _dry_run)) { - const auto sink_reserve_size = _sink->get_reserve_mem_size(_state, _eos); + const auto sink_reserve_size = + _sink->get_reserve_mem_size(_state, _eos, _block.get()); if (sink_reserve_size > 0 && _should_trigger_revoking(sink_reserve_size)) { LOG(INFO) << fmt::format( diff --git a/be/src/exec/pipeline/report_exec_status_size.h b/be/src/exec/pipeline/report_exec_status_size.h new file mode 100644 index 00000000000000..b5920963bfbcd4 --- /dev/null +++ b/be/src/exec/pipeline/report_exec_status_size.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "util/thrift_util.h" + +namespace doris { + +inline Status validate_report_exec_status_size(const TReportExecStatusParams& params, + size_t thrift_limit) { + ThriftSerializer serializer(false, 256); + uint32_t serialized_size = 0; + uint8_t* buffer = nullptr; + RETURN_IF_ERROR(serializer.serialize(¶ms, &serialized_size, &buffer)); + // Include the args field header and RPC method/version/sequence envelope around the params. + constexpr size_t rpc_envelope_bytes = 64; + if (thrift_limit < rpc_envelope_bytes || serialized_size > thrift_limit - rpc_envelope_bytes) { + return Status::InternalError( + "ReportExecStatus exceeds the coordinator Thrift message limit"); + } + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index 172fdd28177c62..92d26b560f7160 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -34,6 +34,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" +#include "exec/sink/writer/iceberg/iceberg_writer_compatibility.h" #include "exprs/vexpr.h" #include "format/table/deletion_vector.h" #include "format/table/iceberg_delete_file_reader_helper.h" @@ -203,6 +204,8 @@ Status VIcebergDeleteSink::init_properties(ObjectPool* pool) { Status VIcebergDeleteSink::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; + RETURN_IF_ERROR(validate_iceberg_external_file_report_ack(state->query_options())); + // Initialize counters _written_rows_counter = ADD_COUNTER(profile, "RowsWritten", TUnit::UNIT); _send_data_timer = ADD_TIMER(profile, "SendDataTime"); @@ -283,13 +286,17 @@ Status VIcebergDeleteSink::close(Status close_status) { _delete_file_count); if (_state != nullptr) { - for (const auto& commit_data : _commit_data_list) { - _state->add_iceberg_commit_datas(commit_data); + for (auto& commit_data : _commit_data_list) { + Status report_status = _state->add_iceberg_commit_datas(std::move(commit_data)); + if (!report_status.ok()) { + _cleanup_created_files(); + return report_status; + } } } if (!_defer_file_cleanup_until_outer_close) { - _created_files.clear(); + _transfer_created_files_to_report_cleanup(); } return Status::OK(); @@ -299,11 +306,24 @@ void VIcebergDeleteSink::finish_deferred_file_cleanup(Status outer_status) { if (!outer_status.ok()) { _cleanup_created_files(); } else { - _created_files.clear(); + _transfer_created_files_to_report_cleanup(); } _defer_file_cleanup_until_outer_close = false; } +void VIcebergDeleteSink::_transfer_created_files_to_report_cleanup() { + DCHECK(_state != nullptr); + for (auto& created_file : _created_files) { + _state->add_rejected_external_file_report_cleanup( + [cleanup_fs = std::move(created_file.first), + cleanup_path = std::move(created_file.second)] { + WARN_IF_ERROR(cleanup_fs->delete_file(cleanup_path), + "failed to delete an Iceberg delete file after report failure"); + }); + } + _created_files.clear(); +} + void VIcebergDeleteSink::_cleanup_created_files() { for (const auto& [fs, path] : _created_files) { Status delete_status = fs->delete_file(path); diff --git a/be/src/exec/sink/viceberg_delete_sink.h b/be/src/exec/sink/viceberg_delete_sink.h index 55698ae0404b14..625134ba3a3d51 100644 --- a/be/src/exec/sink/viceberg_delete_sink.h +++ b/be/src/exec/sink/viceberg_delete_sink.h @@ -134,6 +134,7 @@ class VIcebergDeleteSink final : public AsyncResultWriter { Status _init_position_delete_output_exprs(); std::string _get_file_extension() const; void _cleanup_created_files(); + void _transfer_created_files_to_report_cleanup(); TDataSink _t_sink; RuntimeState* _state = nullptr; diff --git a/be/src/exec/sink/writer/async_result_writer.cpp b/be/src/exec/sink/writer/async_result_writer.cpp index 00d274507712ab..a5346085d2af3c 100644 --- a/be/src/exec/sink/writer/async_result_writer.cpp +++ b/be/src/exec/sink/writer/async_result_writer.cpp @@ -45,6 +45,8 @@ Status AsyncResultWriter::sink(Block* block, bool eos) { add_block = _get_free_block(block, rows); } + // The pipeline reservation protects allocations performed after this block is dequeued. + auto reservation = thread_context()->thread_mem_tracker_mgr->take_reserved_memory(); std::lock_guard l(_m); // if io task failed, just return error status to // end the query @@ -56,9 +58,12 @@ Status AsyncResultWriter::sink(Block* block, bool eos) { if (_is_finished()) { _dependency->set_ready(); } - if (rows) { - _memory_used_counter->update(add_block->allocated_bytes()); - _data_queue.emplace_back(std::move(add_block)); + if (rows || eos) { + if (rows) { + _memory_used_counter->update(add_block->allocated_bytes()); + } + _data_queue.emplace_back(QueuedBlock { + .block = std::move(add_block), .reservation = std::move(reservation), .eos = eos}); if (!_data_queue_is_available() && !_is_finished()) { _dependency->block(); } @@ -72,17 +77,31 @@ Status AsyncResultWriter::sink(Block* block, bool eos) { return Status::OK(); } -std::unique_ptr AsyncResultWriter::_get_block_from_queue() { +AsyncResultWriter::QueuedBlock AsyncResultWriter::_get_block_from_queue() { std::lock_guard l(_m); DCHECK(!_data_queue.empty()); - auto block = std::move(_data_queue.front()); + auto queued = std::move(_data_queue.front()); _data_queue.pop_front(); + _queue_admission.begin_processing(); DCHECK(_dependency); if (_data_queue_is_available()) { _dependency->set_ready(); } - _memory_used_counter->update(-block->allocated_bytes()); - return block; + if (queued.block) { + _memory_used_counter->update(-queued.block->allocated_bytes()); + } + return queued; +} + +void AsyncResultWriter::_notify_block_processed() { + if (!_queue_admission.waits_for_processing()) { + return; + } + std::lock_guard l(_m); + _queue_admission.finish_processing(); + if (_data_queue_is_available()) { + _dependency->set_ready(); + } } Status AsyncResultWriter::start_writer(RuntimeState* state, RuntimeProfile* operator_profile) { @@ -132,6 +151,12 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera } DCHECK(_dependency); + bool reservation_held_for_finalize = false; + Defer release_final_reservation {[&]() { + if (reservation_held_for_finalize) { + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); + } + }}; while (_writer_status.ok()) { ThreadCpuStopWatch cpu_time_stop_watch; cpu_time_stop_watch.start(); @@ -160,24 +185,49 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* opera //check if eos or writer error if ((_eos && _data_queue.empty()) || !_writer_status.ok()) { - _data_queue.clear(); break; } } //2) get the block from data queue and write to downstream - auto block = _get_block_from_queue(); - auto status = write(state, *block); + auto queued = _get_block_from_queue(); + thread_context()->thread_mem_tracker_mgr->adopt_reserved_memory( + std::move(queued.reservation)); + Status status = queued.block ? write(state, *queued.block) : Status::OK(); if (!status.ok()) [[unlikely]] { + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); std::unique_lock l(_m); + _queue_admission.finish_processing(); _writer_status.update(status); - if (_is_finished()) { + if (_is_finished() || _data_queue_is_available()) { _dependency->set_ready(); } break; } - _return_free_block(std::move(block)); + if (queued.block) { + _return_free_block(std::move(queued.block)); + } + if (queued.eos) { + // Some writers finalize buffered data in close(), so the EOS reservation must outlive + // both finish() and close() instead of being released between the two callbacks. + reservation_held_for_finalize = true; + _notify_block_processed(); + break; + } + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); + _notify_block_processed(); + } + + { + std::lock_guard l(_m); + drain_async_writer_queue(_data_queue, [this](const QueuedBlock& queued) { + if (queued.block) { + _memory_used_counter->update(-queued.block->allocated_bytes()); + } + }); + _queue_admission.finish_processing(); + _dependency->set_ready(); } bool need_finish = false; diff --git a/be/src/exec/sink/writer/async_result_writer.h b/be/src/exec/sink/writer/async_result_writer.h index 99d4f8eaa59eff..fe851a9171aae4 100644 --- a/be/src/exec/sink/writer/async_result_writer.h +++ b/be/src/exec/sink/writer/async_result_writer.h @@ -21,8 +21,10 @@ #include #include // IWYU pragma: keep +#include "exec/sink/writer/async_writer_queue_admission.h" #include "exec/sink/writer/result_writer.h" #include "exprs/vexpr_fwd.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" #include "runtime/runtime_profile.h" namespace doris { @@ -36,6 +38,7 @@ class Dependency; class PipelineTask; class Block; + /* * In the pipeline execution engine, there are usually a large number of io operations on the sink side that * will block the limited execution threads of the pipeline execution engine, resulting in a sharp performance @@ -69,6 +72,10 @@ class AsyncResultWriter : public ResultWriter { void set_low_memory_mode(); + void wait_for_processing_before_next_sink() { + _queue_admission.wait_for_processing_before_next_sink(); + } + protected: Status _projection_block(Block& input_block, Block* output_block); const VExprContextSPtrs& _vec_output_expr_ctxs; @@ -77,21 +84,30 @@ class AsyncResultWriter : public ResultWriter { std::unique_ptr _get_free_block(Block*, size_t rows); private: + struct QueuedBlock { + std::unique_ptr block; + ReservedMemoryToken reservation; + bool eos = false; + }; + void process_block(RuntimeState* state, RuntimeProfile* operator_profile); - [[nodiscard]] bool _data_queue_is_available() const { return _data_queue.size() < QUEUE_SIZE; } + [[nodiscard]] bool _data_queue_is_available() const { + return _queue_admission.is_available(_data_queue.size()); + } [[nodiscard]] bool _is_finished() const { return !_writer_status.ok() || _eos; } void _set_ready_to_finish(); void _return_free_block(std::unique_ptr); - std::unique_ptr _get_block_from_queue(); + QueuedBlock _get_block_from_queue(); + void _notify_block_processed(); - static constexpr auto QUEUE_SIZE = 3; std::mutex _m; std::condition_variable _cv; - std::deque> _data_queue; + std::deque _data_queue; // Default value is ok AtomicStatus _writer_status; bool _eos = false; + AsyncWriterQueueAdmission _queue_admission; std::atomic_bool _low_memory_mode = false; std::shared_ptr _dependency; diff --git a/be/src/exec/sink/writer/async_writer_queue_admission.h b/be/src/exec/sink/writer/async_writer_queue_admission.h new file mode 100644 index 00000000000000..b5a73cb72aa881 --- /dev/null +++ b/be/src/exec/sink/writer/async_writer_queue_admission.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris { + +inline constexpr size_t ASYNC_WRITER_QUEUE_SIZE = 3; + +class AsyncWriterQueueAdmission { +public: + void wait_for_processing_before_next_sink() { _wait_for_processing = true; } + void begin_processing() { _block_being_processed = _wait_for_processing; } + void finish_processing() { _block_being_processed = false; } + + [[nodiscard]] bool is_available(size_t queued_blocks) const { + return _wait_for_processing ? queued_blocks == 0 && !_block_being_processed + : queued_blocks < ASYNC_WRITER_QUEUE_SIZE; + } + + [[nodiscard]] bool waits_for_processing() const { return _wait_for_processing; } + +private: + bool _block_being_processed = false; + bool _wait_for_processing = false; +}; + +template +void drain_async_writer_queue(Queue& queue, BeforeRelease before_release) { + for (const auto& queued : queue) { + before_release(queued); + } + // Queued reservation tokens must be destroyed as soon as the writer reaches a terminal state. + queue.clear(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/hive_multipart_compatibility.h b/be/src/exec/sink/writer/hive_multipart_compatibility.h new file mode 100644 index 00000000000000..c7546f314bfa8b --- /dev/null +++ b/be/src/exec/sink/writer/hive_multipart_compatibility.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "io/fs/obj_storage_client.h" + +namespace doris { + +inline bool hive_multipart_protocol_supported(io::ObjStorageType provider, + bool supports_deferred_azure_multipart) { + return provider != io::ObjStorageType::AZURE || supports_deferred_azure_multipart; +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h b/be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h new file mode 100644 index 00000000000000..ae06f4847a4c82 --- /dev/null +++ b/be/src/exec/sink/writer/iceberg/iceberg_writer_compatibility.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "gen_cpp/PaloInternalService_types.h" + +namespace doris { + +inline Status validate_iceberg_external_file_report_ack(const TQueryOptions& query_options) { + if (!query_options.__isset.supports_external_file_report_ack || + !query_options.supports_external_file_report_ack) { + // A pre-ACK coordinator cannot safely take ownership of files created by this sink. + return Status::NotSupported( + "Iceberg writes require a coordinator that acknowledges external-file reports"); + } + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index ba7644daec751f..0d4653400e6530 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -69,6 +69,7 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false}; RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options)); + Status open_status; switch (_file_format_type) { case TFileFormatType::FORMAT_PARQUET: { TParquetCompressionType::type parquet_compression_type; @@ -92,9 +93,13 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil break; } default: { - return Status::InternalError("Unsupported compress type {} with parquet", - to_string(_compress_type)); + open_status = Status::InternalError("Unsupported compress type {} with parquet", + to_string(_compress_type)); + break; + } } + if (!open_status.ok()) { + break; } ParquetFileOptions parquet_options = {.compression_type = parquet_compression_type, .parquet_version = TParquetVersion::PARQUET_1_0, @@ -103,19 +108,27 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil _file_format_transformer = std::make_unique( state, _file_writer.get(), _write_output_expr_ctxs, _write_column_names, false, parquet_options, _iceberg_schema_json, &_schema); - return _file_format_transformer->open(); + open_status = _file_format_transformer->open(); + break; } case TFileFormatType::FORMAT_ORC: { _file_format_transformer = std::make_unique( state, _file_writer.get(), _write_output_expr_ctxs, "", _write_column_names, false, _compress_type, &_schema, _fs); - return _file_format_transformer->open(); + open_status = _file_format_transformer->open(); + break; } default: { - return Status::InternalError("Unsupported file format type {}", - to_string(_file_format_type)); + open_status = Status::InternalError("Unsupported file format type {}", + to_string(_file_format_type)); + break; } } + if (!open_status.ok()) { + // A transformer failure happens after object creation, so remove any published file. + WARN_IF_ERROR(_fs->delete_file(_path), "failed to delete Iceberg file after open error"); + } + return open_status; } Status VIcebergPartitionWriter::close(const Status& status) { @@ -147,7 +160,12 @@ Status VIcebergPartitionWriter::close(const Status& status) { } return commit_status; } - _state->add_iceberg_commit_datas(commit_data); + Status report_status = _state->add_iceberg_commit_datas(std::move(commit_data)); + if (!report_status.ok()) { + // A closed object that cannot be reported can never be committed, so remove it immediately. + WARN_IF_ERROR(_fs->delete_file(_path), "failed to delete unreportable Iceberg file"); + return report_status; + } if (_closed_file_callback) { _closed_file_callback(_fs, _path); } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp index db453c511fa086..32f195366dc300 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp @@ -17,6 +17,7 @@ #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" +#include "exec/operator/iceberg_sorter_reserve_memory.h" #include "exec/spill/spill_file_manager.h" #include "exec/spill/spill_file_reader.h" #include "exec/spill/spill_file_writer.h" @@ -85,6 +86,44 @@ size_t VIcebergSortWriter::get_reserve_mem_size(RuntimeState* state, bool eos) c return _sorter == nullptr ? 0 : _sorter->get_reserve_mem_size(state, eos); } +SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components(RuntimeState* state, + bool eos) const { + std::lock_guard lock(_sorter_mutex); + if (_sorter == nullptr) { + return {}; + } + auto reservation = _sorter->get_reserve_mem_size_components(state, eos); + _include_spill_merge_reservation(state, eos, &reservation); + return reservation; +} + +SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components( + RuntimeState* state, bool eos, size_t incoming_rows, size_t incoming_bytes) const { + std::lock_guard lock(_sorter_mutex); + if (_sorter == nullptr) { + return {}; + } + auto reservation = + _sorter->get_reserve_mem_size_components(state, eos, incoming_rows, incoming_bytes); + _include_spill_merge_reservation(state, eos, &reservation); + return reservation; +} + +void VIcebergSortWriter::_include_spill_merge_reservation(RuntimeState* state, bool eos, + SorterReserveMemory* reservation) const { + if (eos && !_sorted_spill_files.empty()) { + size_t spill_file_count = _sorted_spill_files.size(); + if (_sorter->data_size() > 0) { + ++spill_file_count; + } + const size_t merge_workspace = + iceberg_spill_merge_workspace(spill_file_count, state->spill_buffer_size_bytes(), + state->spill_sort_merge_mem_limit_bytes()); + reservation->transient_workspace = + std::max(reservation->transient_workspace, merge_workspace); + } +} + Status VIcebergSortWriter::trigger_spill() { std::lock_guard lock(_sorter_mutex); if (_closed || _sorter == nullptr) { @@ -103,80 +142,35 @@ Status VIcebergSortWriter::close(const Status& status) { } Status VIcebergSortWriter::_close_locked(const Status& status) { - // Track the actual internal status of operations performed during close. - // This is important because if intermediate operations (like do_sort()) fail, - // we need to propagate the actual error status to the underlying partition writer's - // close() call, rather than the original status parameter which could be OK. - Status internal_status = Status::OK(); - // Track the close status of the underlying partition writer. - // If _iceberg_partition_writer->close() fails (e.g., Parquet file flush error), - // we must propagate this error to the caller to avoid silent data loss. - Status close_status = Status::OK(); - - // Defer ensures the underlying partition writer is always closed and - // spill streams are cleaned up, regardless of whether intermediate operations succeed. - // Uses internal_status to propagate any errors that occurred during close operations. - Defer defer {[&]() { - // If any intermediate operation failed, pass that error to the partition writer; - // otherwise, pass the original status from the caller. - close_status = - _iceberg_partition_writer->close(internal_status.ok() ? status : internal_status); - if (!close_status.ok()) { - LOG(WARNING) << fmt::format("_iceberg_partition_writer close failed, reason: {}", - close_status.to_string()); - } - _cleanup_spill_streams(); - }}; - - // If the original status is already an error or the query is cancelled, - // skip all close operations and propagate the original error - if (!status.ok() || _runtime_state->is_cancelled()) { - return status; - } - - // If sorter was never initialized (e.g., no data was written), nothing to do - if (_sorter == nullptr) { - return Status::OK(); - } - - // Check if there is any remaining data in the sorter (either unsorted or already sorted blocks) - if (!_sorter->merge_sort_state()->unsorted_block()->empty() || - !_sorter->merge_sort_state()->get_sorted_block().empty()) { - if (_sorted_spill_files.empty()) { - // No spill has occurred, all data is in memory. - // Sort the remaining data, prepare for reading, and write to file. - internal_status = _sorter->do_sort(); - if (!internal_status.ok()) { - return internal_status; - } - internal_status = _sorter->prepare_for_read(false); - if (!internal_status.ok()) { - return internal_status; + Status internal_status = status; + if (status.ok() && !_runtime_state->is_cancelled()) { + internal_status = Status::OK(); + if (_sorter != nullptr && (!_sorter->merge_sort_state()->unsorted_block()->empty() || + !_sorter->merge_sort_state()->get_sorted_block().empty())) { + if (_sorted_spill_files.empty()) { + internal_status = _sorter->do_sort(); + if (internal_status.ok()) { + internal_status = _sorter->prepare_for_read(false); + } + if (internal_status.ok()) { + internal_status = _write_sorted_data(); + } + } else { + internal_status = _do_spill(); } - internal_status = _write_sorted_data(); - return internal_status; } - - // Some data has already been spilled to disk. - // Spill the remaining in-memory data to a new spill stream. - internal_status = _do_spill(); - if (!internal_status.ok()) { - return internal_status; + if (internal_status.ok() && !_sorted_spill_files.empty()) { + internal_status = _combine_files_output(); } } - // Merge all spilled streams using multi-way merge sort and output final sorted data to files - if (!_sorted_spill_files.empty()) { - internal_status = _combine_files_output(); - if (!internal_status.ok()) { - return internal_status; - } + // Form the return value only after the underlying close runs; a deferred assignment is too late. + Status close_status = + _iceberg_partition_writer->close(internal_status.ok() ? status : internal_status); + _cleanup_spill_streams(); + if (!internal_status.ok()) { + return internal_status; } - - // Return close_status if internal operations succeeded but the underlying - // partition writer's close() failed (e.g., file flush error). - // This prevents silent data loss where the caller thinks the write succeeded - // but the file was not properly closed. return close_status; } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h index e1e512f0a0cf79..b41c31828431f1 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.h @@ -105,10 +105,19 @@ class VIcebergSortWriter : public IPartitionWriterBase { size_t get_reserve_mem_size(RuntimeState* state, bool eos) const; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos) const; + + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t incoming_rows, + size_t incoming_bytes) const; + // Called by the memory management system to trigger spilling data to disk Status trigger_spill(); private: + void _include_spill_merge_reservation(RuntimeState* state, bool eos, + SorterReserveMemory* reservation) const; + // Calculate average row size from the first non-empty block to determine // the optimal batch row count for spill operations void _update_spill_block_batch_row_count(const Block& block); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index b9eeeac38a7a30..44e996d1e9cefc 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -17,15 +17,20 @@ #include "exec/sink/writer/iceberg/viceberg_table_writer.h" +#include + #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/block/materialize_block.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" #include "core/data_type_serde/data_type_serde.h" #include "exec/sink/writer/iceberg/iceberg_partition_path.h" +#include "exec/sink/writer/iceberg/iceberg_writer_compatibility.h" #include "exec/sink/writer/iceberg/partition_transformers.h" #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" @@ -45,12 +50,15 @@ VIcebergTableWriter::VIcebergTableWriter(const TDataSink& t_sink, std::shared_ptr fin_dep) : AsyncResultWriter(output_expr_ctxs, dep, fin_dep), _t_sink(t_sink) { DCHECK(_t_sink.__isset.iceberg_table_sink); + _active_writers.store(std::make_shared()); } Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; _operator_profile = profile; + RETURN_IF_ERROR(validate_iceberg_external_file_report_ack(state->query_options())); + // Get target file size from query options // If value is 0 or not set, use config::iceberg_sink_max_file_size _target_file_size_bytes = config::iceberg_sink_max_file_size; @@ -121,24 +129,106 @@ std::vector VIcebergTableWriter::_to_iceberg_partition_columns() { std::vector partition_columns; - std::unordered_map id_to_column_idx; - id_to_column_idx.reserve(_schema->columns().size()); - for (int i = 0; i < _schema->columns().size(); i++) { - id_to_column_idx[_schema->columns()[i].field_id()] = i; - } for (const auto& partition_field : _partition_spec->fields()) { - int column_idx = id_to_column_idx[partition_field.source_id()]; + const auto* field_path = _schema->find_field_path(partition_field.source_id()); + if (field_path == nullptr || field_path->empty()) { + throw Exception( + ErrorCode::INTERNAL_ERROR, + "Iceberg partition field {} references source field {} outside writer schema", + partition_field.field_id(), partition_field.source_id()); + } + int column_idx = -1; + for (int i = 0; i < _schema->columns().size(); ++i) { + if (_schema->columns()[i].field_id() == field_path->front()->field_id()) { + column_idx = i; + break; + } + } + DORIS_CHECK(column_idx >= 0); + std::vector child_indices; + iceberg::Type* iceberg_type = field_path->front()->field_type(); + DataTypePtr source_type = _vec_output_expr_ctxs[column_idx]->root()->data_type(); + for (size_t depth = 1; depth < field_path->size(); ++depth) { + if (!iceberg_type->is_struct_type()) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "Iceberg partition source field {} has a non-struct ancestor", + partition_field.source_id()); + } + const auto& fields = iceberg_type->as_struct_type()->fields(); + auto child = std::find_if(fields.begin(), fields.end(), [&](const auto& candidate) { + return candidate.field_id() == (*field_path)[depth]->field_id(); + }); + DORIS_CHECK(child != fields.end()); + const size_t child_idx = std::distance(fields.begin(), child); + const auto* struct_type = + check_and_get_data_type(remove_nullable(source_type).get()); + if (struct_type == nullptr || child_idx >= struct_type->get_elements().size()) { + throw Exception( + ErrorCode::INTERNAL_ERROR, + "Iceberg nested partition source field {} does not match writer type", + partition_field.source_id()); + } + child_indices.push_back(child_idx); + iceberg_type = child->field_type(); + source_type = struct_type->get_element(child_idx); + } + if (!iceberg_type->is_primitive_type()) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "Iceberg partition source field {} is not primitive", + partition_field.source_id()); + } std::unique_ptr partition_column_transform = - PartitionColumnTransforms::create( - partition_field, _vec_output_expr_ctxs[column_idx]->root()->data_type()); + PartitionColumnTransforms::create(partition_field, source_type); partition_columns.emplace_back( - partition_field, - _vec_output_expr_ctxs[column_idx]->root()->data_type()->get_primitive_type(), - column_idx, std::move(partition_column_transform)); + partition_field, remove_nullable(source_type)->get_primitive_type(), column_idx, + std::move(child_indices), std::move(partition_column_transform)); } return partition_columns; } +ColumnWithTypeAndName VIcebergTableWriter::_nested_partition_source( + const Block& block, const IcebergPartitionColumn& partition_column) const { + ColumnWithTypeAndName source = block.get_by_position(partition_column.source_idx()); + if (partition_column.child_indices().empty()) { + return source; + } + ColumnPtr column = source.column->convert_to_full_column_if_const(); + DataTypePtr type = source.type; + auto combined_nulls = ColumnUInt8::create(block.rows(), 0); + bool nullable = false; + auto unwrap_nullable = [&]() { + if (const auto* nullable_column = check_and_get_column(column.get())) { + nullable = true; + const auto& nulls = nullable_column->get_null_map_data(); + auto& combined = combined_nulls->get_data(); + for (size_t row = 0; row < combined.size(); ++row) { + combined[row] |= nulls[row]; + } + column = nullable_column->get_nested_column_ptr(); + type = remove_nullable(type); + } + }; + for (size_t child_idx : partition_column.child_indices()) { + unwrap_nullable(); + const auto* struct_column = check_and_get_column(column.get()); + const auto* struct_type = check_and_get_data_type(type.get()); + if (struct_column == nullptr || struct_type == nullptr || + child_idx >= struct_column->tuple_size()) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "Iceberg nested partition source does not match writer block"); + } + column = struct_column->get_column_ptr(child_idx); + type = struct_type->get_element(child_idx); + } + // Parent NULL masks the leaf even when the nested storage column contains a materialized value. + unwrap_nullable(); + if (nullable) { + column = ColumnNullable::create(column, std::move(combined_nulls)); + type = make_nullable(type); + } + return {std::move(column), std::move(type), source.name}; +} + void VIcebergTableWriter::_init_static_partition_values() { auto& iceberg_sink = _t_sink.iceberg_table_sink; if (!iceberg_sink.__isset.static_partition_values || @@ -243,7 +333,8 @@ Status VIcebergTableWriter::_process_row_lineage_columns(Block& block) { Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { RETURN_IF_ERROR(_process_row_lineage_columns(output_block)); - std::unordered_map, IColumn::Filter> writer_positions; + std::unordered_map, IColumn::Permutation> + writer_positions; _row_count += output_block.rows(); // Case 1: Full static partition - all data goes to a single partition @@ -260,6 +351,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } _partitions_to_writers.insert({_static_partition_path, writer}); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); + _publish_active_writers(); } else { if (writer_iter->second->written_len() > _target_file_size_bytes) { std::string file_name(writer_iter->second->file_name()); @@ -269,6 +361,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { RETURN_IF_ERROR(writer_iter->second->close(Status::OK())); } _partitions_to_writers.erase(writer_iter); + _publish_active_writers(); try { writer = _create_partition_writer(nullptr, -1, &file_name, file_name_index + 1); @@ -277,6 +370,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } _partitions_to_writers.insert({_static_partition_path, writer}); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); + _publish_active_writers(); } else { writer = writer_iter->second; } @@ -285,7 +379,6 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { SCOPED_RAW_TIMER(&_partition_writers_write_ns); output_block.erase(_non_write_columns_indices); RETURN_IF_ERROR(writer->write(output_block)); - _current_writer.store(writer); return Status::OK(); } @@ -303,6 +396,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } _partitions_to_writers.insert({"", writer}); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); + _publish_active_writers(); } else { if (writer_iter->second->written_len() > _target_file_size_bytes) { std::string file_name(writer_iter->second->file_name()); @@ -312,6 +406,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { RETURN_IF_ERROR(writer_iter->second->close(Status::OK())); } _partitions_to_writers.erase(writer_iter); + _publish_active_writers(); try { writer = _create_partition_writer(nullptr, -1, &file_name, file_name_index + 1); @@ -320,6 +415,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } _partitions_to_writers.insert({"", writer}); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); + _publish_active_writers(); } else { writer = writer_iter->second; } @@ -328,7 +424,6 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { SCOPED_RAW_TIMER(&_partition_writers_write_ns); output_block.erase(_non_write_columns_indices); RETURN_IF_ERROR(writer->write(output_block)); - _current_writer.store(writer); return Status::OK(); } @@ -351,9 +446,12 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { transformed_block.insert( {std::move(col), result_type, iceberg_partition_columns.field().name()}); } else { + Block source_block; + source_block.insert( + _nested_partition_source(output_block, iceberg_partition_columns)); transformed_block.insert( - iceberg_partition_columns.partition_column_transform().apply( - output_block, iceberg_partition_columns.source_idx())); + iceberg_partition_columns.partition_column_transform().apply(source_block, + 0)); } } for (int i = 0; i < output_block.rows(); ++i) { @@ -377,10 +475,8 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { auto writer = _create_partition_writer(&transformed_block, position, file_name, file_name_index); RETURN_IF_ERROR(writer->open(_state, _operator_profile, _row_desc)); - IColumn::Filter filter(output_block.rows(), 0); - filter[position] = 1; - writer_positions.insert({writer, std::move(filter)}); _partitions_to_writers.insert({partition_name, writer}); + _publish_active_writers(); writer_ptr = writer; } catch (doris::Exception& e) { return e.to_status(); @@ -389,8 +485,8 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { }; auto writer_iter = _partitions_to_writers.find(partition_name); + std::shared_ptr writer; if (writer_iter == _partitions_to_writers.end()) { - std::shared_ptr writer; if (_partitions_to_writers.size() + 1 > config::table_sink_partition_write_max_partition_nums_per_writer) { return Status::InternalError( @@ -399,7 +495,6 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } RETURN_IF_ERROR(create_and_open_writer(partition_name, i, nullptr, 0, writer)); } else { - std::shared_ptr writer; if (writer_iter->second->written_len() > _target_file_size_bytes) { std::string file_name(writer_iter->second->file_name()); int file_name_index = writer_iter->second->file_name_index(); @@ -409,53 +504,53 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } writer_positions.erase(writer_iter->second); _partitions_to_writers.erase(writer_iter); + _publish_active_writers(); RETURN_IF_ERROR(create_and_open_writer(partition_name, i, &file_name, file_name_index + 1, writer)); } else { writer = writer_iter->second; } - auto writer_pos_iter = writer_positions.find(writer); - if (writer_pos_iter == writer_positions.end()) { - IColumn::Filter filter(output_block.rows(), 0); - filter[i] = 1; - writer_positions.insert({writer, std::move(filter)}); - } else { - writer_pos_iter->second[i] = 1; - } + } + auto writer_pos_iter = writer_positions.find(writer); + if (writer_pos_iter == writer_positions.end()) { + IColumn::Permutation rows {static_cast(i)}; + writer_positions.insert({writer, std::move(rows)}); + } else { + writer_pos_iter->second.push_back(static_cast(i)); } } } SCOPED_RAW_TIMER(&_partition_writers_write_ns); output_block.erase(_non_write_columns_indices); for (auto it = writer_positions.begin(); it != writer_positions.end(); ++it) { - Block filtered_block; - RETURN_IF_ERROR(_filter_block(output_block, &it->second, &filtered_block)); - RETURN_IF_ERROR(it->first->write(filtered_block)); - _current_writer.store(it->first); + Block selected_block; + RETURN_IF_ERROR(_select_block(output_block, it->second, &selected_block)); + RETURN_IF_ERROR(it->first->write(selected_block)); } return Status::OK(); } -Status VIcebergTableWriter::_filter_block(doris::Block& block, const IColumn::Filter* filter, +Status VIcebergTableWriter::_select_block(doris::Block& block, const IColumn::Permutation& rows, doris::Block* output_block) { const ColumnsWithTypeAndName& columns_with_type_and_name = block.get_columns_with_type_and_name(); ColumnsWithTypeAndName result_columns; + result_columns.reserve(columns_with_type_and_name.size()); for (const auto& col : columns_with_type_and_name) { - result_columns.emplace_back(col.column->clone_resized(col.column->size()), col.type, - col.name); + // Across all partitions the permutations contain exactly one entry per input row, avoiding O(P*C*R). + result_columns.emplace_back(col.column->permute(rows, rows.size()), col.type, col.name); } *output_block = {std::move(result_columns)}; + return Status::OK(); +} - std::vector columns_to_filter; - int column_to_keep = output_block->columns(); - columns_to_filter.resize(column_to_keep); - for (uint32_t i = 0; i < column_to_keep; ++i) { - columns_to_filter[i] = i; +void VIcebergTableWriter::_publish_active_writers() { + auto snapshot = std::make_shared(); + snapshot->reserve(_partitions_to_writers.size()); + for (const auto& entry : _partitions_to_writers) { + snapshot->push_back(entry.second); } - - Block::filter_block_internal(output_block, columns_to_filter, *filter); - return Status::OK(); + _active_writers.store(std::move(snapshot)); } Status VIcebergTableWriter::close(Status status) { @@ -475,6 +570,7 @@ Status VIcebergTableWriter::close(Status status) { } } _partitions_to_writers.clear(); + _publish_active_writers(); } if (status.ok()) { SCOPED_TIMER(_operator_profile->total_time_counter()); @@ -490,7 +586,7 @@ Status VIcebergTableWriter::close(Status status) { if (!status.ok() || !result_status.ok()) { _cleanup_closed_files(); } else if (!_defer_file_cleanup_until_outer_close) { - _closed_files.clear(); + _transfer_closed_files_to_report_cleanup(); } return result_status; } @@ -501,11 +597,24 @@ void VIcebergTableWriter::finish_deferred_file_cleanup(Status outer_status) { if (!outer_status.ok()) { _cleanup_closed_files(); } else { - _closed_files.clear(); + _transfer_closed_files_to_report_cleanup(); } _defer_file_cleanup_until_outer_close = false; } +void VIcebergTableWriter::_transfer_closed_files_to_report_cleanup() { + DCHECK(_state != nullptr); + for (auto& closed_file : _closed_files) { + _state->add_rejected_external_file_report_cleanup( + [cleanup_fs = std::move(closed_file.first), + cleanup_path = std::move(closed_file.second)] { + WARN_IF_ERROR(cleanup_fs->delete_file(cleanup_path), + "failed to delete an Iceberg file after report failure"); + }); + } + _closed_files.clear(); +} + void VIcebergTableWriter::_cleanup_closed_files() { for (const auto& [fs, path] : _closed_files) { Status delete_status = fs->delete_file(path); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h index 070019d85db9be..7dbd703ef1fcfb 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h @@ -72,28 +72,27 @@ class VIcebergTableWriter final : public AsyncResultWriter { TIcebergWriteType::type write_type() const { return _write_type; } - // Getter for the current partition writer. - // Used by SpillIcebergTableSinkLocalState to access the current writer for - // memory management operations (get_reserve_mem_size, revocable_mem_size, etc.). - // Returns a snapshot by value: the async writer thread updates _current_writer - // concurrently with the spill/revoke path, so callers must hold their own copy - // while operating on it instead of dereferencing the underlying member directly. - std::shared_ptr current_writer() const { return _current_writer.load(); } + using ActiveWriterSnapshot = std::vector>; + std::shared_ptr active_writers() const { return _active_writers.load(); } private: - // The currently active partition writer (may be VIcebergPartitionWriter or VIcebergSortWriter). - // Updated during write() to track which writer received the most recent data. - // Wrapped in atomic_shared_ptr because revoke_memory / get_revocable_mem_size run on - // a different thread than the async writer that assigns to it. - doris::atomic_shared_ptr _current_writer; + FRIEND_TEST(VIcebergTableWriterTest, RejectMissingPartitionSource); + FRIEND_TEST(VIcebergTableWriterTest, ResolvesNestedPartitionSource); + // The lifecycle fixture verifies that the spill thread always observes a stable snapshot. + friend class VIcebergTableWriterLifecycleTest; + + // The spill thread needs every partition sorter while the async writer exclusively owns the map. + doris::atomic_shared_ptr _active_writers; class IcebergPartitionColumn { public: IcebergPartitionColumn(const iceberg::PartitionField& field, const PrimitiveType& source_type, int source_idx, + std::vector child_indices, std::unique_ptr partition_column_transform) : _field(field), _source_type(source_type), _source_idx(source_idx), + _child_indices(std::move(child_indices)), _partition_column_transform(std::move(partition_column_transform)) {} public: @@ -101,6 +100,7 @@ class VIcebergTableWriter final : public AsyncResultWriter { const PrimitiveType& source_type() const { return _source_type; } int source_idx() const { return _source_idx; } + const std::vector& child_indices() const { return _child_indices; } const PartitionColumnTransform& partition_column_transform() const { return *_partition_column_transform; @@ -114,10 +114,13 @@ class VIcebergTableWriter final : public AsyncResultWriter { const iceberg::PartitionField& _field; PrimitiveType _source_type; int _source_idx; + std::vector _child_indices; std::unique_ptr _partition_column_transform; }; std::vector _to_iceberg_partition_columns(); + ColumnWithTypeAndName _nested_partition_source( + const Block& block, const IcebergPartitionColumn& partition_column) const; std::string _partition_to_path(const doris::iceberg::StructLike& data); std::string _escape(const std::string& path); @@ -140,12 +143,14 @@ class VIcebergTableWriter final : public AsyncResultWriter { std::string _compute_file_name(); - Status _filter_block(doris::Block& block, const IColumn::Filter* filter, + Status _select_block(doris::Block& block, const IColumn::Permutation& rows, doris::Block* output_block); + void _publish_active_writers(); Status _write_prepared_block(Block& output_block); Status _process_row_lineage_columns(Block& block); void _cleanup_closed_files(); + void _transfer_closed_files_to_report_cleanup(); // Currently it is a copy, maybe it is better to use move semantics to eliminate it. TDataSink _t_sink; diff --git a/be/src/exec/sink/writer/vhive_partition_writer.cpp b/be/src/exec/sink/writer/vhive_partition_writer.cpp index 8331efac54bd47..40d7b38fc30236 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/exec/sink/writer/vhive_partition_writer.cpp @@ -21,10 +21,12 @@ #include "core/block/materialize_block.h" #include "core/column/column_map.h" +#include "exec/sink/writer/hive_multipart_compatibility.h" #include "format/transformer/vcsv_transformer.h" #include "format/transformer/vorc_transformer.h" #include "format/transformer/vparquet_transformer.h" #include "io/file_factory.h" +#include "io/fs/s3_file_system.h" #include "io/fs/s3_file_writer.h" #include "runtime/runtime_state.h" @@ -50,7 +52,10 @@ VHivePartitionWriter::VHivePartitionWriter(const TDataSink& t_sink, std::string _file_format_type(file_format_type), _hive_compress_type(hive_compress_type), _hive_serde_properties(hive_serde_properties), - _hadoop_conf(hadoop_conf) {} + _hadoop_conf(hadoop_conf), + _supports_deferred_azure_multipart( + t_sink.hive_table_sink.__isset.supports_deferred_azure_multipart && + t_sink.hive_table_sink.supports_deferred_azure_multipart) {} Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_profile) { _state = state; @@ -64,6 +69,16 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* operator_ .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()), .fs_name {}}; _fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description)); + if (auto* s3_fs = dynamic_cast(_fs.get()); + s3_fs != nullptr && + !hive_multipart_protocol_supported(s3_fs->client_holder()->s3_client_conf().provider, + _supports_deferred_azure_multipart)) { + // An old coordinator cannot publish namespaced Azure block IDs; lease expiry is not a + // compatibility fence, so reject before creating an upload that it could corrupt. + return Status::NotSupported( + "Azure Hive writes require a coordinator that supports deferred multipart " + "completion"); + } io::FileWriterOptions file_writer_options = {.used_by_s3_committer = true}; RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options)); @@ -209,7 +224,6 @@ void VHivePartitionWriter::_add_s3_mpu_pending_upload_for_rollback() { if (!_build_s3_mpu_pending_upload(&s3_mpu_pending_upload)) { return; } - THivePartitionUpdate hive_partition_update; hive_partition_update.__set_name(_partition_name); hive_partition_update.__set_update_mode(_update_mode); diff --git a/be/src/exec/sink/writer/vhive_partition_writer.h b/be/src/exec/sink/writer/vhive_partition_writer.h index 0b124108623fa1..92e316a95c8e10 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.h +++ b/be/src/exec/sink/writer/vhive_partition_writer.h @@ -101,6 +101,7 @@ class VHivePartitionWriter { TFileCompressType::type _hive_compress_type; const THiveSerDeProperties* _hive_serde_properties; const std::map& _hadoop_conf; + bool _supports_deferred_azure_multipart = false; std::shared_ptr _fs = nullptr; diff --git a/be/src/exec/sort/sorter.cpp b/be/src/exec/sort/sorter.cpp index 26deeef3a8539b..8a7f5809e8c1ae 100644 --- a/be/src/exec/sort/sorter.cpp +++ b/be/src/exec/sort/sorter.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,20 @@ #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" +namespace { + +size_t saturating_add_size(size_t lhs, size_t rhs) { + return std::min(std::numeric_limits::max() - lhs, rhs) + lhs; +} + +size_t saturating_multiply_size(size_t lhs, size_t rhs) { + return lhs == 0 || rhs <= std::numeric_limits::max() / lhs + ? lhs * rhs + : std::numeric_limits::max(); +} + +} // namespace + namespace doris { class RowDescriptor; } // namespace doris @@ -184,35 +199,62 @@ bool FullSorter::has_enough_capacity(Block* input_block, Block* unsorted_block) } size_t FullSorter::get_reserve_mem_size(RuntimeState* state, bool eos) const { - size_t size_to_reserve = 0; + return get_reserve_mem_size_components(state, eos).total(); +} + +SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, + bool eos) const { + const auto rows = _state->unsorted_block()->rows(); + const auto bytes = _state->unsorted_block()->bytes(); + const auto bytes_per_row = rows == 0 ? 0 : bytes / rows; + return get_reserve_mem_size_components( + state, eos, state->batch_size(), + saturating_multiply_size(bytes_per_row, state->batch_size())); +} + +SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t incoming_rows, + size_t incoming_bytes) const { + SorterReserveMemory reserve; const auto rows = _state->unsorted_block()->rows(); if (rows != 0) { const auto bytes = _state->unsorted_block()->bytes(); const auto allocated_bytes = _state->unsorted_block()->allocated_bytes(); - const auto bytes_per_row = bytes / rows; - const auto estimated_size_of_next_block = bytes_per_row * state->batch_size(); - auto new_block_bytes = estimated_size_of_next_block + bytes; - auto new_rows = rows + state->batch_size(); + auto new_block_bytes = saturating_add_size(bytes, incoming_bytes); + auto new_rows = saturating_add_size(rows, incoming_rows); // If the new size is greater than 85% of allocalted bytes, it maybe need to realloc. - if ((new_block_bytes * 100 / allocated_bytes) >= 85) { - size_to_reserve += (size_t)(allocated_bytes * 1.15); + const auto growth_threshold = static_cast( + (static_cast(allocated_bytes) * 85 + 99) / 100); + const size_t growth_trigger_bytes = growth_threshold > bytes ? growth_threshold - bytes : 0; + if (incoming_rows > 0 && growth_trigger_bytes <= incoming_bytes) { + reserve.retained_growth = static_cast(std::min( + (static_cast(allocated_bytes) * 115 + 99) / 100, + std::numeric_limits::max())); + reserve.retained_growth_trigger_bytes = growth_trigger_bytes; } - auto sort = new_rows > _buffered_block_size || new_block_bytes > _buffered_block_bytes; + // Iceberg close forces every nonempty pending run to sort at EOS, even when the generic + // append thresholds are not reached, so admission must cover that final allocation too. + auto sort = (eos && new_rows > 0) || new_rows > _buffered_block_size || + new_block_bytes > _buffered_block_bytes; if (sort) { - // new column is created when doing sort, reserve average size of one column - // for estimation - size_to_reserve += new_block_bytes / _state->unsorted_block()->columns(); + // sort_block keeps the source columns live while materializing a fully permuted destination. + reserve.transient_workspace = + saturating_add_size(reserve.transient_workspace, new_block_bytes); // helping data structures used during sorting - size_to_reserve += new_rows * sizeof(IColumn::Permutation::value_type); + reserve.transient_workspace = saturating_add_size( + reserve.transient_workspace, + saturating_multiply_size(new_rows, sizeof(IColumn::Permutation::value_type))); auto sort_columns_count = _ordering_expr_ctxs.size(); if (1 != sort_columns_count) { - size_to_reserve += new_rows * sizeof(EqualRangeIterator); + reserve.transient_workspace = saturating_add_size( + reserve.transient_workspace, + saturating_multiply_size(new_rows, sizeof(EqualRangeIterator))); } } } - return size_to_reserve; + return reserve; } Status FullSorter::append_block(Block* block) { diff --git a/be/src/exec/sort/sorter.h b/be/src/exec/sort/sorter.h index 8984b3e608d2be..5d3401e52aae21 100644 --- a/be/src/exec/sort/sorter.h +++ b/be/src/exec/sort/sorter.h @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -40,6 +41,19 @@ namespace doris { #include "common/compile_check_begin.h" + +struct SorterReserveMemory { + size_t retained_growth = 0; + size_t retained_growth_trigger_bytes = 0; + size_t transient_workspace = 0; + + size_t total() const { + return retained_growth > std::numeric_limits::max() - transient_workspace + ? std::numeric_limits::max() + : retained_growth + transient_workspace; + } +}; + class ObjectPool; class RowDescriptor; } // namespace doris @@ -196,6 +210,12 @@ class FullSorter final : public Sorter { size_t get_reserve_mem_size(RuntimeState* state, bool eos) const override; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos) const; + + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t incoming_rows, + size_t incoming_bytes) const; + Status merge_sort_read_for_spill(RuntimeState* state, doris::Block* block, int batch_size, bool* eos) override; void reset() override; diff --git a/be/src/format/table/iceberg/schema.cpp b/be/src/format/table/iceberg/schema.cpp index 5a1bdaf5c93451..25e2d20919a487 100644 --- a/be/src/format/table/iceberg/schema.cpp +++ b/be/src/format/table/iceberg/schema.cpp @@ -17,6 +17,8 @@ #include "format/table/iceberg/schema.h" +#include + namespace doris::iceberg { #include "common/compile_check_begin.h" @@ -25,10 +27,26 @@ const int Schema::DEFAULT_SCHEMA_ID = 0; Schema::Schema(int schema_id, std::vector columns) : _schema_id(schema_id), _root_struct(std::move(columns)) { - _id_to_field.reserve(_root_struct.fields().size()); + FieldPath path; + std::function index_field = [&](const NestedField& field) { + path.push_back(&field); + _id_to_field[field.field_id()] = &field; + _id_to_field_path[field.field_id()] = path; + Type* type = field.field_type(); + if (type->is_struct_type()) { + for (const auto& child : type->as_struct_type()->fields()) { + index_field(child); + } + } else if (type->is_list_type()) { + index_field(type->as_list_type()->element_field()); + } else if (type->is_map_type()) { + index_field(type->as_map_type()->key_field()); + index_field(type->as_map_type()->value_field()); + } + path.pop_back(); + }; for (const auto& field : _root_struct.fields()) { - int field_id = field.field_id(); - _id_to_field[field_id] = &field; + index_field(field); } } Schema::Schema(std::vector columns) : Schema(DEFAULT_SCHEMA_ID, std::move(columns)) {} @@ -49,5 +67,10 @@ const NestedField* Schema::find_field(int id) const { return nullptr; } +const Schema::FieldPath* Schema::find_field_path(int id) const { + auto it = _id_to_field_path.find(id); + return it == _id_to_field_path.end() ? nullptr : &it->second; +} + #include "common/compile_check_end.h" } // namespace doris::iceberg diff --git a/be/src/format/table/iceberg/schema.h b/be/src/format/table/iceberg/schema.h index 29ea62510ee2e4..70377a63ac0f9d 100644 --- a/be/src/format/table/iceberg/schema.h +++ b/be/src/format/table/iceberg/schema.h @@ -27,6 +27,7 @@ class StructType; class Schema { public: + using FieldPath = std::vector; Schema(int schema_id, std::vector columns); Schema(std::vector columns); @@ -41,6 +42,8 @@ class Schema { const NestedField* find_field(int id) const; + const FieldPath* find_field_path(int id) const; + private: static const char NEWLINE = '\n'; static const std::string ALL_COLUMNS; @@ -49,6 +52,7 @@ class Schema { int _schema_id; StructType _root_struct; std::unordered_map _id_to_field; + std::unordered_map _id_to_field_path; }; #include "common/compile_check_end.h" diff --git a/be/src/format/table/iceberg_default_value.h b/be/src/format/table/iceberg_default_value.h new file mode 100644 index 00000000000000..6667cb87318fff --- /dev/null +++ b/be/src/format/table/iceberg_default_value.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" + +namespace doris::iceberg::detail { + +inline bool parse_non_finite_default(doris::PrimitiveType type, std::string_view value, + doris::Field* result) { + DORIS_CHECK(result != nullptr); + if (type != TYPE_FLOAT && type != TYPE_DOUBLE) { + return false; + } + double parsed; + if (value == "NaN") { + parsed = std::numeric_limits::quiet_NaN(); + } else if (value == "Infinity") { + parsed = std::numeric_limits::infinity(); + } else if (value == "-Infinity") { + parsed = -std::numeric_limits::infinity(); + } else { + return false; + } + // Iceberg serializes non-finite defaults as strings, which generic numeric parsers reject. + *result = type == TYPE_FLOAT ? Field::create_field(static_cast(parsed)) + : Field::create_field(parsed); + return true; +} + +} // namespace doris::iceberg::detail diff --git a/be/src/format/table/iceberg_scan_semantics.h b/be/src/format/table/iceberg_scan_semantics.h index f579f063b76327..c708a3d6222585 100644 --- a/be/src/format/table/iceberg_scan_semantics.h +++ b/be/src/format/table/iceberg_scan_semantics.h @@ -22,6 +22,7 @@ namespace doris { inline constexpr int32_t ICEBERG_SCAN_SEMANTICS_VERSION_1 = 1; +inline constexpr int32_t ICEBERG_SCAN_SEMANTICS_VERSION_2 = 2; inline bool supports_iceberg_scan_semantics_v1(const TFileScanRangeParams* params) { // Old FE plans can carry IDs and encoded defaults too, so only this explicit version marker @@ -30,4 +31,9 @@ inline bool supports_iceberg_scan_semantics_v1(const TFileScanRangeParams* param params->iceberg_scan_semantics_version >= ICEBERG_SCAN_SEMANTICS_VERSION_1; } +inline bool supports_iceberg_scan_semantics_v2(const TFileScanRangeParams* params) { + return params != nullptr && params->__isset.iceberg_scan_semantics_version && + params->iceberg_scan_semantics_version >= ICEBERG_SCAN_SEMANTICS_VERSION_2; +} + } // namespace doris diff --git a/be/src/format/transformer/iceberg_partition_function.cpp b/be/src/format/transformer/iceberg_partition_function.cpp index 14eba85e28a88e..d22795122e50f0 100644 --- a/be/src/format/transformer/iceberg_partition_function.cpp +++ b/be/src/format/transformer/iceberg_partition_function.cpp @@ -23,6 +23,8 @@ #include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" #include "exec/sink/writer/iceberg/partition_transformers.h" #include "format/table/iceberg/partition_spec.h" @@ -88,6 +90,9 @@ Status IcebergInsertPartitionFunction::init(const std::vector& texprs) { insert_field.expr_ctx = std::move(ctx); insert_field.source_id = field.__isset.source_id ? field.source_id : 0; insert_field.name = field.__isset.name ? field.name : ""; + if (field.__isset.source_field_path) { + insert_field.source_field_path = field.source_field_path; + } _partition_fields.emplace_back(std::move(insert_field)); } } @@ -168,12 +173,60 @@ Status IcebergInsertPartitionFunction::clone(RuntimeState* state, field.expr_ctx = dst_field_ctxs[i]; field.source_id = _partition_fields[i].source_id; field.name = _partition_fields[i].name; + field.source_field_path = _partition_fields[i].source_field_path; new_function->_partition_fields.emplace_back(std::move(field)); } } return Status::OK(); } +Status IcebergInsertPartitionFunction::_nested_partition_source( + size_t rows, const InsertPartitionField& field, ColumnWithTypeAndName* source) const { + if (field.source_field_path.empty()) { + return Status::OK(); + } + ColumnPtr column = source->column->convert_to_full_column_if_const(); + DataTypePtr type = source->type; + ColumnUInt8::MutablePtr combined_nulls; + bool nullable = false; + auto unwrap_nullable = [&]() { + if (const auto* nullable_column = check_and_get_column(column.get())) { + nullable = true; + if (!combined_nulls) { + combined_nulls = ColumnUInt8::create(rows, 0); + } + const auto& nulls = nullable_column->get_null_map_data(); + auto& combined = combined_nulls->get_data(); + for (size_t row = 0; row < combined.size(); ++row) { + combined[row] |= nulls[row]; + } + column = nullable_column->get_nested_column_ptr(); + type = remove_nullable(type); + } + }; + for (int32_t child_index : field.source_field_path) { + unwrap_nullable(); + const auto* struct_column = check_and_get_column(column.get()); + const auto* struct_type = check_and_get_data_type(type.get()); + if (child_index < 0 || struct_column == nullptr || struct_type == nullptr || + static_cast(child_index) >= struct_column->tuple_size()) { + return Status::InternalError( + "Iceberg nested merge partition source does not match input block"); + } + column = struct_column->get_column_ptr(static_cast(child_index)); + type = struct_type->get_element(static_cast(child_index)); + } + // A nullable parent masks a materialized child value; exchange routing must match the writer's partition. + unwrap_nullable(); + if (nullable) { + column = ColumnNullable::create(column, std::move(combined_nulls)); + type = make_nullable(type); + } + std::string name = source->name; + *source = {std::move(column), std::move(type), std::move(name)}; + return Status::OK(); +} + Status IcebergInsertPartitionFunction::_compute_hashes_with_transform( Block* block, std::vector& partitions) const { const size_t rows = block->rows(); @@ -196,8 +249,13 @@ Status IcebergInsertPartitionFunction::_compute_hashes_with_transform( if (_partition_fields[i].transformer == nullptr) { return Status::InternalError("Merge partitioning transform is not initialized"); } + ColumnWithTypeAndName source = block->get_by_position(results[i]); + if (!_partition_fields[i].source_field_path.empty()) { + RETURN_IF_ERROR(_nested_partition_source(rows, _partition_fields[i], &source)); + } + Block source_block({source}); ColumnWithTypeAndName transformed = - _partition_fields[i].transformer->apply(*block, results[i]); + _partition_fields[i].transformer->apply(source_block, 0); const auto& [column, is_const] = unpack_if_const(transformed.column); if (is_const) { // A const column has the same value for all rows in this block, diff --git a/be/src/format/transformer/iceberg_partition_function.h b/be/src/format/transformer/iceberg_partition_function.h index d7698ed78b4eff..57f255991b0a99 100644 --- a/be/src/format/transformer/iceberg_partition_function.h +++ b/be/src/format/transformer/iceberg_partition_function.h @@ -23,6 +23,7 @@ #include #include +#include "core/block/column_with_type_and_name.h" #include "exec/partitioner/partitioner.h" #include "exec/sink/writer/iceberg/partition_transformers.h" @@ -51,8 +52,11 @@ class IcebergInsertPartitionFunction final : public PartitionFunction { std::unique_ptr transformer; int32_t source_id = 0; std::string name; + std::vector source_field_path; }; + Status _nested_partition_source(size_t rows, const InsertPartitionField& field, + ColumnWithTypeAndName* source) const; Status _compute_hashes_with_transform(Block* block, std::vector& partitions) const; Status _compute_hashes_with_exprs(Block* block, std::vector& partitions) const; Status _clone_expr_ctxs(RuntimeState* state, const VExprContextSPtrs& src, diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index 867fe2de06d0fd..f54628acf36170 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -281,6 +281,8 @@ struct ColumnDefinition { // that are absent from the query projection. std::optional initial_default_value = std::nullopt; bool initial_default_value_is_base64 = false; + // Iceberg uses explicit false to reject files missing a required field without a default. + std::optional is_optional = std::nullopt; // Partition columns are constants from split metadata and should not be matched against file // schema unless table-format logic explicitly asks for it. bool is_partition_key = false; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index d5d37614292db7..b59de94f9259a6 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -59,48 +59,6 @@ namespace doris::format { namespace { -Status build_initial_default_column(const ColumnDefinition& column, ColumnPtr* value) { - DORIS_CHECK(value != nullptr); - *value = nullptr; - if (!column.initial_default_value.has_value()) { - return Status::OK(); - } - const auto nested_type = remove_nullable(column.type); - Field parsed; - if (column.initial_default_value_is_base64 || - nested_type->get_primitive_type() == TYPE_VARBINARY) { - std::string decoded; - if (!base64_decode(*column.initial_default_value, &decoded)) { - return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", - column.name); - } - parsed = nested_type->get_primitive_type() == TYPE_VARBINARY - ? Field::create_field(StringView(decoded)) - : Field::create_field(decoded); - // Variable-width Fields borrow their input. Materialize while decoded is alive so the - // resulting column owns the payload before it crosses a mapping/literal boundary. - *value = column.type->create_column_const(1, parsed); - return Status::OK(); - } else { - RETURN_IF_ERROR( - nested_type->get_serde()->from_fe_string(*column.initial_default_value, parsed)); - } - *value = column.type->create_column_const(1, parsed); - return Status::OK(); -} - -Status build_initial_default_literal(const ColumnDefinition& column, VExprContextSPtr* literal) { - DORIS_CHECK(literal != nullptr); - ColumnPtr owned_value; - RETURN_IF_ERROR(build_initial_default_column(column, &owned_value)); - DORIS_CHECK(static_cast(owned_value)); - Field value; - owned_value->get(0, value); - // VLiteral copies the borrowed Field into its own column while owned_value is still alive. - *literal = VExprContext::create_shared(VLiteral::create_shared(column.type, value)); - return Status::OK(); -} - bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) { const auto& table_children = table.identity_children.empty() ? table.children : table.identity_children; @@ -397,6 +355,7 @@ static bool is_binary_comparison_predicate(const VExprSPtr& expr) { std::string TableColumnMapperOptions::debug_string() const { std::ostringstream out; out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) + << ", reject_missing_required_field=" << reject_missing_required_field << ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection << ", enable_row_lineage_virtual_columns=" << enable_row_lineage_virtual_columns << "}"; return out.str(); @@ -2204,15 +2163,15 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab // Doris internal Iceberg row locator is never a physical Iceberg data column. It is built // from file path, row position and partition metadata for delete/update/merge. mapping->virtual_column_type = TableVirtualColumnType::ICEBERG_ROWID; - } else if (table_column.initial_default_value.has_value()) { - VExprContextSPtr initial_default; - RETURN_IF_ERROR(build_initial_default_literal(table_column, &initial_default)); - // Iceberg metadata is the authoritative logical value for files written before the field - // existed; the generic FE expression may still contain its Base64 transport text. - _set_constant_mapping(mapping, std::move(initial_default)); } else if (table_column.default_expr != nullptr) { - // Missing schema-evolution column with an explicit default expression. + // The table-format reader supplies a typed literal so complex and binary defaults stay exact. _set_constant_mapping(mapping, table_column.default_expr); + } else if (table_column.initial_default_value.has_value()) { + return Status::InvalidArgument( + "Missing typed initial-default expression for table field '{}'", table_column.name); + } else if (_options.reject_missing_required_field && table_column.is_optional.has_value() && + !*table_column.is_optional) { + return Status::InvalidArgument("Missing required field: {}", table_column.name); } else { if (table_column.is_partition_key) { return Status::InvalidArgument( @@ -2770,10 +2729,24 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.file_type = table_child.type; child_mapping.variant_access_paths = table_child.variant_access_paths; child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; - // A missing nested field still has its Iceberg initial-default value in every row - // written before the field was added; carry it into recursive materialization. - RETURN_IF_ERROR(build_initial_default_column( - table_child, &child_mapping.initial_default_column)); + if (table_child.default_expr != nullptr) { + const auto* literal = + dynamic_cast(table_child.default_expr->root().get()); + if (literal == nullptr) { + return Status::InvalidArgument( + "Missing typed initial-default literal for table field '{}'", + table_child.name); + } + // Keep the literal's owning column alive for recursive materialization. + child_mapping.initial_default_column = literal->get_column_ptr(); + } else if (table_child.initial_default_value.has_value()) { + return Status::InvalidArgument( + "Missing typed initial-default expression for table field '{}'", + table_child.name); + } else if (_options.reject_missing_required_field && + table_child.is_optional.has_value() && !*table_child.is_optional) { + return Status::InvalidArgument("Missing required field: {}", table_child.name); + } mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 1c2b6acfadf89d..74e438e45d6caa 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -161,6 +161,8 @@ struct ColumnMapping { struct TableColumnMapperOptions { TableColumnMappingMode mode = TableColumnMappingMode::BY_FIELD_ID; + // Iceberg required fields cannot silently materialize NULL when the file predates the field. + bool reject_missing_required_field = false; bool allow_idless_complex_wrapper_projection = false; bool enable_row_lineage_virtual_columns = false; diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index b2b37c1bf09226..94f489747e95bd 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -35,6 +35,7 @@ #include "core/types.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/table/parquet_utils.h" +#include "format_v2/table/iceberg_reader.h" #include "format_v2/table/iceberg_schema_utils.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" @@ -147,6 +148,8 @@ class PositionDeleteFileTableReader final : public format::TableReader { void configure_mapper_options(format::TableColumnMapperOptions* options) const override { options->enable_row_lineage_virtual_columns = true; + // Position-delete row projection must reject a physically absent required field exactly like data scans. + options->reject_missing_required_field = supports_iceberg_scan_semantics_v2(_scan_params); // Parquet may preserve a selected complex wrapper without its own ID; position-delete row // projection must use the same descendant-ID fallback as ordinary Iceberg data scans. options->allow_idless_complex_wrapper_projection = @@ -591,6 +594,9 @@ Status IcebergPositionDeleteSysTableV2Reader::_build_delete_file_projected_colum columns->push_back(*it); columns->back().type = column.type; set_iceberg_delete_field_id(&columns->back()); + // The copied row tree bypasses IcebergTableReader::annotate_projected_column, so prepare its + // typed nested defaults before the generic inner reader builds the column mapper. + RETURN_IF_ERROR(prepare_iceberg_initial_default_exprs(&columns->back())); continue; } auto field = build_delete_file_column(column.name, column.type); diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 5289ba18651bab..443a803b9b9216 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -17,9 +17,16 @@ #include "format_v2/table/iceberg_reader.h" +#include +#include +#include +#include + #include +#include #include #include +#include #include #include "common/cast_set.h" @@ -31,6 +38,8 @@ #include "core/column/column_string.h" #include "core/column/column_struct.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" @@ -38,6 +47,7 @@ #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" +#include "format/table/iceberg_default_value.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/equality_delete_predicate.h" #include "format_v2/orc/orc_reader.h" @@ -46,6 +56,7 @@ #include "format_v2/table_reader.h" #include "io/file_factory.h" #include "util/debug_points.h" +#include "util/string_util.h" #include "util/url_coding.h" namespace doris::format::iceberg { @@ -81,12 +92,402 @@ static bool is_projected_iceberg_rowid(const format::ColumnDefinition& column) { return column.name == BeConsts::ICEBERG_ROWID_COL; } +static int iceberg_hex_value(char value) { + if (value >= '0' && value <= '9') { + return value - '0'; + } + if (value >= 'a' && value <= 'f') { + return value - 'a' + 10; + } + if (value >= 'A' && value <= 'F') { + return value - 'A' + 10; + } + return -1; +} + +static Status decode_iceberg_hex(std::string_view encoded, std::string* decoded) { + DORIS_CHECK(decoded != nullptr); + if ((encoded.size() & 1U) != 0) { + return Status::InvalidArgument("Invalid odd-length Iceberg binary default"); + } + decoded->resize(encoded.size() / 2); + for (size_t index = 0; index < encoded.size(); index += 2) { + const int high = iceberg_hex_value(encoded[index]); + const int low = iceberg_hex_value(encoded[index + 1]); + if (high < 0 || low < 0) { + return Status::InvalidArgument("Invalid hexadecimal Iceberg binary default"); + } + (*decoded)[index / 2] = static_cast((high << 4) | low); + } + return Status::OK(); +} + +static Status decode_iceberg_json_binary(std::string_view encoded, std::string* decoded) { + DORIS_CHECK(decoded != nullptr); + const bool is_uuid = encoded.size() == 36 && encoded[8] == '-' && encoded[13] == '-' && + encoded[18] == '-' && encoded[23] == '-'; + if (!is_uuid) { + return decode_iceberg_hex(encoded, decoded); + } + + std::string uuid_hex; + uuid_hex.reserve(32); + for (size_t index = 0; index < encoded.size(); ++index) { + if (index != 8 && index != 13 && index != 18 && index != 23) { + uuid_hex.push_back(encoded[index]); + } + } + return decode_iceberg_hex(uuid_hex, decoded); +} + +static std::string iceberg_json_scalar_text(const rapidjson::Value& value) { + if (value.IsString()) { + return {value.GetString(), value.GetStringLength()}; + } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return {buffer.GetString(), buffer.GetSize()}; +} + +static void normalize_iceberg_json_timestamp(PrimitiveType primitive_type, std::string* value) { + if (primitive_type != TYPE_DATETIME && primitive_type != TYPE_DATETIMEV2 && + primitive_type != TYPE_TIMESTAMPTZ) { + return; + } + if (const size_t separator = value->find('T'); separator != std::string::npos) { + (*value)[separator] = ' '; + } + if (primitive_type == TYPE_TIMESTAMPTZ) { + return; + } + if (value->ends_with('Z')) { + value->pop_back(); + return; + } + const size_t time_start = value->find(' '); + if (time_start == std::string::npos) { + return; + } + const size_t offset = value->find_first_of("+-", time_start + 1); + if (offset != std::string::npos) { + value->erase(offset); + } +} + +static Status build_v2_null_default(const format::ColumnDefinition& field, + const DataTypePtr& data_type, Field* result) { + DORIS_CHECK(data_type != nullptr); + DORIS_CHECK(result != nullptr); + if (field.is_optional.has_value() && !*field.is_optional) { + return Status::InvalidArgument("Required Iceberg field '{}' has a null default", + field.name); + } + if (!data_type->is_nullable()) { + return Status::InternalError( + "Optional Iceberg field '{}' has a null default, but its Doris type '{}' is not " + "nullable", + field.name, data_type->get_name()); + } + *result = Field(); + return Status::OK(); +} + +static const format::ColumnDefinition* find_v2_struct_child(const format::ColumnDefinition& field, + const std::string& name) { + const auto exact_child = std::ranges::find_if( + field.children, [&](const auto& candidate) { return iequal(candidate.name, name); }); + if (exact_child != field.children.end()) { + return &*exact_child; + } + const auto aliased_child = std::ranges::find_if(field.children, [&](const auto& candidate) { + return std::ranges::any_of(candidate.name_mapping, + [&](const auto& alias) { return iequal(alias, name); }); + }); + return aliased_child == field.children.end() ? nullptr : &*aliased_child; +} + +static Status build_v2_initial_default_field(const format::ColumnDefinition& field, + const DataTypePtr& data_type, + std::deque* binary_storage, + Field* result); + +static Status build_v2_json_default_field(const format::ColumnDefinition& field, + const DataTypePtr& data_type, + const rapidjson::Value& json_value, + std::deque* binary_storage, Field* result); + +static Status build_v2_json_struct_default(const format::ColumnDefinition& field, + const DataTypePtr& value_type, + const rapidjson::Value& json_value, + std::deque* binary_storage, Field* result) { + if (!json_value.IsObject()) { + return Status::InvalidArgument("Invalid Iceberg struct default for field '{}'", field.name); + } + + const auto& struct_type = assert_cast(*value_type); + Struct struct_value; + struct_value.reserve(struct_type.get_elements().size()); + for (size_t index = 0; index < struct_type.get_elements().size(); ++index) { + const auto* child = find_v2_struct_child(field, struct_type.get_element_name(index)); + if (child == nullptr || !child->has_identifier_field_id()) { + return Status::InvalidArgument( + "Iceberg struct default for field '{}' has incomplete child metadata", + field.name); + } + + const std::string child_id = std::to_string(child->get_identifier_field_id()); + const auto member = json_value.FindMember(child_id.c_str()); + Field child_value; + if (member == json_value.MemberEnd()) { + RETURN_IF_ERROR(build_v2_initial_default_field(*child, struct_type.get_element(index), + binary_storage, &child_value)); + } else { + RETURN_IF_ERROR(build_v2_json_default_field(*child, struct_type.get_element(index), + member->value, binary_storage, + &child_value)); + } + struct_value.push_back(std::move(child_value)); + } + *result = Field::create_field(std::move(struct_value)); + return Status::OK(); +} + +// The child ColumnDefinition, recursively transported from the item TField, describes the element +// schema and its field-level default metadata. It cannot represent a particular list literal's +// length or per-position values, so the parent initial-default keeps those values in Iceberg's +// single-value JSON array. +static Status build_v2_json_array_default(const format::ColumnDefinition& field, + const DataTypePtr& value_type, + const rapidjson::Value& json_value, + std::deque* binary_storage, Field* result) { + if (!json_value.IsArray() || field.children.size() != 1) { + return Status::InvalidArgument("Invalid Iceberg list default for field '{}'", field.name); + } + + const auto& array_type = assert_cast(*value_type); + Array array_value; + array_value.reserve(json_value.Size()); + for (const auto& json_element : json_value.GetArray()) { + Field element_value; + RETURN_IF_ERROR(build_v2_json_default_field(field.children.front(), + array_type.get_nested_type(), json_element, + binary_storage, &element_value)); + array_value.push_back(std::move(element_value)); + } + *result = Field::create_field(std::move(array_value)); + return Status::OK(); +} + +// The child ColumnDefinitions, recursively transported from the key/value TFields, describe entry +// schemas and field-level default metadata. They cannot represent the number, order, or concrete +// values of map entries, so the parent initial-default keeps the entries in Iceberg's single-value +// JSON key/value arrays. +static Status build_v2_json_map_default(const format::ColumnDefinition& field, + const DataTypePtr& value_type, + const rapidjson::Value& json_value, + std::deque* binary_storage, Field* result) { + if (!json_value.IsObject() || !json_value.HasMember("keys") || !json_value["keys"].IsArray() || + !json_value.HasMember("values") || !json_value["values"].IsArray() || + field.children.size() != 2) { + return Status::InvalidArgument("Invalid Iceberg map default for field '{}'", field.name); + } + const auto& keys = json_value["keys"]; + const auto& values = json_value["values"]; + if (keys.Size() != values.Size()) { + return Status::InvalidArgument( + "Iceberg map default for field '{}' has {} keys but {} values", field.name, + keys.Size(), values.Size()); + } + + const auto& map_type = assert_cast(*value_type); + Array key_fields; + Array value_fields; + key_fields.reserve(keys.Size()); + value_fields.reserve(values.Size()); + for (rapidjson::SizeType index = 0; index < keys.Size(); ++index) { + Field key_value; + Field mapped_value; + RETURN_IF_ERROR(build_v2_json_default_field(field.children[0], map_type.get_key_type(), + keys[index], binary_storage, &key_value)); + RETURN_IF_ERROR(build_v2_json_default_field(field.children[1], map_type.get_value_type(), + values[index], binary_storage, &mapped_value)); + key_fields.push_back(std::move(key_value)); + value_fields.push_back(std::move(mapped_value)); + } + Map map_value; + map_value.push_back(Field::create_field(std::move(key_fields))); + map_value.push_back(Field::create_field(std::move(value_fields))); + *result = Field::create_field(std::move(map_value)); + return Status::OK(); +} + +static Status build_v2_json_scalar_default(const format::ColumnDefinition& field, + const DataTypePtr& value_type, + const rapidjson::Value& json_value, + std::deque* binary_storage, Field* result) { + const auto primitive_type = value_type->get_primitive_type(); + std::string serialized_value = iceberg_json_scalar_text(json_value); + const bool binary_like = + field.initial_default_value_is_base64 || primitive_type == TYPE_VARBINARY; + if (binary_like) { + if (!json_value.IsString()) { + return Status::InvalidArgument( + "Iceberg binary default for field '{}' is not a JSON string", field.name); + } + binary_storage->emplace_back(); + RETURN_IF_ERROR(decode_iceberg_json_binary(serialized_value, &binary_storage->back())); + if (primitive_type == TYPE_VARBINARY) { + *result = Field::create_field(StringView(binary_storage->back())); + } else if (is_string_type(primitive_type)) { + *result = Field::create_field(binary_storage->back()); + } else { + return Status::InvalidArgument( + "Iceberg binary default for field '{}' has incompatible Doris type '{}'", + field.name, value_type->get_name()); + } + return Status::OK(); + } + + if (is_string_type(primitive_type)) { + if (!json_value.IsString()) { + return Status::InvalidArgument("Iceberg string default for field '{}' is not a string", + field.name); + } + *result = Field::create_field(std::move(serialized_value)); + return Status::OK(); + } + normalize_iceberg_json_timestamp(primitive_type, &serialized_value); + if (doris::iceberg::detail::parse_non_finite_default(primitive_type, serialized_value, + result)) { + return Status::OK(); + } + RETURN_IF_ERROR(value_type->get_serde()->from_fe_string(serialized_value, *result)); + return Status::OK(); +} + +static Status build_v2_json_default_field(const format::ColumnDefinition& field, + const DataTypePtr& data_type, + const rapidjson::Value& json_value, + std::deque* binary_storage, Field* result) { + DORIS_CHECK(data_type != nullptr); + DORIS_CHECK(binary_storage != nullptr); + DORIS_CHECK(result != nullptr); + if (json_value.IsNull()) { + return build_v2_null_default(field, data_type, result); + } + + const auto value_type = remove_nullable(data_type); + switch (value_type->get_primitive_type()) { + case TYPE_STRUCT: + return build_v2_json_struct_default(field, value_type, json_value, binary_storage, result); + case TYPE_ARRAY: + return build_v2_json_array_default(field, value_type, json_value, binary_storage, result); + case TYPE_MAP: + return build_v2_json_map_default(field, value_type, json_value, binary_storage, result); + default: + return build_v2_json_scalar_default(field, value_type, json_value, binary_storage, result); + } +} + +static Status build_v2_initial_default_field(const format::ColumnDefinition& field, + const DataTypePtr& data_type, + std::deque* binary_storage, + Field* result) { + DORIS_CHECK(data_type != nullptr); + DORIS_CHECK(binary_storage != nullptr); + DORIS_CHECK(result != nullptr); + if (!field.initial_default_value.has_value()) { + if (field.is_optional.has_value() && !*field.is_optional) { + return Status::InvalidArgument( + "Required Iceberg field '{}' is missing from the data file and has no initial " + "default", + field.name); + } + return build_v2_null_default(field, data_type, result); + } + + const auto value_type = remove_nullable(data_type); + const auto primitive_type = value_type->get_primitive_type(); + if (is_complex_type(primitive_type)) { + rapidjson::Document document; + document.Parse(field.initial_default_value->data(), field.initial_default_value->size()); + if (document.HasParseError()) { + return Status::InvalidArgument("Invalid Iceberg JSON initial default for field '{}'", + field.name); + } + return build_v2_json_default_field(field, data_type, document, binary_storage, result); + } + + if (field.initial_default_value_is_base64 || primitive_type == TYPE_VARBINARY) { + binary_storage->emplace_back(); + if (!base64_decode(*field.initial_default_value, &binary_storage->back())) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + field.name); + } + if (primitive_type == TYPE_VARBINARY) { + *result = Field::create_field(StringView(binary_storage->back())); + } else if (is_string_type(primitive_type)) { + *result = Field::create_field(binary_storage->back()); + } else { + return Status::InvalidArgument( + "Base64 Iceberg initial default has incompatible Doris type {} for field {}", + data_type->get_name(), field.name); + } + return Status::OK(); + } + + if (doris::iceberg::detail::parse_non_finite_default(primitive_type, + *field.initial_default_value, result)) { + return Status::OK(); + } + RETURN_IF_ERROR(value_type->get_serde()->from_fe_string(*field.initial_default_value, *result)); + return Status::OK(); +} + +static Status build_initial_default_literal(const format::ColumnDefinition& table_field, + VExprSPtr* literal) { + DORIS_CHECK(table_field.type != nullptr); + DORIS_CHECK(table_field.initial_default_value.has_value()); + DORIS_CHECK(literal != nullptr); + + std::deque binary_storage; + Field initial_default; + RETURN_IF_ERROR(build_v2_initial_default_field(table_field, table_field.type, &binary_storage, + &initial_default)); + // VLiteral inserts the Field into an owning column before binary_storage is destroyed. + *literal = VLiteral::create_shared(table_field.type, initial_default); + return Status::OK(); +} + +Status prepare_iceberg_initial_default_exprs(format::ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + if (column->initial_default_value.has_value()) { + VExprSPtr literal; + RETURN_IF_ERROR(build_initial_default_literal(*column, &literal)); + column->default_expr = VExprContext::create_shared(std::move(literal)); + } + for (auto& child : column->children) { + RETURN_IF_ERROR(prepare_iceberg_initial_default_exprs(&child)); + } + return Status::OK(); +} + static Status build_missing_equality_delete_key_expr(const format::ColumnDefinition& table_field, const DataTypePtr& delete_key_type, + bool require_complete_metadata, VExprSPtr* key_expr) { DORIS_CHECK(delete_key_type != nullptr); DORIS_CHECK(key_expr != nullptr); if (!table_field.initial_default_value.has_value()) { + if (require_complete_metadata && !table_field.is_optional.has_value()) { + return Status::InvalidArgument( + "Iceberg equality delete field '{}' is missing optionality metadata", + table_field.name); + } + if (table_field.is_optional.has_value() && !*table_field.is_optional) { + return Status::InvalidArgument("Missing required field: {}", table_field.name); + } // A newly added optional field without an initial default is logically NULL in older // files. EqualityDeletePredicate treats NULL == NULL as a match. *key_expr = VLiteral::create_shared(make_nullable(delete_key_type), Field()); @@ -94,48 +495,47 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit } VExprSPtr literal; - if (table_field.initial_default_value_is_base64 || - table_field.type->get_primitive_type() == TYPE_VARBINARY) { - // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its - // Doris mapping. Keep the VARBINARY fallback for scan descriptors produced before that - // marker existed. Decode before parsing so STRING/CHAR and VARBINARY all compare against - // the raw bytes stored in equality-delete files. - std::string decoded_default; - if (!base64_decode(*table_field.initial_default_value, &decoded_default)) { - return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", - table_field.name); - } - if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { - const auto initial_default = - Field::create_field(StringView(decoded_default)); - // VLiteral must copy the borrowed StringView while decoded_default is alive; UUID and - // long FIXED defaults otherwise retain a pointer into freed decode storage. - literal = VLiteral::create_shared(table_field.type, initial_default); - } else { - DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); - literal = VLiteral::create_shared(table_field.type, - Field::create_field(decoded_default)); - } - } else { - // An added field's initial default is its logical value in every older data file that lacks - // the physical column. FE normalizes the string for the current Doris table type. - Field initial_default; - RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( - *table_field.initial_default_value, initial_default)); - literal = VLiteral::create_shared(table_field.type, initial_default); - } - - DORIS_CHECK(literal != nullptr); + RETURN_IF_ERROR(build_initial_default_literal(table_field, &literal)); if (table_field.type->equals(*delete_key_type)) { *key_expr = std::move(literal); return Status::OK(); } auto cast_expr = Cast::create_shared(delete_key_type); - cast_expr->add_child(std::move(literal)); + cast_expr->add_child(literal); *key_expr = std::move(cast_expr); return Status::OK(); } +Status IcebergTableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info, + format::ProjectedColumnBuildContext* context, + format::ColumnDefinition* column) const { + RETURN_IF_ERROR(format::TableReader::annotate_projected_column(slot_info, context, column)); + DORIS_CHECK(context != nullptr); + DORIS_CHECK(column != nullptr); + if (!supports_iceberg_scan_semantics_v2(context->scan_params)) { + return Status::OK(); + } + if (!context->schema_column.has_value()) { + return Status::OK(); + } + + auto& schema_column = *context->schema_column; + RETURN_IF_ERROR(prepare_iceberg_initial_default_exprs(&schema_column)); + column->initial_default_value = schema_column.initial_default_value; + column->initial_default_value_is_base64 = schema_column.initial_default_value_is_base64; + column->is_optional = schema_column.is_optional; + if (schema_column.default_expr != nullptr) { + // The Iceberg typed literal is authoritative. In particular, this replaces FE's generic + // string expression for Base64-transported UUID/BINARY/FIXED defaults. + column->default_expr = schema_column.default_expr; + } else if (schema_column.is_optional.has_value() && !*schema_column.is_optional) { + // FE's generic external-column metadata currently treats Iceberg columns as nullable. Clear + // that fallback so a physically missing required field is rejected by the Iceberg mapper. + column->default_expr = nullptr; + } + return Status::OK(); +} + static std::string iceberg_delete_file_debug_string(const TIcebergDeleteFileDesc& delete_file) { std::ostringstream out; out << "TIcebergDeleteFileDesc{path=" << (delete_file.__isset.path ? delete_file.path : "null") @@ -607,7 +1007,8 @@ Status IcebergTableReader::_append_equality_delete_predicates(format::FileScanRe DORIS_CHECK(table_field.has_value()); VExprSPtr key_expr; RETURN_IF_ERROR(build_missing_equality_delete_key_expr( - *table_field, filter.key_types[idx], &key_expr)); + *table_field, filter.key_types[idx], + supports_iceberg_scan_semantics_v2(_scan_params), &key_expr)); delete_predicate->add_child(key_expr); has_missing_key = true; continue; diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index d28be3d7f98f0b..d1c27f738c2fa0 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -42,6 +42,8 @@ struct FileSystemProperties; namespace doris::format::iceberg { +Status prepare_iceberg_initial_default_exprs(format::ColumnDefinition* column); + // Iceberg table-level reader. // It reuses TableReader for split orchestration, dynamic partition pruning and table-block // finalization, while composing a FileReader for physical data-file reads instead of inheriting @@ -56,6 +58,9 @@ class IcebergTableReader : public format::TableReader { } Status prepare_split(const format::SplitReadOptions& options) override; + Status annotate_projected_column(const TFileScanSlotInfo& slot_info, + format::ProjectedColumnBuildContext* context, + format::ColumnDefinition* column) const override; std::string debug_string() const override; format::TableColumnMappingMode mapping_mode() const override { const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params) @@ -70,6 +75,7 @@ class IcebergTableReader : public format::TableReader { protected: void configure_mapper_options(format::TableColumnMapperOptions* options) const override { options->enable_row_lineage_virtual_columns = true; + options->reject_missing_required_field = supports_iceberg_scan_semantics_v2(_scan_params); options->allow_idless_complex_wrapper_projection = supports_iceberg_scan_semantics_v1(_scan_params) && _format == FileFormat::PARQUET; } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index bd7fe744110fdf..891dab799d14fb 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -329,6 +329,8 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: : std::nullopt, .initial_default_value_is_base64 = field.__isset.initial_default_value_is_base64 && field.initial_default_value_is_base64, + .is_optional = field.__isset.is_optional ? std::make_optional(field.is_optional) + : std::nullopt, .is_partition_key = false, }; if (column.type == nullptr || !field.__isset.nestedField) { @@ -652,6 +654,7 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info column->initial_default_value = context->schema_column->initial_default_value; column->initial_default_value_is_base64 = context->schema_column->initial_default_value_is_base64; + column->is_optional = context->schema_column->is_optional; return Status::OK(); } diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 3ae2077ecbc53d..e8b7225ad0b691 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1231,8 +1231,22 @@ class TableReader { if (const auto* array_type = typeid_cast(table_type.get())) { const auto& array_column = assert_cast(**column); ColumnPtr nested_column = array_column.get_data_ptr(); - RETURN_IF_ERROR( - _align_column_nullability(&nested_column, array_type->get_nested_type())); + NullMap descendant_parent_null_map; + // Collection entries use offset coordinates, so inherited row masks must be projected + // only when a required descendant can consume them. This avoids scratch proportional + // to all array entries for the common all-required schema. + const NullMap* descendant_parent_null_map_ptr = nullptr; + if (_requires_collection_parent_null_map( + nullable_parent_null_map, nested_column, array_type->get_nested_type(), + array_column.size(), array_column.get_offsets())) { + descendant_parent_null_map_ptr = + _project_collection_parent_null_map_for_hidden_entries( + nullptr, nullable_parent_null_map, array_column.size(), + array_column.get_offsets(), nested_column->size(), + &descendant_parent_null_map); + } + RETURN_IF_ERROR(_align_column_nullability(&nested_column, array_type->get_nested_type(), + descendant_parent_null_map_ptr)); *column = ColumnArray::create(nested_column, array_column.get_offsets_ptr()); return Status::OK(); } @@ -1240,8 +1254,25 @@ class TableReader { const auto& map_column = assert_cast(**column); ColumnPtr key_column = map_column.get_keys_ptr(); ColumnPtr value_column = map_column.get_values_ptr(); - RETURN_IF_ERROR(_align_column_nullability(&key_column, map_type->get_key_type())); - RETURN_IF_ERROR(_align_column_nullability(&value_column, map_type->get_value_type())); + NullMap descendant_parent_null_map; + const NullMap* descendant_parent_null_map_ptr = nullptr; + if (_requires_collection_parent_null_map(nullable_parent_null_map, key_column, + map_type->get_key_type(), map_column.size(), + map_column.get_offsets()) || + _requires_collection_parent_null_map(nullable_parent_null_map, value_column, + map_type->get_value_type(), map_column.size(), + map_column.get_offsets())) { + // Keys and values share offsets, so one projected mask safely covers both streams. + descendant_parent_null_map_ptr = + _project_collection_parent_null_map_for_hidden_entries( + nullptr, nullable_parent_null_map, map_column.size(), + map_column.get_offsets(), key_column->size(), + &descendant_parent_null_map); + } + RETURN_IF_ERROR(_align_column_nullability(&key_column, map_type->get_key_type(), + descendant_parent_null_map_ptr)); + RETURN_IF_ERROR(_align_column_nullability(&value_column, map_type->get_value_type(), + descendant_parent_null_map_ptr)); *column = ColumnMap::create(key_column, value_column, map_column.get_offsets_ptr()); return Status::OK(); } @@ -1506,16 +1537,195 @@ class TableReader { return column.get(); } + static bool _requires_parent_null_map_for_alignment(const ColumnPtr& column, + const DataTypePtr& table_type) { + DORIS_CHECK(column.get() != nullptr); + DORIS_CHECK(table_type != nullptr); + if (table_type->is_nullable()) { + const auto& nested_type = + assert_cast(*table_type).get_nested_type(); + if (const auto* nullable_column = check_and_get_column(*column)) { + return _requires_parent_null_map_for_alignment( + nullable_column->get_nested_column_ptr(), nested_type); + } + return _requires_parent_null_map_for_alignment(column, nested_type); + } + if (const auto* nullable_column = check_and_get_column(*column)) { + if (nullable_column->has_null()) { + return true; + } + return _requires_parent_null_map_for_alignment(nullable_column->get_nested_column_ptr(), + table_type); + } + if (const auto* array_type = typeid_cast(table_type.get())) { + const auto& array_column = assert_cast(*column); + return _requires_parent_null_map_for_alignment(array_column.get_data_ptr(), + array_type->get_nested_type()); + } + if (const auto* map_type = typeid_cast(table_type.get())) { + const auto& map_column = assert_cast(*column); + return _requires_parent_null_map_for_alignment(map_column.get_keys_ptr(), + map_type->get_key_type()) || + _requires_parent_null_map_for_alignment(map_column.get_values_ptr(), + map_type->get_value_type()); + } + if (const auto* struct_type = typeid_cast(table_type.get())) { + const auto& struct_column = assert_cast(*column); + DORIS_CHECK(struct_column.tuple_size() == struct_type->get_elements().size()); + for (size_t i = 0; i < struct_column.tuple_size(); ++i) { + if (_requires_parent_null_map_for_alignment(struct_column.get_column_ptr(i), + struct_type->get_element(i))) { + return true; + } + } + } + return false; + } + + static bool _requires_parent_null_map_for_alignment_at(const ColumnPtr& column, + const DataTypePtr& table_type, + const size_t row) { + DORIS_CHECK(column.get() != nullptr); + DORIS_CHECK(table_type != nullptr); + DORIS_CHECK(row < column->size()); + if (table_type->is_nullable()) { + const auto& nested_type = + assert_cast(*table_type).get_nested_type(); + if (const auto* nullable_column = check_and_get_column(*column)) { + // A nearer nullable wrapper already protects its descendants at this entry, so an + // inherited collection mask cannot be needed there. + if (nullable_column->is_null_at(row)) { + return false; + } + return _requires_parent_null_map_for_alignment_at( + nullable_column->get_nested_column_ptr(), nested_type, row); + } + return _requires_parent_null_map_for_alignment_at(column, nested_type, row); + } + if (const auto* nullable_column = check_and_get_column(*column)) { + if (nullable_column->is_null_at(row)) { + return true; + } + return _requires_parent_null_map_for_alignment_at( + nullable_column->get_nested_column_ptr(), table_type, row); + } + if (const auto* array_type = typeid_cast(table_type.get())) { + const auto& array_column = assert_cast(*column); + const auto& offsets = array_column.get_offsets(); + const size_t begin = row == 0 ? 0 : offsets[row - 1]; + const size_t end = offsets[row]; + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_requires_parent_null_map_for_alignment_at(array_column.get_data_ptr(), + array_type->get_nested_type(), + child_row)) { + return true; + } + } + return false; + } + if (const auto* map_type = typeid_cast(table_type.get())) { + const auto& map_column = assert_cast(*column); + const auto& offsets = map_column.get_offsets(); + const size_t begin = row == 0 ? 0 : offsets[row - 1]; + const size_t end = offsets[row]; + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_requires_parent_null_map_for_alignment_at( + map_column.get_keys_ptr(), map_type->get_key_type(), child_row) || + _requires_parent_null_map_for_alignment_at( + map_column.get_values_ptr(), map_type->get_value_type(), child_row)) { + return true; + } + } + return false; + } + if (const auto* struct_type = typeid_cast(table_type.get())) { + const auto& struct_column = assert_cast(*column); + DORIS_CHECK(struct_column.tuple_size() == struct_type->get_elements().size()); + for (size_t i = 0; i < struct_column.tuple_size(); ++i) { + if (_requires_parent_null_map_for_alignment_at(struct_column.get_column_ptr(i), + struct_type->get_element(i), row)) { + return true; + } + } + } + return false; + } + + static bool _requires_collection_parent_null_map(const NullMap* parent_null_map, + const ColumnPtr& column, + const DataTypePtr& table_type) { + // Descendant null maps can be entry-sized. Scan them only when an inherited mask can + // actually hide a row; absent/all-clear masks cannot authorize any physical child NULL. + if (parent_null_map == nullptr || + std::ranges::none_of(*parent_null_map, [](const auto value) { return value != 0; })) { + return false; + } + return _requires_parent_null_map_for_alignment(column, table_type); + } + template - static const NullMap* _project_collection_parent_null_map( - const NullMap* container_null_map, const NullMap* ancestor_null_map, const size_t rows, - const Offsets& offsets, const size_t child_rows, NullMap* const projected_null_map) { + static bool _parent_null_map_hides_collection_entries(const NullMap* container_null_map, + const NullMap* ancestor_null_map, + const size_t rows, + const Offsets& offsets) { if (container_null_map == nullptr && ancestor_null_map == nullptr) { - return nullptr; + return false; } DORIS_CHECK(container_null_map == nullptr || container_null_map->size() == rows); DORIS_CHECK(ancestor_null_map == nullptr || ancestor_null_map->size() == rows); DORIS_CHECK(offsets.size() == rows); + size_t begin = 0; + for (size_t row = 0; row < rows; ++row) { + const size_t end = offsets[row]; + const bool hidden = (container_null_map != nullptr && (*container_null_map)[row]) || + (ancestor_null_map != nullptr && (*ancestor_null_map)[row]); + // A hidden collection row protects descendants only when its offset span is nonempty. + if (hidden && end > begin) { + return true; + } + begin = end; + } + return false; + } + + template + static bool _requires_collection_parent_null_map(const NullMap* parent_null_map, + const ColumnPtr& column, + const DataTypePtr& table_type, + const size_t rows, const Offsets& offsets) { + if (parent_null_map == nullptr) { + return false; + } + DORIS_CHECK(parent_null_map->size() == rows); + DORIS_CHECK(offsets.size() == rows); + DORIS_CHECK(offsets.empty() || offsets.back() == column->size()); + size_t begin = 0; + for (size_t row = 0; row < rows; ++row) { + const size_t end = offsets[row]; + if ((*parent_null_map)[row]) { + // Only ancestor-hidden entries can consume this projection. Restricting the probe + // to their spans avoids scanning visible payload covered by nearer nullable masks. + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_requires_parent_null_map_for_alignment_at(column, table_type, child_row)) { + return true; + } + } + } + begin = end; + } + return false; + } + + template + static const NullMap* _project_collection_parent_null_map_for_hidden_entries( + const NullMap* container_null_map, const NullMap* ancestor_null_map, const size_t rows, + const Offsets& offsets, const size_t child_rows, NullMap* const projected_null_map) { + if (!_parent_null_map_hides_collection_entries(container_null_map, ancestor_null_map, rows, + offsets)) { + // Nullable collection wrappers expose a null-map even when every row is present; avoid + // allocating entry-coordinate scratch unless a hidden row owns physical entries. + return nullptr; + } projected_null_map->resize(child_rows); std::fill(projected_null_map->begin(), projected_null_map->end(), 0); size_t begin = 0; @@ -1534,7 +1744,6 @@ class TableReader { DORIS_CHECK(begin == child_rows); return projected_null_map; } - Status _materialize_struct_mapping_column(const ColumnMapping& mapping, const ColumnPtr& file_column, const size_t rows, ColumnPtr* column, @@ -1658,9 +1867,10 @@ class TableReader { // storage invariant, so add it only at the materialization boundary. element_mapping.table_type = make_nullable(element_mapping.table_type); NullMap descendant_parent_null_map; - const NullMap* descendant_parent_null_map_ptr = _project_collection_parent_null_map( - parent_null_map, nullable_parent_null_map, rows, file_array->get_offsets(), - nested_column->size(), &descendant_parent_null_map); + const NullMap* descendant_parent_null_map_ptr = + _project_collection_parent_null_map_for_hidden_entries( + parent_null_map, nullable_parent_null_map, rows, file_array->get_offsets(), + nested_column->size(), &descendant_parent_null_map); RETURN_IF_ERROR(_materialize_present_child_mapping_column( element_mapping, nested_column, nested_column->size(), &nested_column, descendant_parent_null_map_ptr)); @@ -1712,9 +1922,10 @@ class TableReader { ColumnPtr value_column = file_map->get_values_ptr(); DORIS_CHECK(key_column->size() == value_column->size()); NullMap descendant_parent_null_map; - const NullMap* descendant_parent_null_map_ptr = _project_collection_parent_null_map( - parent_null_map, nullable_parent_null_map, rows, file_map->get_offsets(), - key_column->size(), &descendant_parent_null_map); + const NullMap* descendant_parent_null_map_ptr = + _project_collection_parent_null_map_for_hidden_entries( + parent_null_map, nullable_parent_null_map, rows, file_map->get_offsets(), + key_column->size(), &descendant_parent_null_map); const ColumnMapping* key_mapping = nullptr; const ColumnMapping* value_mapping = nullptr; diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 4ac9117fad9ae3..895a8b1d0116f6 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -35,10 +35,12 @@ #include #include #include +#include #include #include #include #include +#include #include "common/exception.h" #include "common/logging.h" @@ -46,8 +48,8 @@ #include "cpp/obj_retry_strategy.h" #include "io/fs/obj_storage_client.h" #include "util/bvar_helper.h" -#include "util/coding.h" #include "util/s3_util.h" +#include "util/uid_util.h" using namespace Azure::Storage::Blobs; @@ -64,10 +66,16 @@ std::string to_lower_ascii(std::string_view input) { return lowered; } -auto base64_encode_part_num(int part_num) { - uint8_t buf[4]; - doris::encode_fixed32_le(buf, static_cast(part_num)); - return Aws::Utils::HashingUtils::Base64Encode({buf, sizeof(buf)}); +std::string encode_azure_block_id(std::string_view upload_id, int part_num) { + // Keep the full upload UUID in every block ID so independent writers cannot stage the + // same block IDs even though Azure has no per-upload multipart namespace. + std::vector raw_id(upload_id.begin(), upload_id.end()); + auto part = static_cast(part_num); + for (size_t i = 0; i < sizeof(part); ++i) { + raw_id.push_back(static_cast(part >> (i * 8))); + } + Aws::Utils::ByteBuffer bytes(raw_id.data(), raw_id.size()); + return Aws::Utils::HashingUtils::Base64Encode(bytes); } template @@ -98,6 +106,10 @@ constexpr char BlobNotFound[] = "BlobNotFound"; namespace doris::io { +std::string azure_multipart_block_id(std::string_view upload_id, int part_num) { + return encode_azure_block_id(upload_id, part_num); +} + // As Azure's doc said, the batch size is 256 // You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id // > Each batch request supports a maximum of 256 subrequests. @@ -215,11 +227,12 @@ struct AzureBatchDeleter { std::vector> deferred_resps; }; -// Azure would do nothing ObjectStorageUploadResponse AzureObjStorageClient::create_multipart_upload( - const ObjectStoragePathOptions& opts) { + const ObjectStoragePathOptions&) { + // Azure has no multipart session; this local UUID only namespaces the writer's block IDs. return ObjectStorageUploadResponse { .resp = ObjectStorageResponse::OK(), + .upload_id = generate_uuid_string(), }; } @@ -240,7 +253,9 @@ ObjectStorageResponse AzureObjStorageClient::put_object(const ObjectStoragePathO ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, std::string_view stream, int part_num) { + DCHECK(opts.upload_id.has_value()); auto client = _client->GetBlockBlobClient(opts.key); + std::string block_id = azure_multipart_block_id(*opts.upload_id, part_num); auto resp = do_azure_client_call( [&]() { Azure::Core::IO::MemoryBodyStream memory_body( @@ -248,28 +263,33 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora // The blockId must be base64 encoded s3_put_rate_limit([&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.StageBlock(base64_encode_part_num(part_num), memory_body); + client.StageBlock(block_id, memory_body); }); }, opts, _tls_debug_context); return ObjectStorageUploadResponse { .resp = resp, + // Hive defers completion to FE, so the exact staged ID must cross that boundary. + .etag = block_id, }; } ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) { - auto client = _client->GetBlockBlobClient(opts.key); + DCHECK(opts.upload_id.has_value()); + auto target_client = _client->GetBlockBlobClient(opts.key); std::vector string_block_ids; - std::ranges::transform( - completed_parts, std::back_inserter(string_block_ids), - [](const ObjectCompleteMultiPart& i) { return base64_encode_part_num(i.part_num); }); + std::ranges::transform(completed_parts, std::back_inserter(string_block_ids), + [&opts](const ObjectCompleteMultiPart& part) { + return azure_multipart_block_id(*opts.upload_id, part.part_num); + }); return do_azure_client_call( [&]() { s3_put_rate_limit([&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.CommitBlockList(string_block_ids); + // Committing the exact writer namespace atomically publishes only its blocks. + target_client.CommitBlockList(string_block_ids); }); }, opts, _tls_debug_context); diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index 7d1cecc502e44d..6cf6493e082af8 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -33,6 +33,7 @@ class ObjClientHolder; bool is_azure_tls_ca_error_message(std::string_view message); std::string build_azure_tls_debug_suffix(std::string_view error_message, std::string_view tls_debug_context); +std::string azure_multipart_block_id(std::string_view upload_id, int part_num); class AzureObjStorageClient final : public ObjStorageClient { public: diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index fa239ca3282e2a..2688ee70e02a4b 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -44,7 +44,7 @@ struct ObjectStoragePathOptions { std::string bucket = std::string(); // blob container in azure std::string key = std::string(); // blob name in azure std::string prefix = std::string(); // for batch delete and recursive delete - std::optional upload_id = std::nullopt; // only used for S3 upload + std::optional upload_id = std::nullopt; // token identifying this writer's parts }; struct ObjectCompleteMultiPart { @@ -86,7 +86,8 @@ struct ObjectStorageHeadResponse : ObjectStorageResponse { class ObjStorageClient { public: virtual ~ObjStorageClient() = default; - // Create a multi-part upload request. On AWS-compatible systems, it will return an upload ID, but not on Azure. + // Create a multi-part upload request. The returned token may be provider-issued or local and + // identifies this writer's parts. // The input parameters should include the bucket and key for the object storage. virtual ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) = 0; diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index ec8e54849f626f..03909e607001a9 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -86,7 +86,8 @@ S3FileWriter::~S3FileWriter() { _wait_until_finish(fmt::format("wait s3 file {} upload to be finished", _obj_storage_path_opts.path.native())); } - // We won't do S3 abort operation in BE, we let s3 service do it own. + // Deferred uploads are reported to FE for cleanup. Uploads that never reach FE are left to + // the provider lifecycle policy, so destroying a writer must not mutate provider state here. if (state() == State::OPENED && !_failed) { s3_bytes_written_total << _bytes_appended; } diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 5a8075e03cf404..83ec75c9184920 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -75,7 +75,6 @@ class S3FileWriter final : public FileWriter { private: Status _close_impl(); - Status _abort(); [[nodiscard]] std::string _dump_completed_part() const; void _wait_until_finish(std::string_view task_name); Status _complete(); diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp index c876eab7bb42ad..ebb8a14d9169a7 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp @@ -26,6 +26,46 @@ namespace doris { #include "common/compile_check_begin.h" +void ReservedMemoryToken::release() { + if (_bytes == 0 && _untracked_bytes == 0) { + return; + } + // A queued item may be discarded after an async failure; its reservation still needs full rollback. + GlobalMemoryArbitrator::shrink_process_reserved(_bytes + _untracked_bytes); + _limiter_tracker->shrink_reserved(_bytes + _untracked_bytes); + _limiter_tracker->release(_bytes); + if (auto wg = _wg_wptr.lock()) { + wg->sub_wg_refresh_interval_memory_growth(_bytes); + } + _bytes = 0; + _untracked_bytes = 0; +} + +ReservedMemoryToken ThreadMemTrackerMgr::take_reserved_memory() { + CHECK(init()); + if (_reserved_mem == 0) { + return {}; + } + ReservedMemoryToken token(_limiter_tracker_sptr, _wg_wptr, _reserved_mem, _untracked_mem); + // Accounting remains reserved globally; only its thread-local ownership moves into the token. + _reserved_mem = 0; + _untracked_mem = 0; + return token; +} + +void ThreadMemTrackerMgr::adopt_reserved_memory(ReservedMemoryToken&& token) { + CHECK(init()); + if (token._bytes == 0 && token._untracked_bytes == 0) { + return; + } + flush_untracked_mem(); + CHECK(token._limiter_tracker == _limiter_tracker_sptr); + _reserved_mem += token._bytes; + _untracked_mem += token._untracked_bytes; + token._bytes = 0; + token._untracked_bytes = 0; +} + void ThreadMemTrackerMgr::attach_limiter_tracker( const std::shared_ptr& mem_tracker) { DCHECK(mem_tracker); diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h index ed98fd1c2467db..6bd85e88892647 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.h +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "common/be_mock_util.h" @@ -40,6 +41,42 @@ namespace doris { #include "common/compile_check_begin.h" +class ReservedMemoryToken { +public: + ReservedMemoryToken() = default; + ReservedMemoryToken(const ReservedMemoryToken&) = delete; + ReservedMemoryToken& operator=(const ReservedMemoryToken&) = delete; + ReservedMemoryToken(ReservedMemoryToken&& other) noexcept { *this = std::move(other); } + ReservedMemoryToken& operator=(ReservedMemoryToken&& other) noexcept { + if (this != &other) { + release(); + _limiter_tracker = std::move(other._limiter_tracker); + _wg_wptr = std::move(other._wg_wptr); + _bytes = std::exchange(other._bytes, 0); + _untracked_bytes = std::exchange(other._untracked_bytes, 0); + } + return *this; + } + ~ReservedMemoryToken() { release(); } + [[nodiscard]] int64_t bytes() const { return _bytes; } + +private: + friend class ThreadMemTrackerMgr; + ReservedMemoryToken(std::shared_ptr limiter_tracker, + std::weak_ptr wg_wptr, int64_t bytes, + int64_t untracked_bytes) + : _limiter_tracker(std::move(limiter_tracker)), + _wg_wptr(std::move(wg_wptr)), + _bytes(bytes), + _untracked_bytes(untracked_bytes) {} + void release(); + + std::shared_ptr _limiter_tracker; + std::weak_ptr _wg_wptr; + int64_t _bytes = 0; + int64_t _untracked_bytes = 0; +}; + constexpr size_t SYNC_PROC_RESERVED_INTERVAL_BYTES = (1ULL << 20); // 1M static std::string MEMORY_ORPHAN_CHECK_MSG = "The ThreadContext of the current thread not attach a valid MemoryTracker. after the " @@ -100,6 +137,9 @@ class ThreadMemTrackerMgr { void shrink_reserved(); + ReservedMemoryToken take_reserved_memory(); + void adopt_reserved_memory(ReservedMemoryToken&& token); + MemTrackerLimiter* limiter_mem_tracker() { CHECK(init()); return _limiter_tracker; diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index a12c8743ee4cfe..3b697b36aaa0c5 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -53,6 +53,7 @@ #include "runtime/thread_context.h" #include "storage/id_manager.h" #include "storage/storage_engine.h" +#include "util/thrift_util.h" #include "util/timezone_utils.h" #include "util/uid_util.h" @@ -60,6 +61,98 @@ namespace doris { #include "common/compile_check_begin.h" using namespace ErrorCode; +Status RuntimeState::add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data) { + ThriftSerializer serializer(false, 256); + uint32_t serialized_size = 0; + uint8_t* buffer = nullptr; + RETURN_IF_ERROR(serializer.serialize(&iceberg_commit_data, &serialized_size, &buffer)); + + // This is an early per-vector guard only; the assembled RPC is measured again before send. + constexpr size_t report_envelope_headroom = 1024 * 1024; + const size_t thrift_limit = coordinator_thrift_message_limit(); + const size_t commit_data_limit = + thrift_limit > report_envelope_headroom ? thrift_limit - report_envelope_headroom : 0; + std::lock_guard budget_lock(_external_file_report_state->mutex); + // Parallel task states share this budget because FE receives their vectors in one fragment report. + if (_external_file_report_state->iceberg_serialized_bytes + serialized_size + sizeof(uint32_t) > + commit_data_limit) { + return Status::InternalError( + "Iceberg commit metadata exceeds the Thrift report limit; reduce output file " + "count"); + } + std::lock_guard data_lock(_iceberg_commit_datas_mutex); + _external_file_report_state->iceberg_serialized_bytes += serialized_size + sizeof(uint32_t); + _iceberg_commit_datas.emplace_back(std::move(iceberg_commit_data)); + return Status::OK(); +} + +size_t RuntimeState::coordinator_thrift_message_limit() const { + int32_t effective_thrift_limit = std::max(config::thrift_max_message_size, 0); + if (_query_options.__isset.coordinator_thrift_max_message_size && + _query_options.coordinator_thrift_max_message_size > 0) { + // An older FE omits this field; otherwise the receiver's smaller limit is authoritative. + effective_thrift_limit = std::min(effective_thrift_limit, + _query_options.coordinator_thrift_max_message_size); + } + return static_cast(effective_thrift_limit); +} + +void RuntimeState::append_external_file_commit_data(TReportExecStatusParams* params, + bool final_report) const { + if (!final_report) { + // Ownership-bearing commit vectors must only appear in the final report that transfers them. + return; + } + if (auto updates = hive_partition_updates(); !updates.empty()) { + params->__isset.hive_partition_updates = true; + params->hive_partition_updates.insert(params->hive_partition_updates.end(), updates.begin(), + updates.end()); + } + append_iceberg_commit_datas(¶ms->iceberg_commit_datas); + if (!params->iceberg_commit_datas.empty()) { + params->__isset.iceberg_commit_datas = true; + } + if (auto commit_datas = mc_commit_datas(); !commit_datas.empty()) { + params->__isset.mc_commit_datas = true; + params->mc_commit_datas.insert(params->mc_commit_datas.end(), commit_datas.begin(), + commit_datas.end()); + } + if (auto commit_messages = paimon_commit_messages(); !commit_messages.empty()) { + // branch-4.1 still carries Paimon commit messages in the shared external-file report. + params->__isset.paimon_commit_messages = true; + params->paimon_commit_messages.insert(params->paimon_commit_messages.end(), + commit_messages.begin(), commit_messages.end()); + } +} + +void RuntimeState::add_rejected_external_file_report_cleanup(std::function cleanup) { + std::lock_guard lock(_external_file_report_state->mutex); + _external_file_report_state->rejected_report_cleanups.emplace_back(std::move(cleanup)); +} + +void RuntimeState::finalize_external_file_report_cleanup(ExternalFileReportOutcome outcome) { + std::vector> cleanups; + { + std::lock_guard lock(_external_file_report_state->mutex); + if (outcome == ExternalFileReportOutcome::ACKNOWLEDGED) { + _external_file_report_state->rejected_report_cleanups.clear(); + return; + } + if (outcome == ExternalFileReportOutcome::AMBIGUOUS) { + // Once an ACK can have been lost, a later rejection cannot prove FE never accepted the files. + _external_file_report_state->ownership_may_have_transferred = true; + return; + } + if (_external_file_report_state->ownership_may_have_transferred) { + return; + } + cleanups.swap(_external_file_report_state->rejected_report_cleanups); + } + for (auto& cleanup : cleanups) { + cleanup(); + } +} + RuntimeState::RuntimeState(const TPlanFragmentExecParams& fragment_exec_params, const TQueryOptions& query_options, const TQueryGlobals& query_globals, ExecEnv* exec_env, QueryContext* ctx, diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 7b3bb7e3339d2d..3748e1c31bc503 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -75,6 +75,21 @@ class RuntimeFilterConsumer; class RuntimeFilterProducer; class TaskExecutionContext; +// Keep RuntimeState self-contained without importing the full frontend Thrift service header. +class TReportExecStatusParams; + +class ExternalFileReportState { + friend class RuntimeState; + +private: + std::mutex mutex; + size_t iceberg_serialized_bytes = 0; + bool ownership_may_have_transferred = false; + std::vector> rejected_report_cleanups; +}; + +enum class ExternalFileReportOutcome { ACKNOWLEDGED, REJECTED, AMBIGUOUS }; + // A collection of items that are part of the global state of a // query and shared across all execution nodes of that query. class RuntimeState { @@ -532,14 +547,27 @@ class RuntimeState { _hive_partition_updates.emplace_back(hive_partition_update); } - std::vector iceberg_commit_datas() const { + void append_iceberg_commit_datas(std::vector* output) const { std::lock_guard lock(_iceberg_commit_datas_mutex); - return _iceberg_commit_datas; + output->insert(output->end(), _iceberg_commit_datas.begin(), _iceberg_commit_datas.end()); } - void add_iceberg_commit_datas(const TIcebergCommitData& iceberg_commit_data) { - std::lock_guard lock(_iceberg_commit_datas_mutex); - _iceberg_commit_datas.emplace_back(iceberg_commit_data); + Status add_iceberg_commit_datas(TIcebergCommitData iceberg_commit_data); + + size_t coordinator_thrift_message_limit() const; + + void append_external_file_commit_data(TReportExecStatusParams* params, bool final_report) const; + + void add_rejected_external_file_report_cleanup(std::function cleanup); + + void finalize_external_file_report_cleanup(ExternalFileReportOutcome outcome); + + void set_external_file_report_state(std::shared_ptr report_state) { + _external_file_report_state = std::move(report_state); + } + + const std::shared_ptr& external_file_report_state() const { + return _external_file_report_state; } std::vector mc_commit_datas() const { @@ -994,6 +1022,8 @@ class RuntimeState { mutable std::mutex _iceberg_commit_datas_mutex; std::vector _iceberg_commit_datas; + std::shared_ptr _external_file_report_state = + std::make_shared(); mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; diff --git a/be/test/core/value/merge_partitioner_test.cpp b/be/test/core/value/merge_partitioner_test.cpp index 0836c6f875395c..2df06815b44f75 100644 --- a/be/test/core/value/merge_partitioner_test.cpp +++ b/be/test/core/value/merge_partitioner_test.cpp @@ -83,6 +83,41 @@ class MergePartitionerTest : public ::testing::Test { return expr; } + TTypeDesc _nested_int_struct_type_desc() { + TTypeNode struct_node; + struct_node.__set_type(TTypeNodeType::STRUCT); + TStructField child; + child.__set_name("part"); + child.__set_contains_null(true); + struct_node.__set_struct_fields({child}); + + TTypeNode int_node; + int_node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar; + scalar.__set_type(TPrimitiveType::INT); + int_node.__set_scalar_type(scalar); + + TTypeDesc type_desc; + type_desc.__set_types({struct_node, int_node}); + type_desc.__set_is_nullable(true); + return type_desc; + } + + TExpr _make_nested_source_expr() { + TExprNode node; + node.__set_node_type(TExprNodeType::SLOT_REF); + node.__set_num_children(0); + TSlotRef slot_ref; + slot_ref.__set_slot_id(_nested_source_slot_id); + slot_ref.__set_tuple_id(_tuple_id); + node.__set_slot_ref(slot_ref); + node.__set_type(_nested_int_struct_type_desc()); + node.__set_is_nullable(true); + TExpr expr; + expr.nodes.emplace_back(std::move(node)); + return expr; + } + TMergePartitionInfo _make_base_merge_info(bool insert_random) { TMergePartitionInfo merge_info; merge_info.__set_operation_expr( @@ -180,6 +215,13 @@ class MergePartitionerTest : public ::testing::Test { .column_name("delete_key") .column_pos(4) .build()); + TTypeDesc nested_type = _nested_int_struct_type_desc(); + tuple_builder.add_slot(TSlotDescriptorBuilder() + .set_slotType(nested_type) + .nullable(true) + .column_name("nested_source") + .column_pos(5) + .build()); tuple_builder.build(&dtb); TDescriptorTable thrift_tbl = dtb.desc_tbl(); @@ -205,11 +247,13 @@ class MergePartitionerTest : public ::testing::Test { _row_id_slot_id = find_slot_id("row_id"); _insert_key_slot_id = find_slot_id("insert_key"); _delete_key_slot_id = find_slot_id("delete_key"); + _nested_source_slot_id = find_slot_id("nested_source"); ASSERT_GE(_operation_slot_id, 0); ASSERT_GE(_row_id_slot_id, 0); ASSERT_GE(_insert_key_slot_id, 0); ASSERT_GE(_delete_key_slot_id, 0); + ASSERT_GE(_nested_source_slot_id, 0); } ObjectPool _pool; @@ -220,6 +264,7 @@ class MergePartitionerTest : public ::testing::Test { TSlotId _row_id_slot_id = -1; TSlotId _insert_key_slot_id = -1; TSlotId _delete_key_slot_id = -1; + TSlotId _nested_source_slot_id = -1; }; TEST_F(MergePartitionerTest, TestInsertDeleteUpdatePartitioning) { @@ -318,6 +363,52 @@ TEST_F(MergePartitionerTest, TestInsertPartitionFieldsIdentity) { ASSERT_TRUE(partitioner.close(&_state).ok()); } +TEST_F(MergePartitionerTest, TestNestedInsertPartitionFieldPreservesParentNulls) { + ScopedConfigValue max_partition_guard( + config::table_sink_partition_write_max_partition_nums_per_writer, 0); + + TMergePartitionInfo merge_info = _make_base_merge_info(false); + TIcebergPartitionField field; + field.__set_transform("identity"); + field.__set_source_expr(_make_nested_source_expr()); + field.__set_name("payload_part"); + field.__set_source_id(3); + field.__set_source_field_path({0}); + merge_info.__set_insert_partition_fields({field}); + + MergePartitioner partitioner(8, merge_info, false); + ASSERT_TRUE(partitioner.init({}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_desc).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block block = _build_block({1, 1, 1, 1}, {"p1", "p2", "p3", "p4"}, {1, 2, 3, 4}, + {10, 11, 12, 13}, {"d1", "d2", "d3", "d4"}); + auto values = ColumnInt32::create(); + values->insert_value(9); + values->insert_value(9); + values->insert_value(7); + values->insert_value(8); + auto child_nulls = ColumnUInt8::create(4, 0); + ColumnPtr child = ColumnNullable::create(std::move(values), std::move(child_nulls)); + auto struct_column = ColumnStruct::create(Columns {std::move(child)}); + auto parent_nulls = ColumnUInt8::create(); + parent_nulls->get_data().assign({0, 0, 1, 1}); + DataTypePtr child_type = make_nullable(std::make_shared()); + DataTypePtr struct_type = + std::make_shared(DataTypes {child_type}, Strings {"part"}); + block.insert(ColumnWithTypeAndName( + ColumnNullable::create(std::move(struct_column), std::move(parent_nulls)), + make_nullable(struct_type), "nested_source")); + + ASSERT_TRUE(partitioner.do_partitioning(&_state, &block).ok()); + const auto& channel_ids = partitioner.get_channel_ids(); + ASSERT_EQ(4, channel_ids.size()); + EXPECT_EQ(channel_ids[0], channel_ids[1]); + EXPECT_EQ(channel_ids[2], channel_ids[3]); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + TEST_F(MergePartitionerTest, TestInvalidTransformFallbacksToRandom) { ScopedConfigValue threshold_guard( config::table_sink_non_partition_write_scaling_data_processed_threshold, 0); diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp new file mode 100644 index 00000000000000..3e004051220504 --- /dev/null +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "core/block/block.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_string.h" +#include "exec/operator/iceberg_sorter_reserve_memory.h" +#include "exec/sink/writer/async_writer_queue_admission.h" +#include "exec/sink/writer/hive_multipart_compatibility.h" + +namespace doris { + +TEST(SpillIcebergTableSinkOperatorTest, BoundsManyPartitionReservationToOneInputBlock) { + std::vector per_partition_reservations( + 128, {.retained_growth = 0, .transient_workspace = 8 * 1024 * 1024}); + + EXPECT_EQ(8 * 1024 * 1024, bounded_iceberg_reserve_size(per_partition_reservations)); +} + +TEST(SpillIcebergTableSinkOperatorTest, AccumulatesRetainedGrowthAcrossTouchedPartitions) { + std::vector per_partition_reservations { + {.retained_growth = 3 * 1024 * 1024, .transient_workspace = 7 * 1024 * 1024}, + {.retained_growth = 4 * 1024 * 1024, .transient_workspace = 5 * 1024 * 1024}}; + + EXPECT_EQ(14 * 1024 * 1024, bounded_iceberg_reserve_size(per_partition_reservations)); +} + +TEST(SpillIcebergTableSinkOperatorTest, BoundsRetainedGrowthByOneInputBlock) { + constexpr size_t MB = 1024 * 1024; + std::vector per_partition_reservations( + 128, {.retained_growth = 8 * MB, + .retained_growth_trigger_bytes = 8 * MB, + .transient_workspace = 4 * MB}); + + // Only one 8 MiB growth threshold can be crossed by this block. Treating the block as a full + // batch for every active partition would incorrectly reserve more than 1 GiB here. + EXPECT_EQ(12 * MB, bounded_iceberg_reserve_size(per_partition_reservations, 128, 8 * MB)); +} + +TEST(SpillIcebergTableSinkOperatorTest, RetainsNearCapacityGrowthAcrossAllPossiblePartitions) { + constexpr size_t MB = 1024 * 1024; + std::vector per_partition_reservations( + 4, {.retained_growth = 3 * MB, + .retained_growth_trigger_bytes = 0, + .transient_workspace = 2 * MB}); + + // A one-row append can grow every already-near-capacity sorter, so persistent growth remains + // cumulative even though the serially used workspace is shared. + EXPECT_EQ(14 * MB, bounded_iceberg_reserve_size(per_partition_reservations, 4, 1)); +} + +TEST(SpillIcebergTableSinkOperatorTest, ReservesIncomingBlockBeforeAnyPartitionWriterExists) { + std::vector no_published_sorters; + + EXPECT_EQ(6 * 1024 * 1024, iceberg_reserve_size(no_published_sorters, 6 * 1024 * 1024)); +} + +TEST(SpillIcebergTableSinkOperatorTest, ColdWriterReserveUsesFirstBlockLargerThanOperatorFloor) { + constexpr size_t operator_floor = 32 * 1024 * 1024; + auto strings = ColumnString::create(); + std::string payload(40 * 1024 * 1024, 'x'); + strings->insert_data(payload.data(), payload.size()); + Block block; + block.insert({std::move(strings), std::make_shared(), "payload"}); + + ASSERT_GT(block.allocated_bytes(), operator_floor); + EXPECT_GE(iceberg_cold_writer_reserve_size(block, operator_floor), + 4 * block.allocated_bytes() + operator_floor); +} + +TEST(SpillIcebergTableSinkOperatorTest, ReservesAllMergeInputsAndOutputAtEos) { + constexpr size_t MB = 1024 * 1024; + + EXPECT_EQ(72 * MB, iceberg_spill_merge_workspace(12, 8 * MB, 64 * MB)); + EXPECT_EQ(32 * MB, iceberg_spill_merge_workspace(3, 8 * MB, 64 * MB)); +} + +TEST(SpillIcebergTableSinkOperatorTest, WaitsUntilDequeuedBlockUpdatesSorterState) { + AsyncWriterQueueAdmission stateful_admission; + stateful_admission.wait_for_processing_before_next_sink(); + + EXPECT_TRUE(stateful_admission.is_available(0)); + EXPECT_FALSE(stateful_admission.is_available(1)); + stateful_admission.begin_processing(); + // Dequeueing does not admit block 2 until block 1 changes the state sampled by admission. + EXPECT_FALSE(stateful_admission.is_available(0)); + stateful_admission.finish_processing(); + EXPECT_TRUE(stateful_admission.is_available(0)); + + // Writers without state-dependent admission retain the existing three-block queue behavior. + AsyncWriterQueueAdmission buffered_admission; + buffered_admission.begin_processing(); + EXPECT_TRUE(buffered_admission.is_available(2)); + EXPECT_FALSE(buffered_admission.is_available(3)); +} + +TEST(SpillIcebergTableSinkOperatorTest, TerminalWriterDrainsQueuedReservations) { + int live_reservations = 0; + struct Reservation { + explicit Reservation(int* live) : live(live) { ++*live; } + ~Reservation() { --*live; } + int* live; + }; + struct Queued { + size_t bytes; + std::unique_ptr reservation; + }; + std::deque queue; + queue.push_back({7, std::make_unique(&live_reservations)}); + queue.push_back({11, std::make_unique(&live_reservations)}); + size_t released_bytes = 0; + + drain_async_writer_queue(queue, [&](const Queued& queued) { released_bytes += queued.bytes; }); + + EXPECT_TRUE(queue.empty()); + EXPECT_EQ(0, live_reservations); + EXPECT_EQ(18, released_bytes); +} + +TEST(SpillIcebergTableSinkOperatorTest, AzureDeferredMultipartRequiresCoordinatorCapability) { + EXPECT_TRUE(hive_multipart_protocol_supported(io::ObjStorageType::AWS, false)); + EXPECT_FALSE(hive_multipart_protocol_supported(io::ObjStorageType::AZURE, false)); + EXPECT_TRUE(hive_multipart_protocol_supported(io::ObjStorageType::AZURE, true)); +} + +} // namespace doris diff --git a/be/test/exec/sink/viceberg_delete_sink_test.cpp b/be/test/exec/sink/viceberg_delete_sink_test.cpp index 7faa77ed702c68..e2897125e497f3 100644 --- a/be/test/exec/sink/viceberg_delete_sink_test.cpp +++ b/be/test/exec/sink/viceberg_delete_sink_test.cpp @@ -94,6 +94,18 @@ TEST_F(VIcebergDeleteSinkTest, TestInitProperties) { ASSERT_TRUE(status.ok()); } +TEST_F(VIcebergDeleteSinkTest, RejectsCoordinatorWithoutExternalFileReportAck) { + VExprContextSPtrs output_exprs; + auto sink = std::make_shared(_t_data_sink, output_exprs, nullptr, nullptr); + RuntimeState state; + RuntimeProfile profile("test"); + + Status status = sink->open(&state, &profile); + + EXPECT_TRUE(status.is()); + EXPECT_NE(std::string::npos, status.to_string().find("acknowledges external-file reports")); +} + TEST_F(VIcebergDeleteSinkTest, TestGetRowIdColumnIndex) { VExprContextSPtrs output_exprs; auto sink = std::make_shared(_t_data_sink, output_exprs, nullptr, nullptr); diff --git a/be/test/exec/sink/viceberg_merge_sink_test.cpp b/be/test/exec/sink/viceberg_merge_sink_test.cpp index d1da8fb7bfe1d5..0de2a61969eb86 100644 --- a/be/test/exec/sink/viceberg_merge_sink_test.cpp +++ b/be/test/exec/sink/viceberg_merge_sink_test.cpp @@ -48,6 +48,16 @@ namespace doris { +class IcebergWriteMockRuntimeState : public MockRuntimeState { +public: + IcebergWriteMockRuntimeState() { + auto query_options = this->query_options(); + // Successful writer tests must model a coordinator that can accept file ownership. + query_options.__set_supports_external_file_report_ack(true); + set_query_options(query_options); + } +}; + class VIcebergMergeSinkTest : public testing::Test { protected: static std::string test_schema_json() { @@ -173,7 +183,7 @@ class VIcebergMergeSinkTest : public testing::Test { TEST_F(VIcebergMergeSinkTest, TestUpdateProducesDeleteAndInsert) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -240,7 +250,7 @@ TEST_F(VIcebergMergeSinkTest, TestDeleteOnlySkipsVariantDataWriter) { TEST_F(VIcebergMergeSinkTest, TestMissingOperationColumn) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -263,7 +273,7 @@ TEST_F(VIcebergMergeSinkTest, TestMissingOperationColumn) { TEST_F(VIcebergMergeSinkTest, TestMissingRowIdColumn) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -286,7 +296,7 @@ TEST_F(VIcebergMergeSinkTest, TestMissingRowIdColumn) { TEST_F(VIcebergMergeSinkTest, TestUnknownOperation) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -313,7 +323,7 @@ TEST_F(VIcebergMergeSinkTest, TestUnknownOperation) { TEST_F(VIcebergMergeSinkTest, TestUpdateInsertAndDeleteOperations) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -343,7 +353,7 @@ TEST_F(VIcebergMergeSinkTest, TestUpdateInsertAndDeleteOperations) { TEST_F(VIcebergMergeSinkTest, TestSchemaMismatch) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -370,7 +380,7 @@ TEST_F(VIcebergMergeSinkTest, TestSchemaMismatch) { TEST_F(VIcebergMergeSinkTest, TestRejectsDuplicateMatchedTargetAcrossBlocks) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -399,7 +409,7 @@ TEST_F(VIcebergMergeSinkTest, TestRejectsDuplicateMatchedTargetAcrossBlocks) { TEST_F(VIcebergMergeSinkTest, TestUpdateSkipsCardinalityState) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -427,7 +437,7 @@ TEST_F(VIcebergMergeSinkTest, TestUpdateSkipsCardinalityState) { TEST_F(VIcebergMergeSinkTest, TestOldFePlanSkipsCardinalityState) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -451,7 +461,7 @@ TEST_F(VIcebergMergeSinkTest, TestOldFePlanSkipsCardinalityState) { TEST_F(VIcebergMergeSinkTest, TestRollingUpgradeSkipsCardinalityState) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; state.set_be_exec_version(SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION - 1); DataTypes types {std::make_shared(), @@ -475,7 +485,7 @@ TEST_F(VIcebergMergeSinkTest, TestRollingUpgradeSkipsCardinalityState) { TEST_F(VIcebergMergeSinkTest, TestErrorCloseRemovesRolledDataFiles) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -507,7 +517,7 @@ TEST_F(VIcebergMergeSinkTest, TestErrorCloseRemovesRolledDataFiles) { TEST_F(VIcebergMergeSinkTest, TestDeleteCloseFailureRemovesBothInnerSinkFiles) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -557,7 +567,7 @@ TEST_F(VIcebergMergeSinkTest, TestDeleteCloseFailureRemovesBothInnerSinkFiles) { TEST_F(VIcebergMergeSinkTest, TestMatchedRowIdsUseCompactRetainedState) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), @@ -586,7 +596,7 @@ TEST_F(VIcebergMergeSinkTest, TestMatchedRowIdsUseCompactRetainedState) { TEST_F(VIcebergMergeSinkTest, TestMatchedRowIdStateAcrossManyFilesAndWrites) { ObjectPool pool; - MockRuntimeState state; + IcebergWriteMockRuntimeState state; DataTypes types {std::make_shared(), std::make_shared(DataTypes {std::make_shared(), diff --git a/be/test/exec/sink/writer/async_result_writer_test.cpp b/be/test/exec/sink/writer/async_result_writer_test.cpp new file mode 100644 index 00000000000000..18339b818de3c8 --- /dev/null +++ b/be/test/exec/sink/writer/async_result_writer_test.cpp @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/async_result_writer.h" + +#include + +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "exec/pipeline/dependency.h" +#include "runtime/exec_env.h" +#include "runtime/fragment_mgr.h" +#include "runtime/memory/global_memory_arbitrator.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/thread_context.h" +#include "runtime/workload_management/resource_context.h" + +namespace doris { + +namespace { + +const VExprContextSPtrs EMPTY_OUTPUT_EXPRS; + +Block make_block() { + auto values = ColumnInt32::create(); + values->insert_value(1); + Block block; + block.insert({std::move(values), std::make_shared(), "value"}); + return block; +} + +class RecordingAsyncWriter final : public AsyncResultWriter { +public: + RecordingAsyncWriter(std::shared_ptr dependency, + std::shared_ptr finish_dependency, Status open_status) + : AsyncResultWriter(EMPTY_OUTPUT_EXPRS, std::move(dependency), + std::move(finish_dependency)), + _open_status(std::move(open_status)) {} + + Status open(RuntimeState*, RuntimeProfile*) override { return _open_status; } + + Status write(RuntimeState*, Block&) override { + reservation_seen_by_write = thread_context()->thread_mem_tracker_mgr->reserved_mem(); + return Status::OK(); + } + + Status finish(RuntimeState*) override { + reservation_seen_by_finish = thread_context()->thread_mem_tracker_mgr->reserved_mem(); + return Status::OK(); + } + + Status close(Status) override { + reservation_seen_by_close = thread_context()->thread_mem_tracker_mgr->reserved_mem(); + return Status::OK(); + } + + int64_t reservation_seen_by_write = 0; + int64_t reservation_seen_by_finish = 0; + int64_t reservation_seen_by_close = 0; + +private: + Status _open_status; +}; + +struct AsyncWriterHarness { + AsyncWriterHarness() + : dependency(std::make_shared(0, 0, "writer", true)), + finish_dependency(std::make_shared(0, 0, "finish", false)), + common_profile("CommonCounters"), + memory_usage(common_profile.AddHighWaterMarkCounter("MemoryUsage", TUnit::BYTES)) {} + + void prepare(AsyncResultWriter* writer) { + writer->_operator_profile = &operator_profile; + writer->_memory_used_counter = memory_usage; + } + + void process(AsyncResultWriter* writer) { writer->process_block(nullptr, &operator_profile); } + + std::shared_ptr dependency; + std::shared_ptr finish_dependency; + RuntimeProfile operator_profile {"operator"}; + RuntimeProfile common_profile; + RuntimeProfile::Counter* memory_usage; +}; + +} // namespace + +class AsyncResultWriterTest : public testing::Test { +protected: + void SetUp() override { + _exec_env = ExecEnv::GetInstance(); + if (_exec_env->fragment_mgr() == nullptr) { + _fragment_mgr = std::make_unique(_exec_env); + _exec_env->_fragment_mgr = _fragment_mgr.get(); + } + _tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, + "UT-AsyncResultWriterReservation"); + _resource_context = ResourceContext::create_shared(); + _resource_context->memory_context()->set_mem_tracker(_tracker); + thread_context()->attach_task(_resource_context); + } + + void TearDown() override { + thread_context()->thread_mem_tracker_mgr->shrink_reserved(); + thread_context()->detach_task(); + EXPECT_EQ(0, GlobalMemoryArbitrator::process_reserved_memory()); + if (_fragment_mgr != nullptr) { + _fragment_mgr->stop(); + _exec_env->_fragment_mgr = nullptr; + _fragment_mgr.reset(); + } + } + + std::shared_ptr _tracker; + std::shared_ptr _resource_context; + ExecEnv* _exec_env = nullptr; + std::unique_ptr _fragment_mgr; +}; + +TEST_F(AsyncResultWriterTest, TransfersQueuedReservationIntoActualWrite) { + AsyncWriterHarness harness; + RecordingAsyncWriter writer(harness.dependency, harness.finish_dependency, Status::OK()); + harness.prepare(&writer); + constexpr int64_t reservation = 4 * 1024 * 1024; + Block block = make_block(); + ASSERT_TRUE(thread_context()->thread_mem_tracker_mgr->try_reserve(reservation).ok()); + + ASSERT_TRUE(writer.sink(&block, true).ok()); + EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); + harness.process(&writer); + + EXPECT_GT(writer.reservation_seen_by_write, 0); + EXPECT_LE(writer.reservation_seen_by_write, reservation); + EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); +} + +TEST_F(AsyncResultWriterTest, RetainsEosReservationThroughActualClose) { + AsyncWriterHarness harness; + RecordingAsyncWriter writer(harness.dependency, harness.finish_dependency, Status::OK()); + harness.prepare(&writer); + constexpr int64_t reservation = 4 * 1024 * 1024; + ASSERT_TRUE(thread_context()->thread_mem_tracker_mgr->try_reserve(reservation).ok()); + Block block; + + ASSERT_TRUE(writer.sink(&block, true).ok()); + harness.process(&writer); + + EXPECT_EQ(reservation, writer.reservation_seen_by_finish); + EXPECT_EQ(reservation, writer.reservation_seen_by_close); + EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); +} + +TEST_F(AsyncResultWriterTest, OpenFailureDrainsQueuedReservation) { + AsyncWriterHarness harness; + RecordingAsyncWriter writer(harness.dependency, harness.finish_dependency, + Status::IOError("injected open failure")); + harness.prepare(&writer); + constexpr int64_t reservation = 4 * 1024 * 1024; + Block block = make_block(); + ASSERT_TRUE(thread_context()->thread_mem_tracker_mgr->try_reserve(reservation).ok()); + + ASSERT_TRUE(writer.sink(&block, true).ok()); + harness.process(&writer); + + EXPECT_FALSE(writer.get_writer_status().ok()); + EXPECT_EQ(0, thread_context()->thread_mem_tracker_mgr->reserved_mem()); + EXPECT_EQ(0, GlobalMemoryArbitrator::process_reserved_memory()); +} + +} // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp index d453177cf25044..d5a7e3f9ab3351 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp @@ -20,6 +20,10 @@ #include #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" +#include "exec/sink/writer/iceberg/viceberg_sort_writer.h" +#include "testutil/mock/mock_descriptors.h" +#include "testutil/mock/mock_runtime_state.h" +#include "testutil/mock/mock_slot_ref.h" namespace doris { @@ -27,13 +31,18 @@ namespace { class FakeFileFormatTransformer final : public VFileFormatTransformer { public: - explicit FakeFileFormatTransformer(const VExprContextSPtrs& output_exprs) - : VFileFormatTransformer(nullptr, output_exprs, false) {} + explicit FakeFileFormatTransformer(const VExprContextSPtrs& output_exprs, + Status close_status = Status::OK()) + : VFileFormatTransformer(nullptr, output_exprs, false), + _close_status(std::move(close_status)) {} Status open() override { return Status::OK(); } Status write(const Block&) override { return Status::OK(); } - Status close() override { return Status::OK(); } + Status close() override { return _close_status; } int64_t written_len() override { return 64; } + +private: + Status _close_status; }; TDataSink make_table_sink(std::optional collect_column_stats) { @@ -64,9 +73,10 @@ class VIcebergPartitionWriterTest : public testing::Test { } static void install_fake_transformer(VIcebergPartitionWriter* writer, - const VExprContextSPtrs& output_exprs) { + const VExprContextSPtrs& output_exprs, + Status close_status = Status::OK()) { writer->_file_format_transformer = - std::make_unique(output_exprs); + std::make_unique(output_exprs, std::move(close_status)); } static Status build_commit_data(VIcebergPartitionWriter* writer, @@ -104,4 +114,48 @@ TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollin EXPECT_TRUE(collect_column_stats(*writer)); } +TEST_F(VIcebergPartitionWriterTest, SortWriterPropagatesUnderlyingCloseFailure) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto partition_writer = std::shared_ptr( + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf)); + install_fake_transformer(partition_writer.get(), output_exprs, + Status::IOError("injected close failure")); + VIcebergSortWriter sort_writer(partition_writer, TSortInfo(), 1024); + MockRuntimeState state; + sort_writer._runtime_state = &state; + + Status status = sort_writer.close(Status::OK()); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("injected close failure"), std::string::npos); +} + +TEST_F(VIcebergPartitionWriterTest, EosReservationIncludesActualSpillFanIn) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto partition_writer = std::shared_ptr( + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf)); + VIcebergSortWriter sort_writer(partition_writer, TSortInfo(), 1024); + MockRuntimeState state; + ObjectPool pool; + auto row_desc = std::make_unique( + std::vector {std::make_shared()}, &pool); + auto ordering_expr_ctxs = + MockSlotRef::create_mock_contexts(0, std::make_shared()); + std::vector is_asc_order {true}; + std::vector nulls_first {false}; + sort_writer._sorter = FullSorter::create_unique(ordering_expr_ctxs, -1, 0, &pool, is_asc_order, + nulls_first, *row_desc, &state, nullptr); + sort_writer._sorted_spill_files.resize(12); + + const auto reservation = sort_writer.get_reserve_mem_size_components(&state, true, 0, 0); + + EXPECT_EQ(72 * 1024 * 1024, reservation.transient_workspace); +} + } // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp new file mode 100644 index 00000000000000..ac6c100b37e143 --- /dev/null +++ b/be/test/exec/sink/writer/iceberg/iceberg_table_writer_test.cpp @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exec/operator/iceberg_sorter_reserve_memory.h" +#include "exec/sink/writer/iceberg/viceberg_table_writer.h" +#include "exec/sink/writer/iceberg/vpartition_writer_base.h" +#include "runtime/runtime_state.h" + +namespace doris { + +namespace { + +class FakePartitionWriter final : public IPartitionWriterBase { +public: + explicit FakePartitionWriter(std::atomic* destroyed = nullptr) : _destroyed(destroyed) {} + ~FakePartitionWriter() override { + if (_destroyed != nullptr) { + ++(*_destroyed); + } + } + Status open(RuntimeState*, RuntimeProfile*, const RowDescriptor*) override { + return Status::OK(); + } + Status write(Block&) override { return Status::OK(); } + Status close(const Status&) override { return Status::OK(); } + const std::string& file_name() const override { return _name; } + int file_name_index() const override { return 0; } + size_t written_len() const override { return 0; } + +private: + std::string _name = "fake"; + std::atomic* _destroyed; +}; + +TDataSink make_sink() { + TDataSink sink; + sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK); + sink.__set_iceberg_table_sink(TIcebergTableSink()); + return sink; +} + +} // namespace + +class VIcebergTableWriterLifecycleTest : public testing::Test { +protected: + static Status select_block(VIcebergTableWriter* writer, Block& input, + const IColumn::Permutation& rows, Block* selected) { + return writer->_select_block(input, rows, selected); + } + + static void add_writer(VIcebergTableWriter* writer, std::string partition) { + writer->_partitions_to_writers.emplace(std::move(partition), + std::make_shared()); + } + + static void add_writer(VIcebergTableWriter* writer, std::string partition, + std::shared_ptr partition_writer) { + writer->_partitions_to_writers.emplace(std::move(partition), std::move(partition_writer)); + } + + static void clear_writers(VIcebergTableWriter* writer) { + writer->_partitions_to_writers.clear(); + } + + static void publish_active_writers(VIcebergTableWriter* writer) { + writer->_publish_active_writers(); + } +}; + +TEST_F(VIcebergTableWriterLifecycleTest, RejectsCoordinatorWithoutExternalFileReportAck) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + RuntimeState state; + RuntimeProfile profile("test"); + + Status status = writer.open(&state, &profile); + + EXPECT_TRUE(status.is()); + EXPECT_NE(std::string::npos, status.to_string().find("acknowledges external-file reports")); +} + +TEST_F(VIcebergTableWriterLifecycleTest, SelectBlockUsesRowPermutation) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + auto values = ColumnInt32::create(); + values->insert_value(10); + values->insert_value(20); + values->insert_value(30); + Block input; + input.insert({std::move(values), std::make_shared(), "value"}); + IColumn::Permutation rows {2, 0}; + Block selected; + + ASSERT_TRUE(select_block(&writer, input, rows, &selected).ok()); + + const auto& result = assert_cast(*selected.get_by_position(0).column); + ASSERT_EQ(result.size(), 2); + EXPECT_EQ(result.get_element(0), 30); + EXPECT_EQ(result.get_element(1), 10); +} + +TEST_F(VIcebergTableWriterLifecycleTest, ColdReserveCoversManyRealPartitionSelections) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + auto values = ColumnString::create(); + for (size_t i = 0; i < 128; ++i) { + const std::string value(256 + i, static_cast('a' + i % 26)); + values->insert_data(value.data(), value.size()); + } + Block input; + input.insert({std::move(values), std::make_shared(), "value"}); + size_t selected_bytes = 0; + for (size_t row = 0; row < input.rows(); ++row) { + Block selected; + ASSERT_TRUE(select_block(&writer, input, {row}, &selected).ok()); + selected_bytes += selected.allocated_bytes(); + } + + const size_t reserve = iceberg_cold_writer_reserve_size(input, 0); + EXPECT_GE(reserve, + input.allocated_bytes() + 2 * selected_bytes + input.rows() * sizeof(size_t)); +} + +TEST_F(VIcebergTableWriterLifecycleTest, ActiveWriterSnapshotContainsEveryOpenPartition) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + add_writer(&writer, "p=1"); + add_writer(&writer, "p=2"); + + publish_active_writers(&writer); + + ASSERT_NE(writer.active_writers(), nullptr); + EXPECT_EQ(writer.active_writers()->size(), 2); +} + +TEST_F(VIcebergTableWriterLifecycleTest, LoadedSnapshotRetainsWritersDuringConcurrentPublication) { + VIcebergTableWriter writer(make_sink(), {}, nullptr, nullptr); + std::atomic destroyed = 0; + add_writer(&writer, "p=1", std::make_shared(&destroyed)); + publish_active_writers(&writer); + std::promise snapshot_loaded; + std::promise replacement_published; + + auto reader = std::async(std::launch::async, [&]() { + auto snapshot = writer.active_writers(); + snapshot_loaded.set_value(); + replacement_published.get_future().wait(); + EXPECT_EQ(1, snapshot->size()); + EXPECT_EQ("fake", snapshot->front()->file_name()); + }); + + snapshot_loaded.get_future().wait(); + clear_writers(&writer); + publish_active_writers(&writer); + EXPECT_EQ(0, destroyed.load()); + replacement_published.set_value(); + reader.get(); + EXPECT_EQ(1, destroyed.load()); +} + +} // namespace doris diff --git a/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp new file mode 100644 index 00000000000000..73862ac8b0580d --- /dev/null +++ b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "exec/pipeline/pipeline_fragment_context.h" +#include "exec/sink/writer/vhive_partition_writer.h" +#include "format/transformer/vfile_format_transformer.h" +#include "io/fs/s3_file_system.h" +#include "io/fs/s3_file_writer.h" +#include "runtime/exec_env.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris { +namespace { + +class RecordingObjStorageClient final : public io::ObjStorageClient { +public: + io::ObjectStorageUploadResponse create_multipart_upload( + const io::ObjectStoragePathOptions&) override { + return {.resp = io::ObjectStorageResponse::OK(), .upload_id = "upload-id"}; + } + + io::ObjectStorageResponse put_object(const io::ObjectStoragePathOptions&, + std::string_view) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageUploadResponse upload_part(const io::ObjectStoragePathOptions&, + std::string_view, int) override { + return {.resp = io::ObjectStorageResponse::OK(), .etag = "etag"}; + } + + io::ObjectStorageResponse complete_multipart_upload( + const io::ObjectStoragePathOptions&, + const std::vector&) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageHeadResponse head_object(const io::ObjectStoragePathOptions&) override { + return {.resp = io::ObjectStorageResponse::OK(), .file_size = 0}; + } + + io::ObjectStorageResponse get_object(const io::ObjectStoragePathOptions&, void*, size_t, size_t, + size_t*) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse list_objects(const io::ObjectStoragePathOptions&, + std::vector*) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse delete_objects(const io::ObjectStoragePathOptions&, + std::vector) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse delete_object(const io::ObjectStoragePathOptions&) override { + return io::ObjectStorageResponse::OK(); + } + + io::ObjectStorageResponse delete_objects_recursively( + const io::ObjectStoragePathOptions&) override { + return io::ObjectStorageResponse::OK(); + } + + std::string generate_presigned_url(const io::ObjectStoragePathOptions&, int64_t, + const S3ClientConf&) override { + return {}; + } +}; + +class FixedLengthTransformer final : public VFileFormatTransformer { +public: + explicit FixedLengthTransformer(const VExprContextSPtrs& output_exprs) + : VFileFormatTransformer(nullptr, output_exprs, false) {} + + Status open() override { return Status::OK(); } + Status write(const Block&) override { return Status::OK(); } + Status close() override { return Status::OK(); } + int64_t written_len() override { return 64; } +}; + +std::unique_ptr create_closed_hive_writer( + RuntimeState* state, const VExprContextSPtrs& output_exprs, + const std::shared_ptr& client, + io::ObjStorageType provider = io::ObjStorageType::AWS, std::string staged_block_id = {}) { + THiveTableSink hive_sink; + TDataSink sink; + sink.__set_type(TDataSinkType::HIVE_TABLE_SINK); + sink.__set_hive_table_sink(hive_sink); + VHivePartitionWriter::WriteInfo write_info {.write_path = "s3://bucket/staging", + .original_write_path = "s3://bucket/table", + .target_path = "s3://bucket/table", + .file_type = TFileType::FILE_S3, + .broker_addresses = {}}; + static const std::map hadoop_conf; + auto writer = std::make_unique( + sink, "", TUpdateMode::APPEND, output_exprs, std::vector {}, + std::move(write_info), "part", 0, TFileFormatType::FORMAT_PARQUET, + TFileCompressType::PLAIN, nullptr, hadoop_conf); + + S3ClientConf client_conf; + client_conf.provider = provider; + auto holder = std::make_shared(client_conf); + holder->_client = client; + io::FileWriterOptions options {.used_by_s3_committer = true}; + auto file_writer = + std::make_unique(holder, "bucket", "table/part.parquet", &options); + file_writer->_obj_storage_path_opts.upload_id = "upload-id"; + if (!staged_block_id.empty()) { + file_writer->_completed_parts.push_back( + {.part_num = 1, .etag = std::move(staged_block_id)}); + } + file_writer->_state = io::FileWriter::State::CLOSED; + writer->_file_writer = std::move(file_writer); + writer->_file_format_transformer = std::make_unique(output_exprs); + writer->_state = state; + EXPECT_TRUE(writer->close(Status::OK()).ok()); + return writer; +} + +std::shared_ptr create_fragment_context(TUniqueId query_id) { + auto query_ctx = MockQueryContext::create(query_id); + return std::make_shared(query_id, TPipelineFragmentParams(), query_ctx, + ExecEnv::GetInstance(), + [](RuntimeState*, Status*) {}); +} + +ReportStatusRequest report_request(RuntimeState* state, bool done) { + TNetworkAddress address; + address.hostname = "external"; + return {.status = Status::OK(), + .runtime_states = {}, + .done = done, + .coord_addr = address, + .query_id = TUniqueId(), + .fragment_id = 0, + .fragment_instance_id = TUniqueId(), + .backend_num = 0, + .runtime_state = state, + .load_error_url = "", + .first_error_msg = "", + .cancel_fn = [](const Status&) {}}; +} + +} // namespace + +TEST(VHivePartitionWriterReportLifecycleTest, + PeriodicReportDefersMetadataAndFinalReportTransfersUploadIdentity) { + MockRuntimeState state; + VExprContextSPtrs output_exprs; + auto client = std::make_shared(); + auto writer = create_closed_hive_writer(&state, output_exprs, client); + auto context = create_fragment_context(TUniqueId()); + + TReportExecStatusParams periodic_params; + auto periodic_request = report_request(&state, false); + context->_append_external_file_commit_data(periodic_request, &periodic_params); + + EXPECT_FALSE(periodic_params.__isset.hive_partition_updates); + + TReportExecStatusParams final_params; + auto final_request = report_request(&state, true); + context->_append_external_file_commit_data(final_request, &final_params); + ASSERT_TRUE(final_params.__isset.hive_partition_updates); + ASSERT_EQ(1, final_params.hive_partition_updates.size()); + ASSERT_TRUE(final_params.hive_partition_updates[0].__isset.s3_mpu_pending_uploads); + ASSERT_EQ(1, final_params.hive_partition_updates[0].s3_mpu_pending_uploads.size()); + const auto& pending_upload = final_params.hive_partition_updates[0].s3_mpu_pending_uploads[0]; + EXPECT_EQ("bucket", pending_upload.bucket); + EXPECT_EQ("table/part.parquet", pending_upload.key); + EXPECT_EQ("upload-id", pending_upload.upload_id); +} + +TEST(VHivePartitionWriterReportLifecycleTest, AzureFinalReportCarriesExactBlockIdentity) { + MockRuntimeState state; + VExprContextSPtrs output_exprs; + auto client = std::make_shared(); + auto writer = create_closed_hive_writer(&state, output_exprs, client, io::ObjStorageType::AZURE, + "exact-block-id"); + auto context = create_fragment_context(TUniqueId()); + auto final_request = report_request(&state, true); + TReportExecStatusParams final_params; + + context->_append_external_file_commit_data(final_request, &final_params); + + ASSERT_TRUE(final_params.__isset.hive_partition_updates); + ASSERT_EQ(1, final_params.hive_partition_updates.size()); + const auto& pending_uploads = final_params.hive_partition_updates[0].s3_mpu_pending_uploads; + ASSERT_EQ(1, pending_uploads.size()); + EXPECT_EQ("upload-id", pending_uploads[0].upload_id); + EXPECT_EQ("exact-block-id", pending_uploads[0].etags.at(1)); +} + +} // namespace doris diff --git a/be/test/exec/sort/full_sort_test.cpp b/be/test/exec/sort/full_sort_test.cpp index e182048c807dad..bd8f91b03cc863 100644 --- a/be/test/exec/sort/full_sort_test.cpp +++ b/be/test/exec/sort/full_sort_test.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -63,6 +64,13 @@ struct FullSorterTest : public testing::Test { std::vector nulls_first {false}; }; +TEST(SorterReserveMemoryTest, TotalSaturatesOnOverflow) { + SorterReserveMemory reservation {.retained_growth = std::numeric_limits::max() - 1, + .transient_workspace = 2}; + + EXPECT_EQ(std::numeric_limits::max(), reservation.total()); +} + TEST_F(FullSorterTest, test_full_sorter1) { sorter = FullSorter::create_unique(ordering_expr_ctxs, -1, 0, &pool, is_asc_order, nulls_first, *row_desc, &_state, nullptr); @@ -94,6 +102,20 @@ TEST_F(FullSorterTest, test_full_sorter2) { std::cout << sorter->get_reserve_mem_size(&_state, false) << std::endl; } +TEST_F(FullSorterTest, EosReservationIncludesForcedSortBelowAppendThresholds) { + sorter = FullSorter::create_unique(ordering_expr_ctxs, -1, 0, &pool, is_asc_order, nulls_first, + *row_desc, &_state, nullptr); + Block block = ColumnHelper::create_block({10, 9, 8, 7, 6, 5, 4, 3, 2, 1}); + const size_t buffered_bytes = block.bytes(); + const size_t buffered_rows = block.rows(); + ASSERT_TRUE(sorter->append_block(&block).ok()); + + const auto reservation = sorter->get_reserve_mem_size_components(&_state, true, 0, 0); + + EXPECT_GE(reservation.transient_workspace, + buffered_bytes + buffered_rows * sizeof(IColumn::Permutation::value_type)); +} + TEST_F(FullSorterTest, test_full_sorter3) { sorter = FullSorter::create_unique(ordering_expr_ctxs, 3, 3, &pool, is_asc_order, nulls_first, *row_desc, &_state, nullptr); @@ -113,4 +135,4 @@ TEST_F(FullSorterTest, test_full_sorter3) { EXPECT_EQ(sorter->_state->get_sorted_block()[1]->rows(), 4); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/format/table/iceberg/schema_test.cpp b/be/test/format/table/iceberg/schema_test.cpp index bccf0f1f3418b2..91c9ae778b308a 100644 --- a/be/test/format/table/iceberg/schema_test.cpp +++ b/be/test/format/table/iceberg/schema_test.cpp @@ -19,6 +19,10 @@ #include +#include + +#include "format/table/iceberg_default_value.h" + namespace doris { namespace iceberg { @@ -66,5 +70,32 @@ TEST(SchemaTest, test_find_field) { EXPECT_EQ(found_field2->field_id(), 2); } +TEST(SchemaTest, FindNestedFieldPath) { + std::vector children; + children.emplace_back(true, 2, "part", std::make_unique(), std::nullopt); + std::vector columns; + columns.emplace_back(true, 1, "payload", std::make_unique(std::move(children)), + std::nullopt); + Schema schema(1, std::move(columns)); + + const auto* path = schema.find_field_path(2); + ASSERT_NE(path, nullptr); + ASSERT_EQ(path->size(), 2); + EXPECT_EQ((*path)[0]->field_id(), 1); + EXPECT_EQ((*path)[1]->field_id(), 2); + EXPECT_EQ(schema.find_type(2)->type_id(), TypeID::INTEGER); +} + +TEST(SchemaTest, ParsesIcebergNonFiniteDefaults) { + Field value; + EXPECT_TRUE(detail::parse_non_finite_default(TYPE_FLOAT, "NaN", &value)); + EXPECT_TRUE(std::isnan(value.get())); + EXPECT_TRUE(detail::parse_non_finite_default(TYPE_DOUBLE, "Infinity", &value)); + EXPECT_TRUE(std::isinf(value.get())); + EXPECT_GT(value.get(), 0); + EXPECT_TRUE(detail::parse_non_finite_default(TYPE_DOUBLE, "-Infinity", &value)); + EXPECT_LT(value.get(), 0); +} + } // namespace iceberg } // namespace doris diff --git a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp index a219ba37f3dd8d..6db534fa06d2a7 100644 --- a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp @@ -19,6 +19,8 @@ #include +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" @@ -59,5 +61,28 @@ TEST(IcebergPositionDeleteSysTableV2ProfileTest, UsesDistinctProfileForNestedPos reader._position_reader_profile); } +TEST(IcebergPositionDeleteSysTableV2ProfileTest, PreparesNestedRowInitialDefaults) { + IcebergPositionDeleteSysTableV2Reader reader; + const auto child_type = make_nullable(std::make_shared()); + const auto row_type = make_nullable( + std::make_shared(DataTypes {child_type}, Strings {"added"})); + ColumnDefinition row; + row.name = "row"; + row.type = row_type; + ColumnDefinition child; + child.name = "added"; + child.type = child_type; + child.initial_default_value = "7"; + row.children.push_back(std::move(child)); + reader._projected_columns = {row}; + reader._read_columns = {{"row", row_type}}; + + std::vector columns; + ASSERT_TRUE(reader._build_delete_file_projected_columns(&columns).ok()); + ASSERT_EQ(columns.size(), 1); + ASSERT_EQ(columns[0].children.size(), 1); + EXPECT_NE(columns[0].children[0].default_expr, nullptr); +} + } // namespace } // namespace doris::format::iceberg diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 9218af0b48ada0..923caf2f8daab7 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1116,6 +1116,9 @@ class TableReaderCastTestHelper final : public TableReader { using TableReader::_materialize_map_mapping_column; using TableReader::_materialize_present_child_mapping_column; using TableReader::_materialize_struct_mapping_column; + using TableReader::_project_collection_parent_null_map_for_hidden_entries; + using TableReader::_requires_collection_parent_null_map; + using TableReader::_requires_parent_null_map_for_alignment; }; TEST(TableReaderTest, TruncateCharOrVarcharPredicateOnlyAppliesToParquetStringWidthMismatch) { @@ -6345,5 +6348,223 @@ TEST(TableReaderTest, ProjectedColumnsUseMapperExpressionsForParquetSchemaMismat std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, CollectionParentMaskSkipsLargeArrayWhenNearerMaskCoversVisibleNull) { + constexpr size_t visible_entries = 500000; + const size_t entries = visible_entries + 1; + const auto int_type = std::make_shared(); + const auto struct_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto nullable_struct_type = make_nullable(struct_type); + + auto values = ColumnInt32::create(entries, 0); + auto value_null_map = ColumnUInt8::create(entries, 0); + value_null_map->get_data().back() = 1; + MutableColumns struct_children; + struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto element_null_map = ColumnUInt8::create(entries, 0); + element_null_map->get_data().back() = 1; + ColumnPtr nullable_elements = ColumnNullable::create( + ColumnStruct::create(std::move(struct_children)), std::move(element_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {1, entries}; + + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_elements, nullable_struct_type, 2, offsets)); +} + +TEST(TableReaderTest, CollectionParentMaskSkipsLargeMapWhenNearerMaskCoversVisibleNull) { + constexpr size_t visible_entries = 500000; + const size_t entries = visible_entries + 1; + const auto int_type = std::make_shared(); + const auto struct_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto nullable_struct_type = make_nullable(struct_type); + + auto values = ColumnInt32::create(entries, 0); + auto value_null_map = ColumnUInt8::create(entries, 0); + value_null_map->get_data().back() = 1; + MutableColumns struct_children; + struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto value_struct_null_map = ColumnUInt8::create(entries, 0); + value_struct_null_map->get_data().back() = 1; + ColumnPtr nullable_value_structs = ColumnNullable::create( + ColumnStruct::create(std::move(struct_children)), std::move(value_struct_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {1, entries}; + + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_value_structs, nullable_struct_type, 2, offsets)); +} + +TEST(TableReaderTest, CollectionParentMaskKeepsRequiredNullInHiddenEntry) { + const auto int_type = std::make_shared(); + const auto struct_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto nullable_struct_type = make_nullable(struct_type); + + auto values = ColumnInt32::create(2, 0); + auto value_null_map = ColumnUInt8::create(); + value_null_map->get_data().assign({1, 1}); + MutableColumns struct_children; + struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto element_null_map = ColumnUInt8::create(); + element_null_map->get_data().assign({0, 1}); + ColumnPtr nullable_elements = ColumnNullable::create( + ColumnStruct::create(std::move(struct_children)), std::move(element_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {1, 2}; + + EXPECT_TRUE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_elements, nullable_struct_type, 2, offsets)); +} + +TEST(TableReaderTest, TrivialArrayChildProjectsNullableStructParentMask) { + const auto int_type = std::make_shared(); + const auto element_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto nullable_element_struct_type = make_nullable(element_struct_type); + const auto array_type = std::make_shared(nullable_element_struct_type); + const auto table_struct_type = make_nullable(std::make_shared( + DataTypes {array_type, int_type}, Strings {"items", "added"})); + const auto file_struct_type = make_nullable( + std::make_shared(DataTypes {array_type}, Strings {"items"})); + + auto table_items = make_table_column(0, "items", array_type); + table_items.type = array_type; + auto table_element = make_table_column(0, "element", element_struct_type); + table_element.type = element_struct_type; + auto table_value = make_table_column(0, "value", int_type); + table_value.type = int_type; + table_element.children = {table_value}; + table_items.children = {table_element}; + auto table_added = make_table_column(1, "added", int_type); + table_added.type = int_type; + auto table_struct = make_table_column(0, "payload", table_struct_type); + table_struct.type = table_struct_type; + table_struct.children = {table_items, table_added}; + + auto file_items = make_file_column(0, "items", array_type); + file_items.type = array_type; + auto file_element = make_file_column(0, "element", element_struct_type); + file_element.type = element_struct_type; + auto file_value = make_file_column(0, "value", int_type); + file_value.type = int_type; + file_element.children = {file_value}; + file_items.children = {file_element}; + auto file_struct = make_file_column(0, "payload", file_struct_type); + file_struct.type = file_struct_type; + file_struct.children = {file_items}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_FALSE(mapper.mappings()[0].is_trivial); + ASSERT_TRUE(mapper.mappings()[0].child_mappings[0].is_trivial); + + auto values = ColumnInt32::create(); + values->get_data().assign({0, 7}); + auto value_null_map = ColumnUInt8::create(); + value_null_map->get_data().assign({1, 0}); + MutableColumns element_children; + element_children.push_back( + ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto element_null_map = ColumnUInt8::create(2, 0); + auto array_values = ColumnNullable::create(ColumnStruct::create(std::move(element_children)), + std::move(element_null_map)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->get_data().assign({1, 2}); + MutableColumns struct_children; + struct_children.push_back(ColumnArray::create(std::move(array_values), std::move(offsets))); + auto parent_null_map = ColumnUInt8::create(); + parent_null_map->get_data().assign({1, 0}); + ColumnPtr file_data = ColumnNullable::create(ColumnStruct::create(std::move(struct_children)), + std::move(parent_null_map)); + + TableReaderCastTestHelper reader; + ColumnPtr result; + const auto status = + reader._materialize_struct_mapping_column(mapper.mappings()[0], file_data, 2, &result); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& result_struct = assert_cast( + assert_cast(*result).get_nested_column()); + const auto& result_array = assert_cast(result_struct.get_column(0)); + const auto& result_elements = assert_cast(result_array.get_data()); + const auto& result_element_struct = + assert_cast(result_elements.get_nested_column()); + ASSERT_FALSE(result_element_struct.get_column(0).is_nullable()); + EXPECT_EQ(assert_cast(result_element_struct.get_column(0)).get_element(1), + 7); +} + +TEST(TableReaderTest, ParentMaskProjectionOnlyWhenRequiredDescendantCanConsumeIt) { + const auto int_type = std::make_shared(); + auto mutable_values = ColumnInt32::create(); + mutable_values->get_data().assign({1, 2}); + ColumnPtr values = std::move(mutable_values); + EXPECT_FALSE( + TableReaderCastTestHelper::_requires_parent_null_map_for_alignment(values, int_type)); + + ColumnPtr nullable_values = ColumnNullable::create(values->clone(), ColumnUInt8::create(2, 0)); + EXPECT_FALSE(TableReaderCastTestHelper::_requires_parent_null_map_for_alignment(nullable_values, + int_type)); + auto null_map = ColumnUInt8::create(2, 0); + null_map->get_data()[0] = 1; + ColumnPtr nullable_values_with_null = + ColumnNullable::create(values->clone(), std::move(null_map)); + EXPECT_TRUE(TableReaderCastTestHelper::_requires_parent_null_map_for_alignment( + nullable_values_with_null, int_type)); + + NullMap all_clear_parent_mask(2, 0); + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &all_clear_parent_mask, nullable_values_with_null, int_type)); + NullMap hidden_parent_mask {1, 0}; + EXPECT_TRUE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &hidden_parent_mask, nullable_values_with_null, int_type)); + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + nullptr, nullable_values_with_null, int_type)); +} + +TEST(TableReaderTest, CollectionParentMaskSkipsLargeArrayWhenOnlyEmptyRowIsHidden) { + constexpr size_t entries = 500000; + auto values = ColumnInt32::create(entries, 0); + auto value_null_map = ColumnUInt8::create(entries, 0); + value_null_map->get_data().back() = 1; + ColumnPtr nullable_values = + ColumnNullable::create(std::move(values), std::move(value_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {0, entries}; + NullMap projected_null_map; + + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_values, std::make_shared(), 2, offsets)); + const auto* result = + TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries( + nullptr, &parent_null_map, 2, offsets, entries, &projected_null_map); + + EXPECT_EQ(nullptr, result); + EXPECT_TRUE(projected_null_map.empty()); +} + +TEST(TableReaderTest, CollectionParentMaskSkipsLargeMapWhenOnlyEmptyRowIsHidden) { + constexpr size_t entries = 500000; + auto values = ColumnInt32::create(entries, 0); + auto value_null_map = ColumnUInt8::create(entries, 0); + value_null_map->get_data().back() = 1; + ColumnPtr nullable_values = + ColumnNullable::create(std::move(values), std::move(value_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {0, entries}; + NullMap projected_null_map; + + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_values, std::make_shared(), 2, offsets)); + const auto* result = + TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries( + nullptr, &parent_null_map, 2, offsets, entries, &projected_null_map); + + EXPECT_EQ(nullptr, result); + EXPECT_TRUE(projected_null_map.empty()); +} + } // namespace } // namespace doris::format diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index 71034020120d12..18f5c904e2b4f6 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -925,9 +925,8 @@ TEST_F(S3FileSystemTest, RateLimiterGetDownloadTest) { // Test: S3 rate limiter for PUT operations - multipart upload TEST_F(S3FileSystemTest, RateLimiterPutMultipartTest) { - // Skip if using Azure provider - Azure's create_multipart_upload is a no-op and doesn't - // consume rate limiter quota, while S3's CreateMultipartUpload does. This causes different - // failure timing that makes the test assertions invalid for Azure. + // This test asserts the S3 provider's exact multipart request/failure sequence; Azure uses + // lease coordination and therefore has different failure timing. if (config_->get_provider() == "AZURE") { GTEST_SKIP() << "This test relies on S3-specific multipart upload quota consumption " "behavior, not applicable for Azure"; @@ -1434,9 +1433,8 @@ TEST_F(S3FileSystemTest, RateLimiterGetDeleteDirectoryListTest) { // Test: S3 rate limiter for PUT operations - multipart upload with UploadPart failure TEST_F(S3FileSystemTest, RateLimiterPutMultipartUploadPartFailureTest) { - // Skip if using Azure provider - Azure's create_multipart_upload is a no-op and doesn't - // consume rate limiter quota, while S3's CreateMultipartUpload does. This causes different - // failure timing that makes the test assertions invalid for Azure. + // This test asserts the S3 provider's exact multipart request/failure sequence; Azure uses + // lease coordination and therefore has different failure timing. if (config_->get_provider() == "AZURE") { GTEST_SKIP() << "This test relies on S3-specific multipart upload quota consumption " "behavior, not applicable for Azure"; @@ -1863,8 +1861,7 @@ TEST_F(S3FileSystemTest, RateLimiterPutDeleteDirectoryDeleteObjectsTest) { // Test: S3 CreateMultipartUpload failure - simulates error when initiating multipart upload TEST_F(S3FileSystemTest, CreateMultipartUploadFailureTest) { - // Skip if using Azure provider - SyncPoint mechanism is S3-specific - // Also, Azure's create_multipart_upload is a no-op that always succeeds + // Skip if using Azure provider because the SyncPoint mechanism is S3-specific. if (config_->get_provider() == "AZURE") { GTEST_SKIP() << "This test uses S3-specific SyncPoint mechanism and multipart semantics, " "not applicable for Azure"; diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 7591b4bf2ea997..a45db87bfe0268 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -19,11 +19,17 @@ #include +#include +#include +#include + #include "io/fs/file_system.h" #include "io/fs/obj_storage_client.h" #include "util/s3_util.h" #ifdef USE_AZURE +#include + #include #include #include @@ -34,6 +40,41 @@ namespace doris { #ifdef USE_AZURE +TEST(AzureObjStorageClientMultipartHelperTest, full_upload_uuid_isolates_writer_blocks) { + constexpr std::string_view first_upload = "09492e3d-e231-4ed9-bf84-b6fc772cda54"; + constexpr std::string_view second_upload = "06996d15-1c2e-4ddd-8853-43816ea84a07"; + auto first_block = io::azure_multipart_block_id(first_upload, 1); + auto second_block = io::azure_multipart_block_id(second_upload, 1); + + EXPECT_NE(first_block, second_block); + EXPECT_EQ(first_block.size(), io::azure_multipart_block_id(first_upload, 999).size()); + auto decoded = Aws::Utils::HashingUtils::Base64Decode(first_block); + ASSERT_EQ(first_upload.size() + sizeof(uint32_t), decoded.GetLength()); + EXPECT_EQ(first_upload, + std::string_view(reinterpret_cast(decoded.GetUnderlyingData()), + first_upload.size())); + EXPECT_EQ(1, decoded.GetUnderlyingData()[first_upload.size()]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 1]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 2]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 3]); +} + +TEST(AzureObjStorageClientMultipartHelperTest, create_upload_is_provider_free) { + io::AzureObjStorageClient client( + std::shared_ptr {}); + + auto first = client.create_multipart_upload({}); + auto second = client.create_multipart_upload({}); + + ASSERT_EQ(ErrorCode::OK, first.resp.status.code); + ASSERT_EQ(ErrorCode::OK, second.resp.status.code); + ASSERT_TRUE(first.upload_id.has_value()); + ASSERT_TRUE(second.upload_id.has_value()); + EXPECT_EQ(36, first.upload_id->size()); + EXPECT_EQ(36, second.upload_id->size()); + EXPECT_NE(first.upload_id, second.upload_id); +} + using namespace Azure::Storage::Blobs; TEST(AzureObjStorageClientTlsHelperTest, detects_tls_ca_error) { @@ -156,6 +197,39 @@ TEST_F(AzureObjStorageClientTest, delete_objects_recursively) { EXPECT_EQ(response.status.code, ErrorCode::OK); EXPECT_EQ(files.size(), 0); } + +TEST_F(AzureObjStorageClientTest, concurrent_multipart_uploads_do_not_share_staged_blocks) { + io::ObjectStoragePathOptions first {.key = "AzureObjStorageClientTest/concurrent_multipart"}; + io::ObjectStoragePathOptions second = first; + auto first_create = obj_storage_client->create_multipart_upload(first); + auto second_create = obj_storage_client->create_multipart_upload(second); + ASSERT_EQ(first_create.resp.status.code, ErrorCode::OK); + ASSERT_EQ(second_create.resp.status.code, ErrorCode::OK); + ASSERT_TRUE(first_create.upload_id.has_value()); + ASSERT_TRUE(second_create.upload_id.has_value()); + ASSERT_NE(first_create.upload_id, second_create.upload_id); + first.upload_id = first_create.upload_id; + second.upload_id = second_create.upload_id; + + auto first_part = obj_storage_client->upload_part(first, "first", 1); + auto second_part = obj_storage_client->upload_part(second, "second", 1); + ASSERT_EQ(first_part.resp.status.code, ErrorCode::OK); + ASSERT_EQ(second_part.resp.status.code, ErrorCode::OK); + ASSERT_NE(first_part.etag, second_part.etag); + ASSERT_EQ(obj_storage_client->complete_multipart_upload(first, {{.part_num = 1}}).status.code, + ErrorCode::OK); + ASSERT_NE(obj_storage_client->complete_multipart_upload(second, {{.part_num = 1}}).status.code, + ErrorCode::OK); + + std::array contents {}; + size_t size_return = 0; + ASSERT_EQ(obj_storage_client + ->get_object(second, contents.data(), 0, contents.size(), &size_return) + .status.code, + ErrorCode::OK); + EXPECT_EQ(std::string_view(contents.data(), size_return), "first"); + EXPECT_EQ(obj_storage_client->delete_object(second).status.code, ErrorCode::OK); +} #else class AzureObjStorageClientTest : public testing::Test { diff --git a/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp b/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp index cbefa422ad2a77..d88d2b5ec4bc19 100644 --- a/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp +++ b/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp @@ -330,6 +330,39 @@ TEST_F(ThreadMemTrackerMgrTest, ReserveMemory) { EXPECT_EQ(doris::GlobalMemoryArbitrator::process_reserved_memory(), 0); } +TEST_F(ThreadMemTrackerMgrTest, TransfersReservationBetweenAsyncTasks) { + auto tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, + "UT-TransferReservation"); + auto resource_context = ResourceContext::create_shared(); + resource_context->memory_context()->set_mem_tracker(tracker); + ThreadContext producer; + ThreadContext consumer; + producer.attach_task(resource_context); + consumer.attach_task(resource_context); + constexpr int64_t reservation = 4 * 1024 * 1024; + + ASSERT_TRUE(producer.thread_mem_tracker_mgr->try_reserve(reservation).ok()); + auto token = producer.thread_mem_tracker_mgr->take_reserved_memory(); + EXPECT_EQ(producer.thread_mem_tracker_mgr->reserved_mem(), 0); + EXPECT_EQ(token.bytes(), reservation); + + consumer.thread_mem_tracker_mgr->adopt_reserved_memory(std::move(token)); + EXPECT_EQ(consumer.thread_mem_tracker_mgr->reserved_mem(), reservation); + consumer.thread_mem_tracker_mgr->consume(reservation); + EXPECT_EQ(consumer.thread_mem_tracker_mgr->reserved_mem(), 0); + + ASSERT_TRUE(producer.thread_mem_tracker_mgr->try_reserve(reservation).ok()); + { + auto abandoned = producer.thread_mem_tracker_mgr->take_reserved_memory(); + EXPECT_EQ(abandoned.bytes(), reservation); + } + EXPECT_EQ(GlobalMemoryArbitrator::process_reserved_memory(), 0); + + producer.detach_task(); + consumer.detach_task(); + EXPECT_EQ(GlobalMemoryArbitrator::process_reserved_memory(), 0); +} + TEST_F(ThreadMemTrackerMgrTest, NestedReserveMemory) { std::unique_ptr thread_context = std::make_unique(); std::shared_ptr t = MemTrackerLimiter::create_shared( diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index 22ebc5ebf8a0ee..5a384378ec382b 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -18,12 +18,131 @@ #include #include "common/config.h" +#include "exec/pipeline/report_exec_status_size.h" #include "runtime/runtime_state.h" #include "testutil/mock/mock_runtime_state.h" #include "util/block_budget.h" namespace doris { +TEST(RuntimeStateIcebergCommitDataTest, RejectsMetadataBeforeItCanExceedTheThriftLimit) { + RuntimeState state; + const int32_t saved_limit = config::thrift_max_message_size; + config::thrift_max_message_size = 128; + TIcebergCommitData commit_data; + commit_data.__set_file_path(std::string(256, 'x')); + + Status status = state.add_iceberg_commit_datas(commit_data); + + config::thrift_max_message_size = saved_limit; + EXPECT_FALSE(status.ok()); + std::vector collected; + state.append_iceberg_commit_datas(&collected); + EXPECT_TRUE(collected.empty()); +} + +TEST(RuntimeStateIcebergCommitDataTest, SharesTheReportBudgetAcrossParallelTasks) { + RuntimeState first; + RuntimeState second; + auto budget = std::make_shared(); + first.set_external_file_report_state(budget); + second.set_external_file_report_state(budget); + const int32_t saved_limit = config::thrift_max_message_size; + config::thrift_max_message_size = 1024 * 1024 + 512; + TIcebergCommitData commit_data; + commit_data.__set_file_path(std::string(300, 'x')); + + Status first_status = first.add_iceberg_commit_datas(commit_data); + Status second_status = second.add_iceberg_commit_datas(commit_data); + + config::thrift_max_message_size = saved_limit; + EXPECT_TRUE(first_status.ok()) << first_status; + EXPECT_FALSE(second_status.ok()); +} + +TEST(RuntimeStateIcebergCommitDataTest, UsesTheSmallerCoordinatorThriftLimit) { + RuntimeState state; + const int32_t saved_limit = config::thrift_max_message_size; + config::thrift_max_message_size = 4 * 1024 * 1024; + state._query_options.__set_coordinator_thrift_max_message_size(1024 * 1024 + 128); + TIcebergCommitData commit_data; + commit_data.__set_file_path(std::string(256, 'x')); + + Status status = state.add_iceberg_commit_datas(commit_data); + + config::thrift_max_message_size = saved_limit; + EXPECT_FALSE(status.ok()); +} + +TEST(RuntimeStateIcebergCommitDataTest, ValidatesTheCompleteReportEnvelope) { + TReportExecStatusParams params; + params.__set_error_log({std::string(2 * 1024 * 1024, 'x')}); + + EXPECT_FALSE(validate_report_exec_status_size(params, 1024 * 1024).ok()); + EXPECT_TRUE(validate_report_exec_status_size(params, 3 * 1024 * 1024).ok()); +} + +TEST(RuntimeStateIcebergCommitDataTest, PeriodicReportOmitsExternalCommitData) { + RuntimeState state; + THivePartitionUpdate hive_update; + state.add_hive_partition_updates(hive_update); + TIcebergCommitData iceberg_data; + iceberg_data.__set_file_path("data.parquet"); + ASSERT_TRUE(state.add_iceberg_commit_datas(iceberg_data).ok()); + TMCCommitData mc_data; + state.add_mc_commit_datas(mc_data); + TReportExecStatusParams periodic_params; + + state.append_external_file_commit_data(&periodic_params, false); + + EXPECT_FALSE(periodic_params.__isset.hive_partition_updates); + EXPECT_FALSE(periodic_params.__isset.iceberg_commit_datas); + EXPECT_FALSE(periodic_params.__isset.mc_commit_datas); + + TReportExecStatusParams final_params; + state.append_external_file_commit_data(&final_params, true); + EXPECT_TRUE(final_params.__isset.hive_partition_updates); + EXPECT_TRUE(final_params.__isset.iceberg_commit_datas); + EXPECT_TRUE(final_params.__isset.mc_commit_datas); +} + +TEST(RuntimeStateIcebergCommitDataTest, RetainsFileCleanupUntilReportAcknowledgement) { + RuntimeState coordinator_state; + RuntimeState task_state; + auto report_state = std::make_shared(); + coordinator_state.set_external_file_report_state(report_state); + task_state.set_external_file_report_state(report_state); + int cleanup_count = 0; + task_state.add_rejected_external_file_report_cleanup([&] { ++cleanup_count; }); + + coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + + EXPECT_EQ(1, cleanup_count); + + task_state.add_rejected_external_file_report_cleanup([&] { ++cleanup_count; }); + coordinator_state.finalize_external_file_report_cleanup( + ExternalFileReportOutcome::ACKNOWLEDGED); + EXPECT_EQ(1, cleanup_count); + + task_state.add_rejected_external_file_report_cleanup([&] { ++cleanup_count; }); + coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::AMBIGUOUS); + EXPECT_EQ(1, cleanup_count); + coordinator_state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + EXPECT_EQ(1, cleanup_count); +} + +TEST(RuntimeStateIcebergCommitDataTest, AmbiguousOwnershipCannotBecomeRejected) { + RuntimeState state; + int cleanup_count = 0; + state.add_rejected_external_file_report_cleanup([&] { ++cleanup_count; }); + + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::AMBIGUOUS); + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + + EXPECT_EQ(0, cleanup_count); +} + // --------------------------------------------------------------------------- // RuntimeState::batch_size() // --------------------------------------------------------------------------- diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index fc214943541985..5f63e9b7dbe7f9 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -478,6 +478,11 @@ public String hideVersionForVersionColumn(Boolean isToSql) { } public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedComment) { + return hideVersionForVersionColumn(isToSql, showNestedComment, false); + } + + public String hideVersionForVersionColumn( + Boolean isToSql, boolean showNestedComment, boolean noBackslashEscapes) { if (isDatetime() || isDatetimeV2()) { StringBuilder typeStr = new StringBuilder("datetime"); if (((ScalarType) this).getScalarScale() > 0) { @@ -507,13 +512,13 @@ public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedCom return typeStr.toString(); } else if (isArrayType()) { String nestedDesc = ((ArrayType) this).getItemType() - .hideVersionForVersionColumn(isToSql, showNestedComment); + .hideVersionForVersionColumn(isToSql, showNestedComment, noBackslashEscapes); return "array<" + nestedDesc + ">"; } else if (isMapType()) { String keyDesc = ((MapType) this).getKeyType() - .hideVersionForVersionColumn(isToSql, showNestedComment); + .hideVersionForVersionColumn(isToSql, showNestedComment, noBackslashEscapes); String valueDesc = ((MapType) this).getValueType() - .hideVersionForVersionColumn(isToSql, showNestedComment); + .hideVersionForVersionColumn(isToSql, showNestedComment, noBackslashEscapes); return "map<" + keyDesc + "," + valueDesc + ">"; } else if (isStructType()) { List fieldDesc = new ArrayList<>(); @@ -521,7 +526,8 @@ public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedCom for (int i = 0; i < structType.getFields().size(); i++) { StructField field = structType.getFields().get(i); StringBuilder desc = new StringBuilder(field.getName()).append(":") - .append(field.getType().hideVersionForVersionColumn(isToSql, showNestedComment)); + .append(field.getType().hideVersionForVersionColumn( + isToSql, showNestedComment, noBackslashEscapes)); // Requiredness is schema semantics and must survive independently of whether // nested documentation is requested for DESCRIBE output. if (!field.getContainsNull()) { @@ -529,7 +535,9 @@ public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedCom } // Nested docs are part of DESCRIBE output only when comments were explicitly requested. if (showNestedComment && field.isCommentSpecified()) { - desc.append(String.format(" comment '%s'", field.getComment())); + // Comments must remain parseable even when they contain quotes or backslashes. + desc.append(" comment ").append( + quoteStringLiteral(field.getComment(), noBackslashEscapes)); } fieldDesc.add(desc.toString()); } @@ -540,6 +548,13 @@ public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedCom return this.toString(); } + private static String quoteStringLiteral(String value, boolean noBackslashEscapes) { + // DESCRIBE output must stay parseable under the caller's SQL mode even though fe-common + // cannot depend on the parser utility in fe-core. + String escaped = noBackslashEscapes ? value : value.replace("\\", "\\\\"); + return "\"" + escaped.replace("\"", "\"\"") + "\""; + } + public boolean isDecimalV3() { return isScalarType(PrimitiveType.DECIMAL32) || isScalarType(PrimitiveType.DECIMAL64) || isScalarType(PrimitiveType.DECIMAL128) || isScalarType(PrimitiveType.DECIMAL256); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index 7578685a771d64..0b0f7d67277f54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -21,6 +21,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SqlModeHelper; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -89,7 +90,8 @@ public static ProcResult createResult(List schema, Set bfColumns String extraStr = StringUtils.join(extras, ","); List rowList = Lists.newArrayList(column.getDisplayName(), - column.getOriginType().hideVersionForVersionColumn(true, showNestedComment), + column.getOriginType().hideVersionForVersionColumn( + true, showNestedComment, SqlModeHelper.hasNoBackSlashEscapes()), column.isAllowNull() ? "Yes" : "No", ((Boolean) column.isKey()).toString(), column.getDefaultValue() == null diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java index 096f94a4aff3aa..dd40f2efb304b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java @@ -58,6 +58,12 @@ public static String escapeQuota(String str) { return str.replaceAll("\"", "\\\\\""); } + /** Quote a value as a SQL string literal under the requested backslash-escape mode. */ + public static String quoteStringLiteral(String value, boolean noBackslashEscapes) { + String escaped = noBackslashEscapes ? value : value.replace("\\", "\\\\"); + return "\"" + escaped.replace("\"", "\"\"") + "\""; + } + public static List splitMultiStmts(String sql) { DorisSqlSeparatorLexer lexer = new DorisSqlSeparatorLexer(CharStreams.fromString(sql)); CommonTokenStream tokenStream = new CommonTokenStream(lexer); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java index 1ea5b4d3eae4a2..bb181988c64175 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java @@ -162,7 +162,11 @@ public List mergePartitions(List hiv THivePartitionUpdate old = mm.get(pu.getName()); old.setFileSize(old.getFileSize() + pu.getFileSize()); old.setRowCount(old.getRowCount() + pu.getRowCount()); - if (old.getS3MpuPendingUploads() != null && pu.getS3MpuPendingUploads() != null) { + if (pu.getS3MpuPendingUploads() != null && !pu.getS3MpuPendingUploads().isEmpty()) { + // A missing legacy list is empty state, not ownership of later completion records. + if (old.getS3MpuPendingUploads() == null) { + old.setS3MpuPendingUploads(new ArrayList<>()); + } old.getS3MpuPendingUploads().addAll(pu.getS3MpuPendingUploads()); } old.getFileNames().addAll(pu.getFileNames()); @@ -175,11 +179,46 @@ public List mergePartitions(List hiv private void collectUncompletedMpuPendingUploads(List hivePUs) { for (THivePartitionUpdate pu : hivePUs) { - if (pu.getS3MpuPendingUploads() != null) { - for (TS3MPUPendingUpload s3MPUPendingUpload : pu.getS3MpuPendingUploads()) { - uncompletedMpuPendingUploads.add( - new UncompletedMpuPendingUpload(s3MPUPendingUpload, pu.getLocation().getWritePath())); + List uploads = pu.getS3MpuPendingUploads(); + if (uploads == null) { + continue; + } + String writePath = pu.getLocation() == null ? null : pu.getLocation().getWritePath(); + if (Strings.isNullOrEmpty(writePath)) { + // One malformed record must not prevent valid sibling uploads from being cleaned up. + LOG.warn("Skipping MPU cleanup record without a write path"); + continue; + } + for (TS3MPUPendingUpload upload : uploads) { + if (!isCompleteObjectStoreUpload(upload)) { + LOG.warn("Skipping incomplete MPU cleanup record for write path {}", writePath); + continue; } + uncompletedMpuPendingUploads.add(new UncompletedMpuPendingUpload(upload, writePath)); + } + } + } + + private static boolean isCompleteObjectStoreUpload(TS3MPUPendingUpload upload) { + return upload != null && !Strings.isNullOrEmpty(upload.getUploadId()) + && !Strings.isNullOrEmpty(upload.getBucket()) && !Strings.isNullOrEmpty(upload.getKey()); + } + + private void validateObjectStoreCommitRecords() { + if (fileType != TFileType.FILE_S3) { + return; + } + for (THivePartitionUpdate update : hivePartitionUpdates) { + int fileCount = update.getFileNames() == null ? 0 : update.getFileNames().size(); + List uploads = update.getS3MpuPendingUploads(); + int uploadCount = uploads == null ? 0 : uploads.size(); + boolean completeRecords = uploads != null + && uploads.stream().allMatch(HMSTransaction::isCompleteObjectStoreUpload); + if (fileCount != uploadCount || (fileCount > 0 && !completeRecords)) { + throw new IllegalStateException(String.format( + "Object-store write reported %d file(s) but %d valid multipart completion record(s); " + + "all backends must support deferred multipart completion before metadata commit", + fileCount, completeRecords ? uploadCount : 0)); } } } @@ -219,6 +258,8 @@ public void beginInsertTable(HiveInsertCommandContext ctx) { } public void finishInsertTable(NameMapping nameMapping) { + // Validate ownership records before classification can publish a filesystem or HMS mutation. + validateObjectStoreCommitRecords(); Table table = getTable(nameMapping); if (hivePartitionUpdates.isEmpty() && isOverwrite && table.getPartitionKeysSize() == 0) { // use an empty hivePartitionUpdate to clean source table @@ -1573,8 +1614,16 @@ private void abortMultiUploads() { return; } for (UncompletedMpuPendingUpload uncompletedMpuPendingUpload : uncompletedMpuPendingUploads) { - S3FileSystem s3FileSystem = (S3FileSystem) ((SwitchingFileSystem) fs) + FileSystem uploadFileSystem = ((SwitchingFileSystem) fs) .fileSystem(uncompletedMpuPendingUpload.path); + if (!(uploadFileSystem instanceof S3FileSystem)) { + // Azure cannot selectively abort UUID-namespaced blocks; deleting the target could + // destroy a competing committed blob, so service-side expiry is the safe cleanup. + LOG.info("Leaving uncommitted object-store blocks for service-side expiry at {}", + uncompletedMpuPendingUpload.path); + continue; + } + S3FileSystem s3FileSystem = (S3FileSystem) uploadFileSystem; S3Client s3Client; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 71935cbd88157b..e5b16817becc69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -45,8 +45,10 @@ import org.apache.iceberg.RewriteFiles; import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.io.CloseableIterable; @@ -136,7 +138,6 @@ public void beginInsert(ExternalTable dorisTable, Table targetTable, // Planning, BE serialization, and commit must share one Iceberg metadata // generation even if the catalog refreshes between those phases. this.table = targetTable; - this.baseSnapshotId = null; // check branch if (insertCtx != null && insertCtx.getBranchName().isPresent()) { this.branchName = insertCtx.getBranchName().get(); @@ -149,6 +150,14 @@ public void beginInsert(ExternalTable dorisTable, Table targetTable, + " is a tag, not a branch. Tags cannot be targets for producing snapshots"); } } + if (insertCtx != null && insertCtx.isOverwrite()) { + // OVERWRITE must validate against the exact target branch generation retained at binding. + Snapshot baseSnapshot = branchName == null + ? table.currentSnapshot() : table.snapshot(branchName); + this.baseSnapshotId = baseSnapshot == null ? null : baseSnapshot.snapshotId(); + } else { + this.baseSnapshotId = null; + } this.transaction = createTransactionTable(dorisTable, table).newTransaction(); this.rewrittenDeleteFilesByReferencedDataFile = Collections.emptyMap(); }); @@ -858,7 +867,13 @@ private void commitReplaceTxn(List pendingResults) { overwriteFiles = overwriteFiles.toBranch(branchName); } overwriteFiles = overwriteFiles.scanManifestsWith(ops.getThreadPoolWithPreAuth()); - try (CloseableIterable fileScanTasks = table.newScan().planFiles()) { + overwriteFiles = validateOverwrite(overwriteFiles, Expressions.alwaysTrue()); + TableScan overwriteScan = table.newScan(); + if (branchName != null) { + // The files removed must come from the same branch whose head anchors OCC validation. + overwriteScan = overwriteScan.useRef(branchName); + } + try (CloseableIterable fileScanTasks = overwriteScan.planFiles()) { OverwriteFiles finalOverwriteFiles = overwriteFiles; fileScanTasks.forEach(f -> finalOverwriteFiles.deleteFile(f.file())); } catch (IOException e) { @@ -875,6 +890,11 @@ private void commitReplaceTxn(List pendingResults) { appendPartitionOp = appendPartitionOp.toBranch(branchName); } appendPartitionOp = appendPartitionOp.scanManifestsWith(ops.getThreadPoolWithPreAuth()); + // Partition replacement must not delete or revive files committed after sink binding. + if (baseSnapshotId != null) { + appendPartitionOp = appendPartitionOp.validateFromSnapshot(baseSnapshotId); + } + appendPartitionOp = appendPartitionOp.validateNoConflictingData().validateNoConflictingDeletes(); for (WriteResult result : pendingResults) { Preconditions.checkState(result.referencedDataFiles().length == 0, "Should have no referenced data files."); @@ -905,6 +925,7 @@ private void commitStaticPartitionOverwrite(List pendingResults) { // Set partition filter to overwrite only matching partitions overwriteFiles = overwriteFiles.overwriteByRowFilter(partitionFilter); + overwriteFiles = validateOverwrite(overwriteFiles, partitionFilter); // Add new data files for (WriteResult result : pendingResults) { @@ -917,6 +938,14 @@ private void commitStaticPartitionOverwrite(List pendingResults) { overwriteFiles.commit(); } + private OverwriteFiles validateOverwrite(OverwriteFiles overwriteFiles, Expression conflictFilter) { + overwriteFiles = overwriteFiles.conflictDetectionFilter(conflictFilter); + if (baseSnapshotId != null) { + overwriteFiles = overwriteFiles.validateFromSnapshot(baseSnapshotId); + } + return overwriteFiles.validateNoConflictingData().validateNoConflictingDeletes(); + } + /** * Build partition filter expression from static partition key-value pairs * diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExecuteActionFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExecuteActionFactory.java index 0d09a9ef35ced8..1911eba4b60ff4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExecuteActionFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExecuteActionFactory.java @@ -41,6 +41,7 @@ public class IcebergExecuteActionFactory { public static final String REWRITE_DATA_FILES = "rewrite_data_files"; public static final String PUBLISH_CHANGES = "publish_changes"; public static final String REWRITE_MANIFESTS = "rewrite_manifests"; + public static final String REMOVE_ORPHAN_FILES = "remove_orphan_files"; /** * Create an Iceberg-specific ExecuteAction instance. @@ -88,6 +89,9 @@ public static ExecuteAction createAction(String actionType, Map case REWRITE_MANIFESTS: return new IcebergRewriteManifestsAction(properties, partitionNamesInfo, whereCondition); + case REMOVE_ORPHAN_FILES: + return new IcebergRemoveOrphanFilesAction(properties, partitionNamesInfo, + whereCondition); default: throw new DdlException("Unsupported Iceberg procedure: " + actionType + ". Supported procedures: " + String.join(", ", getSupportedActions())); @@ -109,7 +113,8 @@ public static String[] getSupportedActions() { EXPIRE_SNAPSHOTS, REWRITE_DATA_FILES, PUBLISH_CHANGES, - REWRITE_MANIFESTS + REWRITE_MANIFESTS, + REMOVE_ORPHAN_FILES }; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesAction.java new file mode 100644 index 00000000000000..90939455ba041f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -0,0 +1,377 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg.action; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ArgumentParsers; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.info.PartitionNamesInfo; +import org.apache.doris.nereids.trees.expressions.Expression; + +import com.google.common.collect.Lists; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.ReachableFileUtil; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.io.FileInfo; +import org.apache.iceberg.io.SupportsPrefixOperations; +import org.apache.iceberg.util.PropertyUtil; + +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** Safely lists or deletes old files that are unreachable from every retained snapshot. */ +public class IcebergRemoveOrphanFilesAction extends BaseIcebergAction { + private static final long MIN_RETENTION_MS = Duration.ofHours(24).toMillis(); + private static final int MAX_REACHABLE_FILES = 5_000_000; + public static final String OLDER_THAN = "older_than"; + public static final String LOCATION = "location"; + public static final String DRY_RUN = "dry_run"; + public static final String ALLOW_UNSAFE_LOCATION = "allow_unsafe_location"; + + public IcebergRemoveOrphanFilesAction(Map properties, + Optional partitionNamesInfo, + Optional whereCondition) { + super("remove_orphan_files", properties, partitionNamesInfo, whereCondition); + } + + @Override + protected void registerIcebergArguments() { + namedArguments.registerRequiredArgument(OLDER_THAN, "Creation time cutoff in milliseconds", + ArgumentParsers.nonNegativeLong(OLDER_THAN)); + namedArguments.registerOptionalArgument(LOCATION, "Prefix to scan for orphan files", + null, ArgumentParsers.nonEmptyString(LOCATION)); + namedArguments.registerOptionalArgument(DRY_RUN, "Only count orphan files", true, + ArgumentParsers.booleanValue(DRY_RUN)); + namedArguments.registerOptionalArgument(ALLOW_UNSAFE_LOCATION, + "Allow an explicitly supplied location whose table ownership cannot be proved", + false, ArgumentParsers.booleanValue(ALLOW_UNSAFE_LOCATION)); + } + + @Override + protected void validateIcebergAction() throws UserException { + validateNoPartitions(); + validateNoWhereCondition(); + String location = namedArguments.getString(LOCATION); + if (location != null) { + try { + normalizeLocation(location); + } catch (IllegalArgumentException e) { + throw new AnalysisException("Invalid location URI: " + location, e); + } + } + } + + @Override + protected List executeAction(TableIf tableIf) throws UserException { + Table table = ((IcebergExternalTable) tableIf).getIcebergTable(); + if (!(table.io() instanceof SupportsPrefixOperations)) { + throw new UserException("remove_orphan_files requires FileIO prefix listing support"); + } + if (!PropertyUtil.propertyAsBoolean(table.properties(), TableProperties.GC_ENABLED, + TableProperties.GC_ENABLED_DEFAULT)) { + // A GC-disabled table may share files with another table, so no destructive scan is safe. + throw new UserException("Cannot remove orphan files: Iceberg GC is disabled"); + } + long olderThan = namedArguments.getLong(OLDER_THAN); + // Reject an unsafe cutoff before opening any metadata or manifest file. + if (olderThan > System.currentTimeMillis() - MIN_RETENTION_MS) { + throw new UserException("older_than must retain at least 24 hours of files"); + } + List scanScopes = resolveScanScopes(table); + + try { + ReachableIndex reachable = collectReachableFiles(table); + long orphanCount = 0; + long deletedCount = 0; + boolean dryRun = namedArguments.getBoolean(DRY_RUN); + for (ScanScope scope : scanScopes) { + // Object stores use raw prefix matching, so the separator excludes sibling prefixes. + String listingPrefix = scope.root.endsWith("/") ? scope.root : scope.root + "/"; + for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { + // Unknown creation time cannot prove the file predates every in-flight writer. + if (scope.owns(file.location()) && file.createdAtMillis() > 0 + && file.createdAtMillis() < olderThan + && !isReachable(file.location(), reachable)) { + orphanCount++; + if (!dryRun) { + table.io().deleteFile(file.location()); + deletedCount++; + } + } + } + } + return Lists.newArrayList(String.valueOf(orphanCount), String.valueOf(deletedCount)); + } catch (Exception e) { + throw new UserException("Failed to remove orphan files: " + e.getMessage(), e); + } + } + + private List resolveScanScopes(Table table) throws UserException { + String tableRoot = normalizeLocation(table.location()); + String requested = namedArguments.getString(LOCATION); + if (requested != null) { + String normalized = normalizeLocation(requested); + if (isWithinLocation(normalized, tableRoot)) { + return Lists.newArrayList(ScanScope.exclusive(normalized)); + } + if (!namedArguments.getBoolean(ALLOW_UNSAFE_LOCATION)) { + throw new UserException( + "Cannot prove that location is owned by this table; set allow_unsafe_location=true " + + "only after verifying the prefix is exclusive to the table"); + } + // This explicit escape hatch also covers historical roots after a table-location migration. + return Lists.newArrayList(ScanScope.exclusive(normalized)); + } + if (nonEmpty(table.properties().get(TableProperties.WRITE_LOCATION_PROVIDER_IMPL)) != null) { + throw new UserException( + "remove_orphan_files cannot infer ownership for a custom write.location-provider.impl; " + + "provide location with allow_unsafe_location=true after verifying exclusivity"); + } + String metadataRoot = nonEmpty(table.properties().get(TableProperties.WRITE_METADATA_LOCATION)); + if (metadataRoot != null && !isWithinLocation(normalizeLocation(metadataRoot), tableRoot)) { + throw new UserException( + "Cannot prove that the configured external metadata location is table-exclusive; " + + "provide location with allow_unsafe_location=true after verifying exclusivity"); + } + List scopes = new ArrayList<>(); + scopes.add(ScanScope.exclusive(tableRoot)); + if (Boolean.parseBoolean(table.properties().get(TableProperties.OBJECT_STORE_ENABLED))) { + // Match Iceberg's ObjectStoreLocationProvider precedence exactly. + String objectRoot = nonEmpty(table.properties().get(TableProperties.WRITE_DATA_LOCATION)); + if (objectRoot == null) { + objectRoot = nonEmpty(table.properties().get(TableProperties.OBJECT_STORE_PATH)); + } + if (objectRoot == null) { + objectRoot = nonEmpty(table.properties().get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION)); + } + if (objectRoot != null && !isWithinLocation(normalizeLocation(objectRoot), tableRoot)) { + // The hashed suffix is not a unique ownership key across catalogs sharing a root. + throw new UserException( + "Cannot prove that the configured object-store root is table-exclusive; " + + "provide location with allow_unsafe_location=true after verifying exclusivity"); + } + } else { + String externalDataRoot = nonEmpty(table.properties().get(TableProperties.WRITE_DATA_LOCATION)); + if (externalDataRoot == null) { + externalDataRoot = nonEmpty( + table.properties().get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION)); + } + if (externalDataRoot != null + && !isWithinLocation(normalizeLocation(externalDataRoot), tableRoot)) { + throw new UserException( + "Cannot prove that the configured external data location is table-exclusive; " + + "provide location with allow_unsafe_location=true after verifying exclusivity"); + } + } + return scopes; + } + + private static String nonEmpty(String location) { + return location == null || location.isEmpty() ? null : location; + } + + private ReachableIndex collectReachableFiles(Table table) throws IOException, UserException { + ReachableIndex reachable = new ReachableIndex(MAX_REACHABLE_FILES); + reachable.addAll(ReachableFileUtil.metadataFileLocations(table, true)); + // Hadoop tables consult this live pointer even though it is not part of the metadata log. + reachable.add(ReachableFileUtil.versionHintLocation(table)); + Set scannedDataManifests = new HashSet<>(); + Set scannedDeleteManifests = new HashSet<>(); + reachable.addAll(ReachableFileUtil.manifestListLocations(table)); + reachable.addAll(ReachableFileUtil.statisticsFilesLocations(table)); + for (Snapshot snapshot : table.snapshots()) { + for (ManifestFile manifest : snapshot.allManifests(table.io())) { + reachable.add(manifest.path()); + if (manifest.content() == ManifestContent.DATA) { + // Snapshots inherit manifests, so read each path once to keep work linear. + if (scannedDataManifests.add(manifest.path())) { + try (ManifestReader dataFiles = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile dataFile : dataFiles) { + reachable.add(dataFile.location()); + } + } + } + } else if (scannedDeleteManifests.add(manifest.path())) { + // Retained delete files may not apply to current data tasks, so read them directly. + try (ManifestReader deletes = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile delete : deletes) { + reachable.add(delete.location()); + } + } + } + } + } + return reachable; + } + + private static boolean isReachable(String candidate, ReachableIndex reachable) throws UserException { + FileIdentity candidateIdentity = FileIdentity.of(candidate); + FileIdentity retainedIdentity = reachable.byPath.get(candidateIdentity.path); + if (candidateIdentity.equals(retainedIdentity)) { + return true; + } + if (retainedIdentity != null) { + // A path collision across unknown providers or authorities must fail closed. + throw new UserException("Cannot determine whether listed and reachable file locations are equivalent"); + } + return false; + } + + static boolean sameFileIdentity(String first, String second) { + return FileIdentity.of(first).equals(FileIdentity.of(second)); + } + + static void verifyReachableIndexLimit(Set locations, int maxEntries) throws UserException { + ReachableIndex index = new ReachableIndex(maxEntries); + index.addAll(locations); + } + + private static final class ReachableIndex { + private final Map byPath = new LinkedHashMap<>(); + private final int maxEntries; + + private ReachableIndex(int maxEntries) { + this.maxEntries = maxEntries; + } + + private void addAll(Iterable locations) throws UserException { + for (String location : locations) { + add(location); + } + } + + private void add(String location) throws UserException { + FileIdentity identity = FileIdentity.of(location); + FileIdentity existing = byPath.putIfAbsent(identity.path, identity); + if (existing != null && !existing.equals(identity)) { + throw new UserException("Cannot determine whether reachable file locations are equivalent"); + } + if (existing == null && byPath.size() > maxEntries) { + throw new UserException( + "Reachable file index exceeds the safe in-memory limit of " + maxEntries); + } + } + } + + private static final class FileIdentity { + private final String scheme; + private final String authority; + private final String path; + + private FileIdentity(String scheme, String authority, String path) { + this.scheme = scheme; + this.authority = authority; + this.path = path; + } + + private static FileIdentity of(String location) { + URI uri = URI.create(location).normalize(); + String scheme = uri.getScheme(); + scheme = scheme == null ? "" : scheme.toLowerCase(Locale.ROOT); + if (scheme.equals("s3a") || scheme.equals("s3n")) { + scheme = "s3"; + } + String authority = uri.getAuthority(); + authority = authority == null ? "" : authority.toLowerCase(Locale.ROOT); + String path = uri.getPath(); + return new FileIdentity(scheme, authority, path == null ? "" : path); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof FileIdentity)) { + return false; + } + FileIdentity that = (FileIdentity) other; + return scheme.equals(that.scheme) && authority.equals(that.authority) && path.equals(that.path); + } + + @Override + public int hashCode() { + return Objects.hash(scheme, authority, path); + } + } + + private static final class ScanScope { + private final String root; + + private ScanScope(String root) { + this.root = root; + } + + private static ScanScope exclusive(String root) { + return new ScanScope(root); + } + + private boolean owns(String candidate) { + return isWithinLocation(candidate, root); + } + } + + private static boolean isWithinLocation(String location, String root) { + FileIdentity child = FileIdentity.of(location); + FileIdentity parent = FileIdentity.of(root); + String pathPrefix = parent.path.endsWith("/") ? parent.path : parent.path + "/"; + return child.scheme.equals(parent.scheme) && child.authority.equals(parent.authority) + && (child.path.equals(parent.path) || child.path.startsWith(pathPrefix)); + } + + private static String normalizeLocation(String location) { + String normalized = URI.create(location).normalize().toString(); + return normalized.length() > 1 && normalized.endsWith("/") + ? normalized.substring(0, normalized.length() - 1) : normalized; + } + + @Override + protected List getResultSchema() { + return Lists.newArrayList( + new Column("orphan_file_count", Type.BIGINT, false, "Number of old unreachable files"), + new Column("deleted_file_count", Type.BIGINT, false, "Number of files deleted")); + } + + @Override + public String getDescription() { + return "List or delete old files unreachable from every retained Iceberg snapshot"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 957ab6ed55e193..56457819a2a3de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -88,6 +88,8 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataTableUtils; import org.apache.iceberg.PartitionData; @@ -138,7 +140,8 @@ public class IcebergScanNode extends FileQueryScanNode { public static final int MIN_DELETE_FILE_SUPPORT_VERSION = 2; - static final int ICEBERG_SCAN_SEMANTICS_VERSION = 1; + // Version 2 opts BE into required-field rejection and typed nested initial-default handling. + static final int ICEBERG_SCAN_SEMANTICS_VERSION = 2; private static final Logger LOG = LogManager.getLogger(IcebergScanNode.class); private IcebergSource source; @@ -283,6 +286,11 @@ protected void doInitialize() throws UserException { // This gate must run during shared initialization: batch split assignment bypasses // doGetSplits(), but it must never assign a semantic Variant projection to an old BE. checkVariantBackendCompatibilityForCurrentScan(backendPolicy.getBackends()); + Iterable backends = backendPolicy.getBackends(); + if (hasSmoothUpgradeSource(backends)) { + // Delete-manifest inspection is only needed while old BEs can actually receive this scan. + checkIcebergScanSemanticsV2Compatibility(requiresIcebergScanSemanticsV2(), backends); + } } void checkVariantBackendCompatibilityForCurrentScan(Iterable backends) @@ -298,6 +306,214 @@ void checkVariantBackendCompatibilityForCurrentScan(Iterable backends) checkVariantBackendCompatibility(projectsVariant, backends); } + private boolean requiresIcebergScanSemanticsV2() throws UserException { + if (isSystemTable) { + // position_deletes already has its stricter dedicated mixed-version gate. + return false; + } + TableScan scan = createTableScan(); + Snapshot snapshot = scan.snapshot(); + if (snapshot == null) { + return false; + } + if (hasApplicableEqualityDeletes(scan)) { + return true; + } + Schema scanSchema = scan.schema(); + Set projectedFieldIds = projectedFieldIds(scanSchema); + Set topLevelIds = new HashSet<>(); + for (NestedField field : scanSchema.columns()) { + topLevelIds.add(field.fieldId()); + } + Map fieldsById = TypeUtil.indexById(scanSchema.asStruct()); + for (Integer fieldId : projectedFieldIds) { + NestedField field = fieldsById.get(fieldId); + if (field.initialDefault() != null + && (!topLevelIds.contains(field.fieldId()) || field.type().isNestedType())) { + return true; + } + } + if (hasProjectedNameAliasCollision(scanSchema, projectedFieldIds, extractNameMapping())) { + return true; + } + return schemaHistoryRequiresMissingRequiredFieldRejection( + scanSchema, projectedFieldIds, icebergTable.schemas().values()); + } + + private boolean hasApplicableEqualityDeletes(TableScan scan) throws UserException { + Snapshot snapshot = scan.snapshot(); + if (snapshot == null || "0".equals(snapshot.summary().get("total-equality-deletes"))) { + return false; + } + // Inspect only delete manifests, not data tasks: equality-delete semantics are snapshot-wide + // compatibility state even when the current predicate happens to prune their partitions. + for (ManifestFile manifest : snapshot.deleteManifests(icebergTable.io())) { + if (!manifest.hasAddedFiles() && !manifest.hasExistingFiles()) { + continue; + } + try (ManifestReader deletes = ManifestFiles.readDeleteManifest( + manifest, icebergTable.io(), icebergTable.specs())) { + for (DeleteFile delete : deletes) { + if (delete.content() == FileContent.EQUALITY_DELETES) { + return true; + } + } + } catch (IOException e) { + throw new UserException( + "Failed to inspect Iceberg delete manifest " + manifest.path(), e); + } + } + return false; + } + + private static boolean hasSmoothUpgradeSource(Iterable backends) { + for (Backend backend : backends) { + if (backend.isSmoothUpgradeSrc()) { + return true; + } + } + return false; + } + + private Set projectedFieldIds(Schema scanSchema) { + Set projected = new HashSet<>(); + for (SlotDescriptor slot : desc.getSlots()) { + int fieldId = slot.getColumn().getUniqueId(); + // Stable Iceberg IDs prevent a dropped-and-readded name from selecting the wrong history. + NestedField field = fieldId >= 0 ? scanSchema.findField(fieldId) + : scanSchema.caseInsensitiveFindField(slot.getColumn().getName()); + if (field != null) { + projected.addAll(TypeUtil.indexById( + org.apache.iceberg.types.Types.StructType.of(field)).keySet()); + } + } + return projected; + } + + @VisibleForTesting + static void checkIcebergScanSemanticsV2Compatibility( + boolean requiresV2, Iterable backends) throws UserException { + if (!requiresV2) { + return; + } + for (Backend backend : backends) { + if (backend.isSmoothUpgradeSrc()) { + // A V1 BE accepts the Thrift field but does not enforce nested defaults/requiredness. + throw new UserException("Current Iceberg scan semantics are unavailable while backend " + + backend.getId() + " is a smooth upgrade source"); + } + } + } + + @VisibleForTesting + static boolean schemaHistoryRequiresMissingRequiredFieldRejection( + Schema scanSchema, Iterable historicalSchemas) { + return schemaHistoryRequiresMissingRequiredFieldRejection( + scanSchema, TypeUtil.indexById(scanSchema.asStruct()).keySet(), historicalSchemas); + } + + private static boolean schemaHistoryRequiresMissingRequiredFieldRejection( + Schema scanSchema, Set projectedFieldIds, Iterable historicalSchemas) { + Map currentFields = TypeUtil.indexById(scanSchema.asStruct()); + Map parentById = TypeUtil.indexParents(scanSchema.asStruct()); + Set collectionWrapperIds = new HashSet<>(); + collectCollectionWrapperFieldIds(scanSchema.asStruct(), collectionWrapperIds); + for (Schema historicalSchema : historicalSchemas) { + Map historicalFields = TypeUtil.indexById(historicalSchema.asStruct()); + for (Integer fieldId : projectedFieldIds) { + NestedField field = currentFields.get(fieldId); + if (field == null || collectionWrapperIds.contains(field.fieldId()) + || field.initialDefault() != null || field.isOptional()) { + continue; + } + NestedField historicalField = historicalFields.get(field.fieldId()); + if (historicalField != null) { + if (historicalField.isOptional()) { + return true; + } + continue; + } + NestedField highestMissing = field; + Integer parentId = parentById.get(field.fieldId()); + while (parentId != null && !historicalFields.containsKey(parentId)) { + highestMissing = Objects.requireNonNull(currentFields.get(parentId), + "Iceberg parent field " + parentId + " is absent from scan schema"); + parentId = parentById.get(parentId); + } + if (!collectionWrapperIds.contains(highestMissing.fieldId()) + && highestMissing.isRequired() && highestMissing.initialDefault() == null) { + return true; + } + } + } + return false; + } + + private static boolean hasProjectedNameAliasCollision( + Schema schema, Set projectedFieldIds, + Optional>> nameMapping) { + if (!nameMapping.isPresent()) { + return false; + } + Set collisions = new HashSet<>(); + collectNameAliasCollisions(schema.asStruct(), nameMapping.get(), collisions); + collisions.retainAll(projectedFieldIds); + return !collisions.isEmpty(); + } + + private static void collectNameAliasCollisions( + Type type, Map> nameMapping, Set collisions) { + switch (type.typeId()) { + case STRUCT: + List fields = type.asStructType().fields(); + for (NestedField field : fields) { + for (String alias : nameMapping.getOrDefault( + field.fieldId(), Collections.emptyList())) { + for (NestedField sibling : fields) { + if (sibling.fieldId() != field.fieldId() + && sibling.name().equalsIgnoreCase(alias)) { + collisions.add(field.fieldId()); + collisions.add(sibling.fieldId()); + } + } + } + collectNameAliasCollisions(field.type(), nameMapping, collisions); + } + break; + case LIST: + collectNameAliasCollisions(type.asListType().elementType(), nameMapping, collisions); + break; + case MAP: + collectNameAliasCollisions(type.asMapType().keyType(), nameMapping, collisions); + collectNameAliasCollisions(type.asMapType().valueType(), nameMapping, collisions); + break; + default: + break; + } + } + + private static void collectCollectionWrapperFieldIds(Type type, Set result) { + switch (type.typeId()) { + case STRUCT: + for (NestedField field : type.asStructType().fields()) { + collectCollectionWrapperFieldIds(field.type(), result); + } + break; + case LIST: + result.add(type.asListType().elementId()); + collectCollectionWrapperFieldIds(type.asListType().elementType(), result); + break; + case MAP: + result.add(type.asMapType().keyId()); + result.add(type.asMapType().valueId()); + collectCollectionWrapperFieldIds(type.asMapType().keyType(), result); + collectCollectionWrapperFieldIds(type.asMapType().valueType(), result); + break; + default: + break; + } + } + private Optional>> extractNameMapping() { Optional snapshot = getPinnedRelationSnapshot(); if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/AzureObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/AzureObjStorage.java index 4929e34e7f5a74..b06d1681e55506 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/AzureObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/AzureObjStorage.java @@ -57,8 +57,6 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.file.FileSystems; import java.nio.file.PathMatcher; import java.nio.file.Paths; @@ -69,7 +67,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.stream.Collectors; public class AzureObjStorage implements ObjStorage { private static final Logger LOG = LogManager.getLogger(AzureObjStorage.class); @@ -280,13 +277,23 @@ public void completeMultipartUpload(String bucket, String key, Map blockIds = parts.keySet().stream() - .map(k -> Base64.getEncoder() - .encodeToString(ByteBuffer.allocate(4) - .order(ByteOrder.LITTLE_ENDIAN) - .putInt(k) - .array())).collect(Collectors.toList()); - blockBlobClient.commitBlockList(blockIds); + blockBlobClient.commitBlockList(multipartBlockIds(parts)); + } + + static List multipartBlockIds(Map parts) { + return parts.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(entry -> { + // The value is the exact UUID-namespaced Azure block ID staged by BE; + // deriving an ID from the part number can publish another writer's block. + if (entry.getValue() == null || entry.getValue().isEmpty()) { + throw new IllegalArgumentException( + "Azure multipart completion requires the staged block ID for part " + + entry.getKey()); + } + return entry.getValue(); + }) + .collect(java.util.stream.Collectors.toList()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 5e33caf1cebea1..3a163699b2b7af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -3418,7 +3418,8 @@ private DataPartition toDataPartition(DistributionSpec distributionSpec/* target field.getTransform(), field.getParam(), field.getName(), - field.getSourceId())); + field.getSourceId(), + field.getSourceFieldPath())); } return new DataPartition(TPartitionType.MERGE_PARTITIONED, operationExpr, insertPartitionExprs, deletePartitionExprs, mergeSpec.isInsertRandom(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMerge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMerge.java index c6ea4e37a8bf97..63935cb8f2df4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMerge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMerge.java @@ -38,17 +38,25 @@ public static class IcebergPartitionField { private final Integer param; private final String name; private final Integer sourceId; + private final ImmutableList sourceFieldPath; /** * Create a partition field mapping for merge insert routing. */ public IcebergPartitionField(String transform, ExprId sourceExprId, Integer param, String name, Integer sourceId) { + this(transform, sourceExprId, param, name, sourceId, ImmutableList.of()); + } + + /** Create a partition field mapping whose source is nested below a top-level slot. */ + public IcebergPartitionField(String transform, ExprId sourceExprId, Integer param, + String name, Integer sourceId, List sourceFieldPath) { this.transform = Objects.requireNonNull(transform, "transform should not be null"); this.sourceExprId = Objects.requireNonNull(sourceExprId, "sourceExprId should not be null"); this.param = param; this.name = name; this.sourceId = sourceId; + this.sourceFieldPath = ImmutableList.copyOf(sourceFieldPath); } public String getTransform() { @@ -71,6 +79,10 @@ public Integer getSourceId() { return sourceId; } + public List getSourceFieldPath() { + return sourceFieldPath; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -84,12 +96,13 @@ public boolean equals(Object o) { && sourceExprId.equals(that.sourceExprId) && Objects.equals(param, that.param) && Objects.equals(name, that.name) - && Objects.equals(sourceId, that.sourceId); + && Objects.equals(sourceId, that.sourceId) + && sourceFieldPath.equals(that.sourceFieldPath); } @Override public int hashCode() { - return Objects.hash(transform, sourceExprId, param, name, sourceId); + return Objects.hash(transform, sourceExprId, param, name, sourceId, sourceFieldPath); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java index b8c34291c93d93..ec952b8c567522 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java @@ -138,6 +138,10 @@ public void unregisterListener(InsertExecutorListener listener) { listeners.remove(listener); } + protected void handleAfterCompleteFailure(Exception e) throws Exception { + throw e; + } + public Coordinator getCoordinator() { return coordinator; } @@ -257,8 +261,9 @@ private void checkStrictModeAndFilterRatio() throws Exception { * execute insert txn for insert into select command. */ public void executeSingleInsert(StmtExecutor executor) throws Exception { - beforeExec(); try { + // Pre-execution work may register external resources, so it must share the transaction cleanup scope. + beforeExec(); executor.updateProfile(false); execImpl(executor); checkStrictModeAndFilterRatio(); @@ -267,7 +272,11 @@ public void executeSingleInsert(StmtExecutor executor) throws Exception { } onComplete(); for (InsertExecutorListener listener : listeners) { - listener.afterComplete(this, executor, jobId); + try { + listener.afterComplete(this, executor, jobId); + } catch (Exception e) { + handleAfterCompleteFailure(e); + } } } catch (Throwable t) { onFail(t); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java index 3d8f74d502a78b..b13e8d28977cfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java @@ -116,8 +116,13 @@ protected void onComplete() throws UserException { txnStatus = TransactionStatus.COMMITTED; long t2 = System.currentTimeMillis(); - // Handle post-commit operations (e.g., cache refresh) - doAfterCommit(); + try { + doAfterCommit(); + } catch (Exception e) { + // Cache refresh cannot undo a durable remote commit, so it must not make clients retry the write. + LOG.warn("Post-commit refresh failed for table {}. Data was committed successfully.", + table.getName(), e); + } long t3 = System.currentTimeMillis(); LOG.info("Transaction commit breakdown: doBeforeCommit={}ms, commit={}ms, doAfterCommit={}ms, total={}ms", t1 - t0, t2 - t1, t3 - t2, t3 - t0); @@ -139,6 +144,16 @@ protected void doAfterCommit() throws DdlException { true); } + @Override + protected void handleAfterCompleteFailure(Exception e) throws Exception { + if (txnStatus != TransactionStatus.COMMITTED) { + super.handleAfterCompleteFailure(e); + return; + } + // A post-commit listener cannot undo remote data, so failing the statement would invite duplicate retries. + LOG.warn("Post-commit listener failed for table {}. Data was committed successfully.", table.getName(), e); + } + @Override protected void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink physicalSink) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index 7618f7801e059d..77349938d2e961 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -264,6 +264,8 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor stmtExec int retryTimes = 0; ctx.getStatementContext().setIsInsert(true); while (++retryTimes < Math.max(ctx.getSessionVariable().dmlPlanRetryTimes, 3)) { + // Each attempt must repin MVCC metadata or it can reuse the schema that triggered the retry. + ctx.getStatementContext().resetMvccSnapshots(); TableIf targetTableIf = getTargetTableIf(ctx, qualifiedTargetTableName); DatabaseIf targetDatabase = getTargetDatabase(targetTableIf); // check auth diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java index 98f0fc9f0fa267..a27abe676f474b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java @@ -235,12 +235,13 @@ public PhysicalProperties getRequirePhysicalProperties() { // Distribution and writer serialization must read the same retained spec/schema. List partitionColumns = getRetainedPartitionColumns(); Map columnExprIdMap = buildColumnExprIdMap(outputSlots, nameToExprId); + Map columnIdToExprId = buildColumnIdExprIdMap(outputSlots); boolean insertExprsOk = false; if (!partitionColumns.isEmpty()) { insertExprsOk = buildInsertPartitionExprIds(insertPartitionExprIds, partitionColumns, columnExprIdMap); } InsertPartitionFieldResult fieldResult = buildInsertPartitionFields( - insertPartitionFields, targetIcebergTable, columnExprIdMap); + insertPartitionFields, targetIcebergTable, columnIdToExprId); boolean insertFieldsOk = fieldResult.success; boolean hasNonIdentity = fieldResult.hasNonIdentity; if (insertFieldsOk) { @@ -317,7 +318,7 @@ private List getDataSlots(List outputSlots) { private InsertPartitionFieldResult buildInsertPartitionFields( List insertPartitionFields, Table table, - Map columnExprIdMap) { + Map columnIdToExprId) { PartitionSpec spec = table.spec(); if (spec == null || !spec.isPartitioned()) { return new InsertPartitionFieldResult(false, false, null); @@ -339,15 +340,24 @@ private InsertPartitionFieldResult buildInsertPartitionFields( insertPartitionFields.clear(); return new InsertPartitionFieldResult(false, hasNonIdentity, spec.specId()); } - ExprId exprId = columnExprIdMap.get(sourceField.name()); + Column rootColumn = findSourceRootColumn(cols, field.sourceId()); + // Nested routing must bind through stable Iceberg IDs; a name fallback can target a + // different field after rename/drop-and-add schema evolution. + ExprId exprId = rootColumn == null || rootColumn.getUniqueId() < 0 + ? null : columnIdToExprId.get(rootColumn.getUniqueId()); if (exprId == null) { insertPartitionFields.clear(); return new InsertPartitionFieldResult(false, hasNonIdentity, spec.specId()); } String transform = field.transform().toString(); Integer param = parseTransformParam(transform); + List sourceFieldPath = resolveSourceFieldPath(rootColumn, field.sourceId()); + if (sourceFieldPath == null) { + insertPartitionFields.clear(); + return new InsertPartitionFieldResult(false, hasNonIdentity, spec.specId()); + } insertPartitionFields.add(new DistributionSpecMerge.IcebergPartitionField( - transform, exprId, param, field.name(), field.sourceId())); + transform, exprId, param, field.name(), field.sourceId(), sourceFieldPath)); } if (insertPartitionFields.isEmpty()) { return new InsertPartitionFieldResult(false, hasNonIdentity, spec.specId()); @@ -355,6 +365,75 @@ private InsertPartitionFieldResult buildInsertPartitionFields( return new InsertPartitionFieldResult(true, hasNonIdentity, spec.specId()); } + private Map buildColumnIdExprIdMap(List outputSlots) { + Map result = new java.util.HashMap<>(); + List visibleColumns = new ArrayList<>(); + for (Column column : cols) { + if (column.isVisible()) { + visibleColumns.add(column); + } + } + List dataSlots = getDataSlots(outputSlots); + if (visibleColumns.size() != dataSlots.size()) { + return result; + } + for (int i = 0; i < visibleColumns.size(); i++) { + if (visibleColumns.get(i).getUniqueId() >= 0) { + result.put(visibleColumns.get(i).getUniqueId(), dataSlots.get(i).getExprId()); + } + } + return result; + } + + private static Column findSourceRootColumn(List columns, int sourceId) { + for (Column column : columns) { + if (column.getUniqueId() == sourceId || containsFieldId(column.getChildren(), sourceId)) { + return column; + } + } + return null; + } + + private static boolean containsFieldId(List columns, int sourceId) { + if (columns == null) { + return false; + } + for (Column column : columns) { + if (column.getUniqueId() == sourceId || containsFieldId(column.getChildren(), sourceId)) { + return true; + } + } + return false; + } + + private static List resolveSourceFieldPath(Column root, int sourceId) { + if (root == null) { + return null; + } + if (root.getUniqueId() == sourceId) { + return ImmutableList.of(); + } + List path = new ArrayList<>(); + return findSourceFieldPath(root.getChildren(), sourceId, path) + ? ImmutableList.copyOf(path) : null; + } + + private static boolean findSourceFieldPath(List columns, int sourceId, List path) { + if (columns == null) { + return false; + } + for (int index = 0; index < columns.size(); index++) { + Column column = columns.get(index); + path.add(index); + if (column.getUniqueId() == sourceId + || findSourceFieldPath(column.getChildren(), sourceId, path)) { + return true; + } + path.remove(path.size() - 1); + } + return false; + } + private List getRetainedPartitionColumns() { Map columnsByName = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (Column column : cols) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java index 5d87bc08f5f56d..0e59cc2c6c1248 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.util; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.qe.SqlModeHelper; /** @@ -89,8 +90,6 @@ public static String parseStringLiteral(String text) { * Quote a value as a STRING_LITERAL that can be parsed under the current SQL mode. */ public static String quoteStringLiteral(String value) { - String escaped = SqlModeHelper.hasNoBackSlashEscapes() - ? value : value.replace("\\", "\\\\"); - return "\"" + escaped.replace("\"", "\"\"") + "\""; + return SqlUtils.quoteStringLiteral(value, SqlModeHelper.hasNoBackSlashEscapes()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java index 8e652821c7663b..9f20824a9c2e93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java @@ -205,14 +205,21 @@ public static class IcebergPartitionField { private final Integer param; private final String name; private final Integer sourceId; + private final ImmutableList sourceFieldPath; public IcebergPartitionField(Expr sourceExpr, String transform, Integer param, String name, Integer sourceId) { + this(sourceExpr, transform, param, name, sourceId, ImmutableList.of()); + } + + public IcebergPartitionField(Expr sourceExpr, String transform, Integer param, + String name, Integer sourceId, List sourceFieldPath) { this.sourceExpr = Preconditions.checkNotNull(sourceExpr, "sourceExpr should not be null"); this.transform = Preconditions.checkNotNull(transform, "transform should not be null"); this.param = param; this.name = name; this.sourceId = sourceId; + this.sourceFieldPath = ImmutableList.copyOf(sourceFieldPath); } public TIcebergPartitionField toThrift() { @@ -228,6 +235,9 @@ public TIcebergPartitionField toThrift() { if (sourceId != null) { field.setSourceId(sourceId); } + if (!sourceFieldPath.isEmpty()) { + field.setSourceFieldPath(sourceFieldPath); + } return field; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java index a9568e225f0647..243d64c26c5787 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java @@ -93,6 +93,9 @@ public String getExplainString(String prefix, TExplainLevel explainLevel) { public void bindDataSink(Optional insertCtx) throws AnalysisException { THiveTableSink tSink = new THiveTableSink(); + // The legacy planner uses the same deferred Azure protocol as the connector planner, so the BE + // must not fall back to publishing blocks before FE has durably accepted the commit records. + tSink.setSupportsDeferredAzureMultipart(true); tSink.setDbName(targetTable.getDbName()); tSink.setTableName(targetTable.getName()); Set partNames = new HashSet<>(targetTable.getPartitionColumnNames()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java index 54e607dd27afa0..afe2a541abe0d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java @@ -84,14 +84,18 @@ public void tryFinishSchedule() { } @Override - public final void updateFragmentExecStatus(TReportExecStatusParams params) { + public final boolean updateFragmentExecStatus(TReportExecStatusParams params) { if (params.status.status_code == TStatusCode.FINISHED) { params.status = new TStatus(TStatusCode.OK); } SingleFragmentPipelineTask fragmentTask = backendFragmentTasks.get().get( new BackendFragmentId(params.getBackendId(), params.getFragmentId())); if (fragmentTask == null) { - return; + if (params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() + || params.isSetMcCommitDatas() || params.isSetPaimonCommitMessages()) { + throw new IllegalStateException("Missing fragment handler for external-file report"); + } + return false; } TUniqueId queryId = coordinatorContext.queryId; @@ -117,6 +121,9 @@ public final void updateFragmentExecStatus(TReportExecStatusParams params) { } } doProcessReportExecStatus(params, fragmentTask); + return (!params.isSetHivePartitionUpdates() && !params.isSetIcebergCommitDatas() + && !params.isSetMcCommitDatas() && !params.isSetPaimonCommitMessages()) + || fragmentTask.isDone(); } private Map buildBackendFragmentTasks( diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 13e14e6ff3eb48..1b752281e15996 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -254,7 +254,8 @@ public class Coordinator implements CoordInterface { private String trackingUrl; private String firstErrorMsg; // related txnId and label of group commit - private long txnId; + // Final reports race with status readers, so the transaction identity must be safely published. + private volatile long txnId; private String label; // for export @@ -2567,7 +2568,7 @@ private void updateScanRangeNumByScanRange(TScanRangeParams param) { } // update job progress from BE - public void updateFragmentExecStatus(TReportExecStatusParams params) { + public boolean updateFragmentExecStatus(TReportExecStatusParams params) { if (params.isSetLoadedRows() && jobId != -1) { if (params.isSetFragmentInstanceReports()) { for (TFragmentInstanceReport report : params.getFragmentInstanceReports()) { @@ -2587,87 +2588,108 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) { } PipelineExecContext ctx = pipelineExecContexts.get(Pair.of(params.getFragmentId(), params.getBackendId())); - if (ctx == null || !ctx.updatePipelineStatus(params)) { + boolean hasExternalCommitData = params.isSetHivePartitionUpdates() + || params.isSetIcebergCommitDatas() || params.isSetMcCommitDatas() + || params.isSetPaimonCommitMessages(); + if (ctx == null) { + if (hasExternalCommitData) { + throw new IllegalStateException("Missing fragment handler for external-file report"); + } + return false; + } + if (!ctx.updatePipelineStatus(params)) { + if (hasExternalCommitData && !ctx.done) { + throw new IllegalStateException("External-file report was not a completed fragment report"); + } LOG.debug("Fragment {} is not done, ignore report status: {}", params.getFragmentId(), params.toString()); - return; + return ctx.done; } - Status status = new Status(params.status); - // for now, abort the query if we see any error except if the error is cancelled - // and returned_all_results_ is true. - // (UpdateStatus() initiates cancellation, if it hasn't already been initiated) - if (!status.ok()) { - if (returnedAllResults && status.isCancelled()) { - LOG.warn("Query {} has returned all results, fragment_id={} instance_id={}, be={}" - + " is reporting failed status {}", - DebugUtil.printId(queryId), params.getFragmentId(), - DebugUtil.printId(params.getFragmentInstanceId()), - params.getBackendId(), - status.toString()); - } else { - LOG.warn("one instance report fail, query_id={} fragment_id={} instance_id={}, be={}," - + " error message: {}", - DebugUtil.printId(queryId), params.getFragmentId(), - DebugUtil.printId(params.getFragmentInstanceId()), - params.getBackendId(), status.toString()); - updateStatus(status); + boolean accepted = false; + try { + Status status = new Status(params.status); + // for now, abort the query if we see any error except if the error is cancelled + // and returned_all_results_ is true. + // (UpdateStatus() initiates cancellation, if it hasn't already been initiated) + if (!status.ok()) { + if (returnedAllResults && status.isCancelled()) { + LOG.warn("Query {} has returned all results, fragment_id={} instance_id={}, be={}" + + " is reporting failed status {}", + DebugUtil.printId(queryId), params.getFragmentId(), + DebugUtil.printId(params.getFragmentInstanceId()), + params.getBackendId(), + status.toString()); + } else { + LOG.warn("one instance report fail, query_id={} fragment_id={} instance_id={}, be={}," + + " error message: {}", + DebugUtil.printId(queryId), params.getFragmentId(), + DebugUtil.printId(params.getFragmentInstanceId()), + params.getBackendId(), status.toString()); + updateStatus(status); + } } - } - if (params.isSetDeltaUrls() && deltaUrls != null) { - updateDeltas(params.getDeltaUrls()); - } - if (params.isSetLoadCounters() && loadCounters != null) { - updateLoadCounters(params.getLoadCounters()); - } - if (params.isSetTrackingUrl()) { - LOG.info("query_id={} tracking_url: {}", DebugUtil.printId(queryId), params.getTrackingUrl()); - trackingUrl = params.getTrackingUrl(); - } - if (params.isSetFirstErrorMsg()) { - LOG.info("query_id={} first_error_msg: {}", DebugUtil.printId(queryId), params.getFirstErrorMsg()); - firstErrorMsg = params.getFirstErrorMsg(); - } - if (params.isSetTxnId()) { - txnId = params.getTxnId(); - } - if (params.isSetLabel()) { - label = params.getLabel(); - } - if (params.isSetExportFiles()) { - updateExportFiles(params.getExportFiles()); - } - if (params.isSetCommitInfos()) { - updateCommitInfos(params.getCommitInfos()); - } - if (params.isSetErrorTabletInfos()) { - updateErrorTabletInfos(params.getErrorTabletInfos()); - } - if (params.isSetHivePartitionUpdates()) { - ((HMSTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) - .updateHivePartitionUpdates(params.getHivePartitionUpdates()); - } - if (params.isSetIcebergCommitDatas()) { - ((IcebergTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) - .updateIcebergCommitData(params.getIcebergCommitDatas()); - } - if (params.isSetMcCommitDatas()) { - ((MCTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) - .updateMCCommitData(params.getMcCommitDatas()); - } - if (params.isSetPaimonCommitMessages()) { - ((PaimonTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() - .getTxnById(txnId)) - .updateCommitMessages(params.getPaimonCommitMessages()); + if (params.isSetDeltaUrls() && deltaUrls != null) { + updateDeltas(params.getDeltaUrls()); + } + if (params.isSetLoadCounters() && loadCounters != null) { + updateLoadCounters(params.getLoadCounters()); + } + if (params.isSetTrackingUrl()) { + LOG.info("query_id={} tracking_url: {}", DebugUtil.printId(queryId), params.getTrackingUrl()); + trackingUrl = params.getTrackingUrl(); + } + if (params.isSetFirstErrorMsg()) { + LOG.info("query_id={} first_error_msg: {}", DebugUtil.printId(queryId), params.getFirstErrorMsg()); + firstErrorMsg = params.getFirstErrorMsg(); + } + // Keep this report's identity local so another report cannot redirect its commit data. + long reportTxnId = params.isSetTxnId() ? params.getTxnId() : txnId; + if (params.isSetTxnId()) { + txnId = reportTxnId; + } + if (params.isSetLabel()) { + label = params.getLabel(); + } + if (params.isSetExportFiles()) { + updateExportFiles(params.getExportFiles()); + } + if (params.isSetCommitInfos()) { + updateCommitInfos(params.getCommitInfos()); + } + if (params.isSetErrorTabletInfos()) { + updateErrorTabletInfos(params.getErrorTabletInfos()); + } + if (params.isSetHivePartitionUpdates()) { + ((HMSTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() + .getTxnById(reportTxnId)).updateHivePartitionUpdates(params.getHivePartitionUpdates()); + } + if (params.isSetIcebergCommitDatas()) { + ((IcebergTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() + .getTxnById(reportTxnId)).updateIcebergCommitData(params.getIcebergCommitDatas()); + } + if (params.isSetMcCommitDatas()) { + ((MCTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() + .getTxnById(reportTxnId)).updateMCCommitData(params.getMcCommitDatas()); + } + if (params.isSetPaimonCommitMessages()) { + ((PaimonTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() + .getTxnById(reportTxnId)).updateCommitMessages(params.getPaimonCommitMessages()); + } + + accepted = true; + } finally { + ctx.finishPipelineStatus(accepted); } - if (ctx.done) { + if (accepted) { if (LOG.isDebugEnabled()) { LOG.debug("Query {} fragment {} is marked done", DebugUtil.printId(queryId), ctx.fragmentId); } fragmentsDoneLatch.markedCountDown(params.getFragmentId(), params.getBackendId()); } + return accepted; } /* @@ -3074,7 +3096,9 @@ public static class PipelineExecContext { TPipelineFragmentParams rpcParams; PlanFragmentId fragmentId; boolean initiated; - boolean done; + // Non-final reports read this outside the monitor after updatePipelineStatus returns. + volatile boolean done; + boolean processingDoneReport; TNetworkAddress brpcAddress; TNetworkAddress address; @@ -3131,10 +3155,30 @@ public synchronized boolean updatePipelineStatus(TReportExecStatusParams params) // duplicate packet return false; } - this.done = true; + while (processingDoneReport) { + try { + wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for a duplicate report", e); + } + if (this.done) { + return false; + } + } + // Serialize ownership processing so no duplicate can be acknowledged before acceptance finishes. + processingDoneReport = true; return true; } + public synchronized void finishPipelineStatus(boolean accepted) { + if (accepted) { + this.done = true; + } + processingDoneReport = false; + notifyAll(); + } + public boolean isBackendStateHealthy() { if (backend.getLastMissingHeartbeatTime() > lastMissingHeartbeatTime && !backend.isAlive()) { LOG.warn("backend {} is down while joining the coordinator. job id: {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java index 6365f26c4f9c2a..bc2b991032adef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java @@ -26,7 +26,7 @@ public interface JobProcessor { void cancel(Status cancelReason); - void updateFragmentExecStatus(TReportExecStatusParams params); + boolean updateFragmentExecStatus(TReportExecStatusParams params); void tryFinishSchedule(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index f8c7509f102678..ac90508a289d38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -260,8 +260,8 @@ public boolean isDone() { } @Override - public void updateFragmentExecStatus(TReportExecStatusParams params) { - coordinatorContext.getJobProcessor().updateFragmentExecStatus(params); + public boolean updateFragmentExecStatus(TReportExecStatusParams params) { + return coordinatorContext.getJobProcessor().updateFragmentExecStatus(params); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java index ff023aeb9394de..c893e02ee3d909 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java @@ -37,6 +37,8 @@ import org.apache.doris.thrift.TUniqueId; import com.google.common.base.Strings; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,6 +49,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; public final class QeProcessorImpl implements QeProcessor { @@ -57,6 +60,10 @@ public final class QeProcessorImpl implements QeProcessor { private Map queryToInstancesNum; private Map userToInstancesCount; private ExecutorService writeProfileExecutor; + private final Cache acceptedExternalFileReports = CacheBuilder.newBuilder() + .maximumSize(1_000_000) + .expireAfterWrite(30, TimeUnit.MINUTES) + .build(); public static final QeProcessor INSTANCE; @@ -277,22 +284,69 @@ public TReportExecStatusResult reportExecStatus(TReportExecStatusParams params, } } + boolean hasExternalCommitData = hasExternalCommitData(params); + String reportKey = hasExternalCommitData ? externalFileReportKey(params) : null; + if (hasExternalCommitData && reportKey == null) { + return rejectedExternalFileReport(result, "External-file report is missing its identity fields"); + } + if (hasExternalCommitData && acceptedExternalFileReports.getIfPresent(reportKey) != null) { + // Keep acceptance available after coordinator removal so a lost response is retry-safe. + result.setStatus(new TStatus(TStatusCode.OK)); + result.setExternalFileCommitDataAccepted(true); + return result; + } + final QueryInfo info = coordinatorMap.get(params.query_id); result.setStatus(new TStatus(TStatusCode.OK)); if (info == null) { // Currently, the execution of query is splited from the exec status process. // So, it is very likely that when exec status arrived on FE asynchronously, coordinator // has been removed from coordinatorMap. - return result; + return hasExternalCommitData + ? rejectedExternalFileReport(result, "Coordinator no longer owns this external-file report") + : result; } try { - info.getCoord().updateFragmentExecStatus(params); + boolean accepted = info.getCoord().updateFragmentExecStatus(params); + if (hasExternalCommitData && !accepted) { + return rejectedExternalFileReport(result, "FE has not accepted the external-file report"); + } } catch (Exception e) { LOG.warn("Exception during handle report, response: {}, query: {}, instance: {}", result.toString(), DebugUtil.printId(params.query_id), DebugUtil.printId(params.fragment_instance_id), e); - return result; + return hasExternalCommitData + ? rejectedExternalFileReport(result, "FE did not accept the external-file report") + : result; } result.setStatus(new TStatus(TStatusCode.OK)); + if (hasExternalCommitData) { + // Publish the retry token before replying; a transport loss cannot revoke FE ownership. + acceptedExternalFileReports.put(reportKey, Boolean.TRUE); + result.setExternalFileCommitDataAccepted(true); + } + return result; + } + + private static boolean hasExternalCommitData(TReportExecStatusParams params) { + // Paimon uses the same ownership-transfer report path in the branch-4.1 connector architecture. + return params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() + || params.isSetMcCommitDatas() || params.isSetPaimonCommitMessages(); + } + + private static String externalFileReportKey(TReportExecStatusParams params) { + if (!params.isSetQueryId() || !params.isSetFragmentId() || !params.isSetBackendId()) { + return null; + } + return params.getQueryId().getHi() + ":" + params.getQueryId().getLo() + ":" + + params.getFragmentId() + ":" + params.getBackendId(); + } + + private static TReportExecStatusResult rejectedExternalFileReport( + TReportExecStatusResult result, String message) { + TStatus status = new TStatus(TStatusCode.INTERNAL_ERROR); + status.addToErrorMsgs(message); + result.setStatus(status); + result.setExternalFileCommitDataAccepted(false); return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 0df70d302881aa..94c9423abf76d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -5548,6 +5548,9 @@ public boolean isRequireSequenceInInsert() { */ public TQueryOptions toThrift() { TQueryOptions tResult = new TQueryOptions(); + // BE must size ownership-bearing reports against the receiving FE's actual Thrift limit. + tResult.setCoordinatorThriftMaxMessageSize(Config.thrift_max_message_size); + tResult.setSupportsExternalFileReportAck(true); tResult.setMemLimit(maxExecMemByte); tResult.setLocalExchangeFreeBlocksLimit(localExchangeFreeBlocksLimit); tResult.setScanQueueMemLimit(maxScanQueueMemByte); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java index c2b769196af07e..6fa059ed18c783 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java @@ -182,12 +182,38 @@ protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleF } } - if (!fragmentTask.processReportExecStatus(params)) { + if (!fragmentTask.processReportExecStatus(params, () -> acceptFinalReport(params))) { + if ((params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() + || params.isSetMcCommitDatas() || params.isSetPaimonCommitMessages()) + && !fragmentTask.isDone()) { + throw new IllegalStateException("External-file report was not a completed fragment report"); + } LOG.debug("Fragment {} is not done, ignore report status: {}", params.getFragmentId(), params.toString()); return; } + if (fragmentTask.isDone()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Query {} fragment {} is marked done", + DebugUtil.printId(coordinatorContext.queryId), params.getFragmentId()); + } + MarkedCountDownLatch latch = this.latch.get(); + latch.markedCountDown(params.getFragmentId(), params.getBackendId()); + + int topFragmentId = coordinatorContext.topDistributedPlan + .getFragmentJob().getFragment().getFragmentId().asInt(); + if (topFragmentId == params.getFragmentId()) { + MarkedCountDownLatch topFragmentLatch = this.topFragmentLatch.get(); + topFragmentLatch.markedCountDown(params.getFragmentId(), params.getBackendId()); + if (topFragmentLatch.getCount() == 0) { + tryFinishSchedule(); + } + } + } + } + + private void acceptFinalReport(TReportExecStatusParams params) { LoadContext loadContext = coordinatorContext.asLoadProcessor().loadContext; if (params.isSetDeltaUrls()) { loadContext.updateDeltaUrls(params.getDeltaUrls()); @@ -234,25 +260,6 @@ protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleF .getTxnById(txnId)) .updateCommitMessages(params.getPaimonCommitMessages()); } - - if (fragmentTask.isDone()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Query {} fragment {} is marked done", - DebugUtil.printId(coordinatorContext.queryId), params.getFragmentId()); - } - MarkedCountDownLatch latch = this.latch.get(); - latch.markedCountDown(params.getFragmentId(), params.getBackendId()); - - int topFragmentId = coordinatorContext.topDistributedPlan - .getFragmentJob().getFragment().getFragmentId().asInt(); - if (topFragmentId == params.getFragmentId()) { - MarkedCountDownLatch topFragmentLatch = this.topFragmentLatch.get(); - topFragmentLatch.markedCountDown(params.getFragmentId(), params.getBackendId()); - if (topFragmentLatch.getCount() == 0) { - tryFinishSchedule(); - } - } - } } // Check backend health for every unfinished load fragment task. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java index c6110d6a35be01..2b5b685bdd0f27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java @@ -58,12 +58,19 @@ public SingleFragmentPipelineTask(Backend backend, int fragmentId, Set mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(false); + String displayedType = IndexSchemaProcNode.createResult( + Lists.newArrayList(column), null, + Lists.newArrayList(IndexSchemaProcNode.COMMENT_COLUMN_TITLE)) + .getRows().get(0).get(1); + + Assert.assertTrue(displayedType.contains("comment \"owner's \\\\path\"")); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSTransactionPathTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSTransactionPathTest.java index c1535e415e5e8b..461f27e77ca5f5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSTransactionPathTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSTransactionPathTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.hive; import org.apache.doris.backup.Status; +import org.apache.doris.datasource.NameMapping; import org.apache.doris.fs.FileSystem; import org.apache.doris.fs.FileSystemProvider; import org.apache.doris.fs.LocalDfsFileSystem; @@ -26,6 +27,7 @@ import org.apache.doris.fs.remote.S3FileSystem; import org.apache.doris.fs.remote.SwitchingFileSystem; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TFileType; import org.apache.doris.thrift.THiveLocationParams; import org.apache.doris.thrift.THivePartitionUpdate; import org.apache.doris.thrift.TS3MPUPendingUpload; @@ -42,6 +44,7 @@ import java.lang.reflect.Field; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; @@ -313,4 +316,38 @@ public void testRollbackAbortsPendingMpuBeforeCommitterCreated() throws Exceptio Assert.assertEquals("warehouse/table/data-0.parquet", request.getValue().key()); Assert.assertEquals("upload-id-1", request.getValue().uploadId()); } + + @Test + public void testMergePreservesLaterMultipartRecordsAfterLegacyNullList() { + HMSTransaction tx = createTransaction(Mockito.mock(FileSystem.class)); + THivePartitionUpdate legacy = new THivePartitionUpdate() + .setName("part=1").setFileSize(1).setRowCount(1) + .setFileNames(new ArrayList<>(Collections.singletonList("old.parquet"))); + TS3MPUPendingUpload upload = new TS3MPUPendingUpload() + .setBucket("bucket").setKey("new.parquet").setUploadId("upload-id"); + THivePartitionUpdate current = new THivePartitionUpdate() + .setName("part=1").setFileSize(2).setRowCount(2) + .setFileNames(new ArrayList<>(Collections.singletonList("new.parquet"))) + .setS3MpuPendingUploads(new ArrayList<>(Collections.singletonList(upload))); + + THivePartitionUpdate merged = tx.mergePartitions(Arrays.asList(legacy, current)).get(0); + + Assert.assertEquals(Collections.singletonList(upload), merged.getS3MpuPendingUploads()); + Assert.assertEquals(Arrays.asList("old.parquet", "new.parquet"), merged.getFileNames()); + } + + @Test + public void testObjectStoreCommitRequiresOneCompleteRecordPerFile() { + HMSTransaction tx = createTransaction(Mockito.mock(FileSystem.class)); + tx.fileType = TFileType.FILE_S3; + THivePartitionUpdate update = new THivePartitionUpdate() + .setName("").setFileSize(1).setRowCount(1) + .setFileNames(Collections.singletonList("data.parquet")); + tx.updateHivePartitionUpdates(Collections.singletonList(update)); + + IllegalStateException error = Assert.assertThrows(IllegalStateException.class, + () -> tx.finishInsertTable(NameMapping.createForTest("db", "table"))); + + Assert.assertTrue(error.getMessage().contains("reported 1 file(s)")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index e2d923f3438863..830f9047e135ba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -435,8 +435,8 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(ctdList); IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); - txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); ctx.setOverwrite(true); + txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); txn.commit(); } @@ -460,8 +460,8 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { IcebergTransaction txn = getTxn(); IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); - txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); ctx.setOverwrite(true); + txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); txn.commit(); } @@ -469,6 +469,45 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { checkSnapshotTotalProperties(table.currentSnapshot().summary(), "0", "0", "0"); } + @Test + public void testEmptyOverwriteReadsAndCommitsTheTargetBranch() throws UserException, IOException { + testUnPartitionedTable(); + + Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + String branch = "overwrite_branch"; + table.manageSnapshots().createBranch(branch, table.currentSnapshot().snapshotId()).commit(); + Path mainOnlyFile = Files.createTempFile("main-only-data-", ".parquet"); + table.newFastAppend() + .appendFile(DataFiles.builder(table.spec()) + .withPath(mainOnlyFile.toString()) + .withFileSizeInBytes(1) + .withRecordCount(1) + .withFormat(FileFormat.PARQUET) + .build()) + .commit(); + long mainSnapshotId = table.currentSnapshot().snapshotId(); + IcebergExternalTable icebergExternalTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(icebergExternalTable.getCatalog()).thenReturn(spyExternalCatalog); + Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); + Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); + + try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { + mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + .thenReturn(table); + IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); + ctx.setOverwrite(true); + ctx.setBranchName(Optional.of(branch)); + IcebergTransaction txn = getTxn(); + txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); + txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); + txn.commit(); + } + + table.refresh(); + Assert.assertEquals(mainSnapshotId, table.currentSnapshot().snapshotId()); + checkSnapshotTotalProperties(table.snapshot(branch).summary(), "0", "0", "0"); + } + @Test public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() throws UserException { List ctdList = new ArrayList<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesActionTest.java new file mode 100644 index 00000000000000..4675812128bee0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesActionTest.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg.action; + +import org.apache.doris.common.UserException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashSet; + +public class IcebergRemoveOrphanFilesActionTest { + @Test + public void testCanonicalFileIdentityAndCollisionSafety() { + Assertions.assertTrue(IcebergRemoveOrphanFilesAction.sameFileIdentity( + "s3a://BUCKET/table/data.parquet", "s3://bucket/table/data.parquet")); + Assertions.assertFalse(IcebergRemoveOrphanFilesAction.sameFileIdentity( + "s3://bucket-a/table/data.parquet", "s3://bucket-b/table/data.parquet")); + Assertions.assertThrows(UserException.class, + () -> IcebergRemoveOrphanFilesAction.verifyReachableIndexLimit( + new HashSet<>(Arrays.asList("s3://bucket-a/table/data.parquet", + "s3://bucket-b/table/data.parquet")), 2)); + } + + @Test + public void testReachableIndexIsBounded() { + Assertions.assertThrows(UserException.class, + () -> IcebergRemoveOrphanFilesAction.verifyReachableIndexLimit( + new HashSet<>(Arrays.asList("s3://bucket/table/a", "s3://bucket/table/b")), 1)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 670754614ddbad..94d6ea42717ff5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -1308,6 +1308,39 @@ public void testRejectSmoothUpgradeSourceBackendForVariantProjection() throws Ex } } + @Test + public void testRejectSmoothUpgradeSourceBackendForScanSemanticsV2() throws Exception { + Backend currentBackend = Mockito.mock(Backend.class); + Mockito.when(currentBackend.isSmoothUpgradeSrc()).thenReturn(false); + Backend smoothUpgradeSource = Mockito.mock(Backend.class); + Mockito.when(smoothUpgradeSource.isSmoothUpgradeSrc()).thenReturn(true); + Mockito.when(smoothUpgradeSource.getId()).thenReturn(10004L); + List backends = ImmutableList.of(currentBackend, smoothUpgradeSource); + + IcebergScanNode.checkIcebergScanSemanticsV2Compatibility(false, backends); + try { + IcebergScanNode.checkIcebergScanSemanticsV2Compatibility(true, backends); + Assert.fail("nested default and requiredness semantics must not run on a V1 backend"); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("backend 10004 is a smooth upgrade source")); + } + } + + @Test + public void testRequirednessHistoryTriggersCurrentScanSemantics() { + Schema current = new Schema(2, ImmutableList.of( + Types.NestedField.required(1, "id", Types.LongType.get()))); + Schema historicalOptional = new Schema(1, ImmutableList.of( + Types.NestedField.optional(1, "id", Types.LongType.get()))); + Schema historicalRequired = new Schema(1, ImmutableList.of( + Types.NestedField.required(1, "id", Types.LongType.get()))); + + Assert.assertTrue(IcebergScanNode.schemaHistoryRequiresMissingRequiredFieldRejection( + current, ImmutableList.of(historicalOptional))); + Assert.assertFalse(IcebergScanNode.schemaHistoryRequiresMissingRequiredFieldRejection( + current, ImmutableList.of(historicalRequired))); + } + @Test public void testBatchVariantProjectionUsesSharedCompatibilityGate() throws Exception { TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable(), false, true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/AzureObjStorageTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/AzureObjStorageTest.java index 5e1681adbda6ef..88899e812bbd87 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/AzureObjStorageTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/AzureObjStorageTest.java @@ -43,6 +43,8 @@ import java.nio.file.Paths; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -189,6 +191,19 @@ public void testMockObj() { // } } + @Test + public void testMultipartCompletionUsesExactStagedBlockIds() { + Map parts = new HashMap<>(); + parts.put(2, "writer-b-part-2"); + parts.put(1, "writer-b-part-1"); + + Assertions.assertEquals( + Arrays.asList("writer-b-part-1", "writer-b-part-2"), + AzureObjStorage.multipartBlockIds(parts)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> AzureObjStorage.multipartBlockIds(Collections.singletonMap(1, ""))); + } + /** * Mock an AzureObjStorage of which getPagedBlobItems() will return objects and split into multiple batches * for testing continuations diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java new file mode 100644 index 00000000000000..1b9d461e00ea4c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.common.profile.ExecutionProfile; +import org.apache.doris.planner.PlanFragmentId; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPaimonCommitMessage; +import org.apache.doris.thrift.TQueryOptions; +import org.apache.doris.thrift.TReportExecStatusParams; +import org.apache.doris.thrift.TReportExecStatusResult; +import org.apache.doris.thrift.TStatus; +import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.cache.Cache; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.Collections; + +class QeProcessorImplReportAckTest { + private TUniqueId registeredQueryId; + + @AfterEach + void cleanup() { + if (registeredQueryId != null) { + QeProcessorImpl.INSTANCE.unregisterQuery(registeredQueryId); + } + } + + @Test + void rejectsExternalReportWithoutCoordinator() { + TReportExecStatusResult result = report(params(new TUniqueId(12345, 1))); + + Assertions.assertEquals(TStatusCode.INTERNAL_ERROR, result.getStatus().getStatusCode()); + Assertions.assertFalse(result.isExternalFileCommitDataAccepted()); + } + + @Test + void rejectsExternalReportWhenHandlerThrows() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 2); + Coordinator coordinator = register(queryId); + Mockito.doThrow(new RuntimeException("injected acceptance failure")) + .when(coordinator).updateFragmentExecStatus(Mockito.any()); + + TReportExecStatusResult result = report(params(queryId)); + + Assertions.assertEquals(TStatusCode.INTERNAL_ERROR, result.getStatus().getStatusCode()); + Assertions.assertFalse(result.isExternalFileCommitDataAccepted()); + } + + @Test + void rejectsExternalReportWhenHandlerDoesNotAcceptIt() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 4); + register(queryId); + + TReportExecStatusResult result = report(params(queryId)); + + Assertions.assertEquals(TStatusCode.INTERNAL_ERROR, result.getStatus().getStatusCode()); + Assertions.assertFalse(result.isExternalFileCommitDataAccepted()); + } + + @Test + void retriesAcceptedExternalReportAfterCoordinatorRemoval() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 3); + Coordinator coordinator = register(queryId); + Mockito.when(coordinator.updateFragmentExecStatus(Mockito.any())).thenReturn(true); + TReportExecStatusParams params = params(queryId); + + TReportExecStatusResult first = report(params); + QeProcessorImpl.INSTANCE.unregisterQuery(queryId); + registeredQueryId = null; + TReportExecStatusResult retry = report(params); + + Assertions.assertTrue(first.isExternalFileCommitDataAccepted()); + Assertions.assertTrue(retry.isExternalFileCommitDataAccepted()); + Assertions.assertEquals(TStatusCode.OK, retry.getStatus().getStatusCode()); + Mockito.verify(coordinator, Mockito.times(1)).updateFragmentExecStatus(params); + } + + @Test + void paimonReportUsesTheSameOwnershipAcknowledgement() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 6); + Coordinator coordinator = register(queryId); + Mockito.when(coordinator.updateFragmentExecStatus(Mockito.any())).thenReturn(true); + TReportExecStatusParams paimonParams = params(queryId); + paimonParams.unsetIcebergCommitDatas(); + paimonParams.setPaimonCommitMessages(Collections.emptyList()); + + TReportExecStatusResult result = report(paimonParams); + + Assertions.assertEquals(TStatusCode.OK, result.getStatus().getStatusCode()); + Assertions.assertTrue(result.isExternalFileCommitDataAccepted()); + } + + @Test + void evictedAcceptanceTokenRejectsRetryAfterCoordinatorRemoval() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 5); + Coordinator coordinator = register(queryId); + Mockito.when(coordinator.updateFragmentExecStatus(Mockito.any())).thenReturn(true); + TReportExecStatusParams params = params(queryId); + + TReportExecStatusResult first = report(params); + QeProcessorImpl.INSTANCE.unregisterQuery(queryId); + registeredQueryId = null; + acceptedExternalFileReports().invalidateAll(); + TReportExecStatusResult retry = report(params); + + Assertions.assertTrue(first.isExternalFileCommitDataAccepted()); + Assertions.assertFalse(retry.isExternalFileCommitDataAccepted()); + Assertions.assertEquals(TStatusCode.INTERNAL_ERROR, retry.getStatus().getStatusCode()); + Mockito.verify(coordinator, Mockito.times(1)).updateFragmentExecStatus(params); + } + + @Test + void coordinatorReportStateUsesCrossThreadVisibility() throws Exception { + Field done = Coordinator.PipelineExecContext.class.getDeclaredField("done"); + Field txnId = Coordinator.class.getDeclaredField("txnId"); + + Assertions.assertTrue(Modifier.isVolatile(done.getModifiers())); + Assertions.assertTrue(Modifier.isVolatile(txnId.getModifiers())); + } + + @Test + void legacyCoordinatorRetriesFailedAcceptanceBeforeMarkingDone() { + Backend backend = Mockito.mock(Backend.class); + Mockito.when(backend.getHost()).thenReturn("127.0.0.1"); + ExecutionProfile profile = Mockito.mock(ExecutionProfile.class); + Coordinator.PipelineExecContext context = new Coordinator.PipelineExecContext( + new PlanFragmentId(7), null, backend, profile, -1); + TReportExecStatusParams report = new TReportExecStatusParams().setDone(true); + + Assertions.assertTrue(context.updatePipelineStatus(report)); + context.finishPipelineStatus(false); + Assertions.assertTrue(context.updatePipelineStatus(report)); + context.finishPipelineStatus(true); + Assertions.assertFalse(context.updatePipelineStatus(report)); + } + + private Coordinator register(TUniqueId queryId) throws Exception { + Coordinator coordinator = Mockito.mock(Coordinator.class); + Mockito.when(coordinator.getQueryOptions()).thenReturn(new TQueryOptions()); + QeProcessorImpl.INSTANCE.registerQuery(queryId, new QeProcessorImpl.QueryInfo(coordinator)); + registeredQueryId = queryId; + return coordinator; + } + + private static TReportExecStatusParams params(TUniqueId queryId) { + return new TReportExecStatusParams() + .setQueryId(queryId) + .setFragmentId(7) + .setBackendId(9) + .setDone(true) + .setStatus(new TStatus(TStatusCode.OK)) + .setIcebergCommitDatas(Collections.emptyList()); + } + + private static TReportExecStatusResult report(TReportExecStatusParams params) { + return QeProcessorImpl.INSTANCE.reportExecStatus(params, new TNetworkAddress("127.0.0.1", 9050)); + } + + @SuppressWarnings("unchecked") + private static Cache acceptedExternalFileReports() throws Exception { + Field field = QeProcessorImpl.class.getDeclaredField("acceptedExternalFileReports"); + field.setAccessible(true); + return (Cache) field.get(QeProcessorImpl.INSTANCE); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java index 85fec53775251c..12464031edc824 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java @@ -335,4 +335,14 @@ public void testFileCacheQueryLimitBytesToThrift() throws Exception { Assertions.assertTrue(queryOptions.isSetFileCacheQueryLimitBytes()); Assertions.assertEquals(262144L, queryOptions.getFileCacheQueryLimitBytes()); } + + @Test + public void testCoordinatorThriftLimitPropagatesToBackends() { + TQueryOptions queryOptions = new SessionVariable().toThrift(); + + Assertions.assertTrue(queryOptions.isSetCoordinatorThriftMaxMessageSize()); + Assertions.assertEquals(Config.thrift_max_message_size, + queryOptions.getCoordinatorThriftMaxMessageSize()); + Assertions.assertTrue(queryOptions.isSupportsExternalFileReportAck()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java index 31becae01dc9db..281da9ab3dfe1d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.Status; import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TReportExecStatusParams; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; @@ -28,6 +29,20 @@ import java.util.Collections; class SingleFragmentPipelineTaskTest { + @Test + void failedAcceptanceLeavesFinalReportRetryable() { + SingleFragmentPipelineTask task = createTask(createBackend(100L)); + TReportExecStatusParams report = new TReportExecStatusParams().setDone(true); + + Assertions.assertThrows(RuntimeException.class, + () -> task.processReportExecStatus(report, () -> { + throw new RuntimeException("injected failure"); + })); + Assertions.assertFalse(task.isDone()); + Assertions.assertTrue(task.processReportExecStatus(report, () -> { })); + Assertions.assertTrue(task.isDone()); + } + @Test void backendWithUnchangedProcessEpochIsHealthy() { Backend backend = createBackend(100L); diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 17e59f24140fbe..b7dc44622a9a11 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -384,6 +384,7 @@ struct THiveTableSink { 10: optional bool overwrite 11: optional THiveSerDeProperties serde_properties 12: optional list broker_addresses; + 13: optional bool supports_deferred_azure_multipart } enum TUpdateMode { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index f06ceb3ceb898d..cd445579a634ac 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -183,6 +183,8 @@ struct TListPrivilegesResult{ struct TReportExecStatusResult { // required in V1 1: optional Status.TStatus status + // Set only after FE accepts the external-file commit vectors for this report. + 2: optional bool external_file_commit_data_accepted } // Service Protocol Details diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index d98c4b3961f308..c3d30cdbbc209a 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -508,6 +508,11 @@ struct TQueryOptions { 227: optional i64 file_presigned_url_ttl_seconds = 3600; + // FE receives fragment reports, so BE must also honor its message limit. + 229: optional i32 coordinator_thrift_max_message_size; + // FE can explicitly and idempotently acknowledge external-file commit reports. + 230: optional bool supports_external_file_report_ack = false; + // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/gensrc/thrift/Partitions.thrift b/gensrc/thrift/Partitions.thrift index 738636961157ac..a49a4c9d1b84c2 100644 --- a/gensrc/thrift/Partitions.thrift +++ b/gensrc/thrift/Partitions.thrift @@ -99,6 +99,8 @@ struct TIcebergPartitionField { 3: required Exprs.TExpr source_expr 4: optional string name 5: optional i32 source_id + // Zero-based STRUCT child indexes below source_expr; empty/unset means a top-level source. + 6: optional list source_field_path } struct TMergePartitionInfo { diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out index 76bede82d3f5fb..f3f51aac96e1c5 100644 --- a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out @@ -3,8 +3,9 @@ 1 A [1, null, 3] {"x":10, "null-value":null} {"metric":10, "label":"old-a", "nested":{"count":1, "comment":null, "score":null}, "tags":null, "attributes":null} 2 N \N {"x":null} {"metric":20, "label":null, "nested":{"count":null, "comment":"old-null", "score":null}, "tags":null, "attributes":null} 3 B [] {} \N -4 A1 [4000000000, null] {"large":5000000000, "null-value":null} {"metric":6000000000, "label":"new-a", "nested":{"count":7000000000, "comment":"nested-new", "score":7.5}, "tags":["x", null, "z"], "attributes":{"a":8000000000, "b":null}} +4 A2 [4000000000, null] {"large":5000000000, "null-value":null} {"metric":6000000000, "label":"new-a", "nested":{"count":7000000000, "comment":"nested-new", "score":7.5}, "tags":["x", null, "z"], "attributes":{"a":8000000000, "b":null}} 5 N2 [null] \N {"metric":50, "label":null, "nested":{"count":5, "comment":null, "score":null}, "tags":null, "attributes":{"null-value":null}} +6 Z4 \N \N \N -- !complex_children -- 1 10 1 \N \N \N @@ -27,4 +28,3 @@ 1 [1, null, 3] {"x":10, "null-value":null} 10 old-a 1 \N 2 \N {"x":null} 20 \N \N old-null 3 [] {} \N \N \N \N - diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 4f4130d94de992..1aab09558bca5a 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -2239,10 +2239,16 @@ class Suite implements GroovyInterceptable { } } + static String buildJobNameQuery(String dbName, String mtmvName) { + return ("select Name from jobs('type'='mv') where MvDatabaseName = '${dbName}' " + + "and MvName = '${mtmvName}'") + } + String getJobName(String dbName, String mtmvName) { - String showMTMV = "select JobName from mv_infos('database'='${dbName}') where Name = '${mtmvName}'"; - logger.info(showMTMV) - List> result = sql(showMTMV) + // Job lookup must not materialize unrelated MVs whose external metadata may be unavailable. + String showJob = buildJobNameQuery(dbName, mtmvName) + logger.info(showJob) + List> result = sql(showJob) logger.info("result: " + result.toString()) if (result.isEmpty()) { Assert.fail(); diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteJobLookupTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteJobLookupTest.groovy new file mode 100644 index 00000000000000..e66b1dad09ef86 --- /dev/null +++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteJobLookupTest.groovy @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import org.junit.jupiter.api.Test + +import static org.junit.jupiter.api.Assertions.assertEquals + +class SuiteJobLookupTest { + @Test + void jobLookupUsesJobMetadataWithoutMaterializingMvStatus() { + String query = Suite.buildJobNameQuery("db1", "mv1") + + assertEquals( + "select Name from jobs('type'='mv') where MvDatabaseName = 'db1' and MvName = 'mv1'", + query) + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.groovy index e5ad9e7c6ed19e..3460561577b036 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.groovy +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.groovy @@ -116,6 +116,15 @@ suite("test_iceberg_write_complex_evolution", """ sql """alter table complex_evolution add partition key bucket(8, id) as id_bucket""" sql """alter table complex_evolution add partition key truncate(1, group_key) as group_prefix""" + // Iceberg permits a nested primitive source. Create it through Spark to verify Doris can plan and + // physically partition the following INSERT by the schema-wide nested field id. Invalidate Spark's + // cached table first so its commit requirement sees the partition ids assigned by the Doris DDLs. + spark_iceberg """refresh table demo.${dbName}.complex_evolution""" + spark_iceberg """ + alter table demo.${dbName}.complex_evolution + add partition field bucket(4, payload.nested.count) + """ + sql """refresh table complex_evolution""" sql """ insert into complex_evolution values @@ -135,7 +144,15 @@ suite("test_iceberg_write_complex_evolution", struct(cast(5 as bigint), null, null), null, map('null-value', null) - )) + )), + (6, 'Z3', null, null, null) + """ + + // Route an UPDATE insert image by the nested source and preserve the parent-NULL partition value. + sql """ + update complex_evolution + set group_key = case id when 4 then 'A2' else 'Z4' end + where id in (4, 6) """ // W02-S03: Current schema reads both old and new files without moving old child values. @@ -166,6 +183,13 @@ suite("test_iceberg_write_complex_evolution", group by spec_id order by spec_id """ + order_qt_complex_nested_partition_pruning """ + select id + from complex_evolution + where payload.nested.count = cast(7000000000 as bigint) + or (id = 6 and payload.nested.count is null) + order by id + """ assertSparkMatchesDoris() // W02-S04: A pre-evolution tag binds the old files to their historical complex schema. From 5433110bcdb6aaac50ffe168c56cfa337425095a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 17 Aug 2026 15:08:41 +0800 Subject: [PATCH 2/4] [fix](iceberg) Address branch-4.1 review feedback --- .../pipeline/pipeline_fragment_context.cpp | 11 +++- be/src/exec/scan/access_path_parser.cpp | 4 ++ .../sink/writer/vhive_partition_writer.cpp | 16 ++++++ .../exec/sink/writer/vhive_partition_writer.h | 2 + .../iceberg_partition_function.cpp | 16 +++++- be/src/format_v2/table/iceberg_reader.cpp | 6 +- be/src/io/fs/obj_storage_client.h | 5 ++ be/src/io/fs/s3_file_writer.cpp | 16 ++++++ be/src/io/fs/s3_file_writer.h | 3 + be/src/io/fs/s3_obj_storage_client.cpp | 19 +++++++ be/src/io/fs/s3_obj_storage_client.h | 1 + be/test/core/value/merge_partitioner_test.cpp | 2 +- ...partition_writer_report_lifecycle_test.cpp | 31 ++++++++++ .../apache/doris/datasource/ExternalUtil.java | 48 ++++++++++++---- .../datasource/iceberg/IcebergUtils.java | 51 ++++++++++++++++- .../iceberg/source/IcebergScanNode.java | 22 +++++-- .../apache/doris/planner/HiveTableSink.java | 31 ++++++++-- .../apache/doris/qe/AbstractJobProcessor.java | 11 ++-- .../java/org/apache/doris/qe/Coordinator.java | 7 +-- .../org/apache/doris/qe/QeProcessorImpl.java | 17 +++--- .../doris/qe/runtime/LoadProcessor.java | 6 +- .../doris/datasource/ExternalUtilTest.java | 21 ++++++- .../datasource/iceberg/IcebergUtilsTest.java | 57 ++++++++++++++++++- .../iceberg/source/IcebergScanNodeTest.java | 2 + .../doris/planner/HiveTableSinkTest.java | 18 ++++++ .../qe/QeProcessorImplReportAckTest.java | 23 ++++++++ gensrc/thrift/ExternalTableSchema.thrift | 6 +- 27 files changed, 396 insertions(+), 56 deletions(-) diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 7e16f1c92d3060..490f6c7c65b68f 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -2201,6 +2201,9 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r try { (*coord)->reportExecStatus(res, params); } catch ([[maybe_unused]] apache::thrift::transport::TTransportException& e) { + // The coordinator may have durably accepted the first call before its response was + // lost. Once that happens, a retry rejection must never make BE delete staged files. + report_outcome_ambiguous = true; #ifndef ADDRESS_SANITIZER LOG(WARNING) << "Retrying ReportExecStatus. query id: " << print_id(req.query_id) << ", instance id: " << print_id(req.fragment_instance_id) << " to " @@ -2209,6 +2212,10 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r rpc_status = coord->reopen(); if (!rpc_status.ok()) { + if (req.done) { + req.runtime_state->finalize_external_file_report_cleanup( + ExternalFileReportOutcome::AMBIGUOUS); + } req.cancel_fn(rpc_status); return; } @@ -2222,8 +2229,8 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r PrintThriftNetworkAddress(req.coord_addr), e.what()); } - // Only Iceberg keeps BE rollback callbacks after close; the other vectors remain compatible - // with coordinators that acknowledge acceptance through the RPC status alone. + // Iceberg requires the explicit new-protocol ACK. Legacy Hive/Paimon coordinators transfer + // ownership through RPC success, which remains valid during a rolling FE upgrade. const bool requires_external_file_ack = params.__isset.iceberg_commit_datas; if (rpc_status.ok() && requires_external_file_ack && (!res.__isset.external_file_commit_data_accepted || diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index 36a08d531ed5e1..17300c289c2677 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -94,6 +94,10 @@ void inherit_schema_metadata(format::ColumnDefinition* column, // access-path pruning must retain them just like it retains rename metadata. column->initial_default_value = schema_column->initial_default_value; column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64; + // Access-path pruning rebuilds children, so carry the prepared typed literal and requiredness + // that make a missing Iceberg child semantically identical to the complete projection. + column->is_optional = schema_column->is_optional; + column->default_expr = schema_column->default_expr; } const format::ColumnDefinition* find_schema_child_by_path( diff --git a/be/src/exec/sink/writer/vhive_partition_writer.cpp b/be/src/exec/sink/writer/vhive_partition_writer.cpp index 40d7b38fc30236..a263f64433e183 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/exec/sink/writer/vhive_partition_writer.cpp @@ -147,6 +147,7 @@ Status VHivePartitionWriter::close(const Status& status) { result_status.to_string()); } } + _register_rejected_report_cleanup(); bool status_ok = result_status.ok() && status.ok(); if (!status_ok) { _add_s3_mpu_pending_upload_for_rollback(); @@ -167,6 +168,21 @@ Status VHivePartitionWriter::close(const Status& status) { return result_status; } +void VHivePartitionWriter::_register_rejected_report_cleanup() { + if (_rejected_report_cleanup_registered || _write_info.file_type != TFileType::FILE_S3 || + _file_writer == nullptr) { + return; + } + auto* s3_writer = dynamic_cast(_file_writer.get()); + if (s3_writer == nullptr || s3_writer->upload_id().empty()) { + return; + } + // The writer can be destroyed before the final RPC result; the cleanup owns only the client + // and immutable upload identity needed when FE explicitly rejects ownership. + _state->add_rejected_external_file_report_cleanup(s3_writer->rejected_report_cleanup()); + _rejected_report_cleanup_registered = true; +} + Status VHivePartitionWriter::write(Block& block) { RETURN_IF_ERROR(_file_format_transformer->write(block)); _row_count += block.rows(); diff --git a/be/src/exec/sink/writer/vhive_partition_writer.h b/be/src/exec/sink/writer/vhive_partition_writer.h index 92e316a95c8e10..e762060566b794 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.h +++ b/be/src/exec/sink/writer/vhive_partition_writer.h @@ -77,6 +77,7 @@ class VHivePartitionWriter { private: THivePartitionUpdate _build_partition_update(); bool _build_s3_mpu_pending_upload(TS3MPUPendingUpload* pending_upload); + void _register_rejected_report_cleanup(); void _add_s3_mpu_pending_upload_for_rollback(); std::string _get_file_extension(TFileFormatType::type file_format_type, @@ -102,6 +103,7 @@ class VHivePartitionWriter { const THiveSerDeProperties* _hive_serde_properties; const std::map& _hadoop_conf; bool _supports_deferred_azure_multipart = false; + bool _rejected_report_cleanup_registered = false; std::shared_ptr _fs = nullptr; diff --git a/be/src/format/transformer/iceberg_partition_function.cpp b/be/src/format/transformer/iceberg_partition_function.cpp index d22795122e50f0..64062e7ee5355e 100644 --- a/be/src/format/transformer/iceberg_partition_function.cpp +++ b/be/src/format/transformer/iceberg_partition_function.cpp @@ -123,10 +123,22 @@ Status IcebergInsertPartitionFunction::open(RuntimeState* state) { RETURN_IF_ERROR(VExpr::open(field_ctxs, state)); for (auto& field : _partition_fields) { try { + DataTypePtr source_type = field.expr_ctx->root()->data_type(); + for (int32_t child_index : field.source_field_path) { + const auto* struct_type = check_and_get_data_type( + remove_nullable(source_type).get()); + if (child_index < 0 || struct_type == nullptr || + static_cast(child_index) >= struct_type->get_elements().size()) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "Iceberg nested merge partition source does not match " + "expression type"); + } + // Transform validation must use the leaf type that get_partitions() extracts. + source_type = struct_type->get_element(static_cast(child_index)); + } doris::iceberg::PartitionField partition_field(field.source_id, 0, field.name, field.transform); - field.transformer = PartitionColumnTransforms::create( - partition_field, field.expr_ctx->root()->data_type()); + field.transformer = PartitionColumnTransforms::create(partition_field, source_type); } catch (const doris::Exception& e) { return Status::NotSupported("Unsupported Iceberg partition transform: {}", e.what()); diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 443a803b9b9216..fefe6b007d4002 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -512,9 +512,6 @@ Status IcebergTableReader::annotate_projected_column(const TFileScanSlotInfo& sl RETURN_IF_ERROR(format::TableReader::annotate_projected_column(slot_info, context, column)); DORIS_CHECK(context != nullptr); DORIS_CHECK(column != nullptr); - if (!supports_iceberg_scan_semantics_v2(context->scan_params)) { - return Status::OK(); - } if (!context->schema_column.has_value()) { return Status::OK(); } @@ -528,7 +525,8 @@ Status IcebergTableReader::annotate_projected_column(const TFileScanSlotInfo& sl // The Iceberg typed literal is authoritative. In particular, this replaces FE's generic // string expression for Base64-transported UUID/BINARY/FIXED defaults. column->default_expr = schema_column.default_expr; - } else if (schema_column.is_optional.has_value() && !*schema_column.is_optional) { + } else if (supports_iceberg_scan_semantics_v2(context->scan_params) && + schema_column.is_optional.has_value() && !*schema_column.is_optional) { // FE's generic external-column metadata currently treats Iceberg columns as nullable. Clear // that fallback so a physically missing required field is rejected by the Iceberg mapper. column->default_expr = nullptr; diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index 2688ee70e02a4b..45dce933acd69d 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -107,6 +107,11 @@ class ObjStorageClient { virtual ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) = 0; + // Providers without an explicit abort operation may let unpublished parts expire. S3 + // overrides this so a coordinator rejection can release multipart resources immediately. + virtual ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions&) { + return ObjectStorageResponse::OK(); + } // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. // If it exists, it will return the corresponding file size virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 03909e607001a9..b47eeff1f38fa2 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -107,6 +107,22 @@ Status S3FileWriter::_create_multi_upload_request() { return {resp.resp.status.code, std::move(resp.resp.status.msg)}; } +std::function S3FileWriter::rejected_report_cleanup() const { + auto client_holder = _obj_client; + auto path_opts = _obj_storage_path_opts; + return [client_holder = std::move(client_holder), path_opts = std::move(path_opts)]() { + auto client = client_holder->get(); + if (client == nullptr || !path_opts.upload_id.has_value()) { + return; + } + auto response = client->abort_multipart_upload(path_opts); + if (response.status.code != ErrorCode::OK) { + LOG(WARNING) << "Failed to abort rejected multipart upload " << path_opts.path.native() + << ": " << response.status.msg; + } + }; +} + void S3FileWriter::_wait_until_finish(std::string_view task_name) { auto timeout_duration = config::s3_file_writer_log_interval_second; auto msg = fmt::format( diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 83ec75c9184920..fbebbaeb11de2e 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -70,6 +71,8 @@ class S3FileWriter final : public FileWriter { : std::string(); } + std::function rejected_report_cleanup() const; + Status close(bool non_block = false) override; Status try_finish_close() override; diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index f9ed8e155ff59c..864b208634fe0a 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -300,6 +300,25 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( return ObjectStorageResponse::OK(); } +ObjectStorageResponse S3ObjStorageClient::abort_multipart_upload( + const ObjectStoragePathOptions& opts) { + if (!opts.upload_id.has_value()) { + return {convert_to_obj_response(Status::InvalidArgument("Missing multipart upload id"))}; + } + Aws::S3::Model::AbortMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); + auto outcome = s3_put_rate_limit([&]() { return _client->AbortMultipartUpload(request); }); + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + return {convert_to_obj_response( + s3fs_error(outcome.GetError(), + fmt::format("failed to abort multipart upload {}", opts.key))), + static_cast(outcome.GetError().GetResponseCode()), + outcome.GetError().GetRequestId()}; + } + return ObjectStorageResponse::OK(); +} + ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { Aws::S3::Model::HeadObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index 45294226594d81..10bcf6b2e9495b 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -43,6 +43,7 @@ class S3ObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, diff --git a/be/test/core/value/merge_partitioner_test.cpp b/be/test/core/value/merge_partitioner_test.cpp index 2df06815b44f75..094c635a2a448c 100644 --- a/be/test/core/value/merge_partitioner_test.cpp +++ b/be/test/core/value/merge_partitioner_test.cpp @@ -369,7 +369,7 @@ TEST_F(MergePartitionerTest, TestNestedInsertPartitionFieldPreservesParentNulls) TMergePartitionInfo merge_info = _make_base_merge_info(false); TIcebergPartitionField field; - field.__set_transform("identity"); + field.__set_transform("bucket[8]"); field.__set_source_expr(_make_nested_source_expr()); field.__set_name("payload_part"); field.__set_source_id(3); diff --git a/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp index 73862ac8b0580d..7de9569f7984d5 100644 --- a/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp +++ b/be/test/exec/sink/writer/vhive_partition_writer_report_lifecycle_test.cpp @@ -17,6 +17,7 @@ #include +#include #include #include #include @@ -36,6 +37,8 @@ namespace { class RecordingObjStorageClient final : public io::ObjStorageClient { public: + std::atomic abort_count {0}; + io::ObjectStorageUploadResponse create_multipart_upload( const io::ObjectStoragePathOptions&) override { return {.resp = io::ObjectStorageResponse::OK(), .upload_id = "upload-id"}; @@ -57,6 +60,11 @@ class RecordingObjStorageClient final : public io::ObjStorageClient { return io::ObjectStorageResponse::OK(); } + io::ObjectStorageResponse abort_multipart_upload(const io::ObjectStoragePathOptions&) override { + ++abort_count; + return io::ObjectStorageResponse::OK(); + } + io::ObjectStorageHeadResponse head_object(const io::ObjectStoragePathOptions&) override { return {.resp = io::ObjectStorageResponse::OK(), .file_size = 0}; } @@ -214,4 +222,27 @@ TEST(VHivePartitionWriterReportLifecycleTest, AzureFinalReportCarriesExactBlockI EXPECT_EQ("exact-block-id", pending_uploads[0].etags.at(1)); } +TEST(VHivePartitionWriterReportLifecycleTest, RejectedFinalReportAbortsStagedS3Upload) { + MockRuntimeState state; + VExprContextSPtrs output_exprs; + auto client = std::make_shared(); + auto writer = create_closed_hive_writer(&state, output_exprs, client); + + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + + EXPECT_EQ(1, client->abort_count.load()); +} + +TEST(VHivePartitionWriterReportLifecycleTest, AmbiguousFinalReportRetainsStagedS3Upload) { + MockRuntimeState state; + VExprContextSPtrs output_exprs; + auto client = std::make_shared(); + auto writer = create_closed_hive_writer(&state, output_exprs, client); + + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::AMBIGUOUS); + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + + EXPECT_EQ(0, client->abort_count.load()); +} + } // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 37dbf4cdd390ac..44bdea87e061a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -38,6 +38,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; public class ExternalUtil { private static TField getExternalSchema(Column column) { @@ -147,22 +148,40 @@ public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, List columns, Map> nameMapping, boolean hasNameMapping, Map base64InitialDefaults) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, hasNameMapping, + base64InitialDefaults, Collections.emptySet()); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults, Set binaryLikeFieldIds) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, hasNameMapping, + base64InitialDefaults, binaryLikeFieldIds, Collections.emptyMap()); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults, Set binaryLikeFieldIds, + Map fieldOptionality) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); tSchema.setRootField(getExternalSchemaForAllColumn( - columns, nameMapping, hasNameMapping, base64InitialDefaults)); + columns, nameMapping, hasNameMapping, base64InitialDefaults, binaryLikeFieldIds, + fieldOptionality)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, Map> nameMapping, boolean hasNameMapping, - Map base64InitialDefaults) { + Map base64InitialDefaults, Set binaryLikeFieldIds, + Map fieldOptionality) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - child.getType(), child, nameMapping, hasNameMapping, base64InitialDefaults)); + child.getType(), child, nameMapping, hasNameMapping, base64InitialDefaults, + binaryLikeFieldIds, fieldOptionality)); structField.addToFields(fieldPtr); } return structField; @@ -171,23 +190,30 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { return getExternalSchema(columnType, dorisColumn, nameMapping, - nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap(), + Collections.emptySet(), Collections.emptyMap()); } private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping, boolean hasNameMapping, - Map base64InitialDefaults) { + Map base64InitialDefaults, Set binaryLikeFieldIds, + Map fieldOptionality) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); - root.setIsOptional(dorisColumn.isAllowNull()); + root.setIsOptional(fieldOptionality.getOrDefault( + dorisColumn.getUniqueId(), dorisColumn.isAllowNull())); root.setType(dorisColumn.getType().toColumnTypeThrift()); if (base64InitialDefaults.containsKey(dorisColumn.getUniqueId())) { root.setInitialDefaultValue(base64InitialDefaults.get(dorisColumn.getUniqueId())); - root.setInitialDefaultValueIsBase64(true); } else if (dorisColumn.getDefaultValue() != null) { root.setInitialDefaultValue(dorisColumn.getDefaultValue()); } + if (base64InitialDefaults.containsKey(dorisColumn.getUniqueId()) + || binaryLikeFieldIds.contains(dorisColumn.getUniqueId())) { + // The marker describes the Iceberg source type, not only ownership of a direct default. + root.setInitialDefaultValueIsBase64(true); + } if (hasNameMapping) { // The explicit capability keeps old-FE plans on legacy fallback while making an empty @@ -214,7 +240,7 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, Column subColumn = subNameToSubColumn.get(subField.getName()); fieldPtr.setFieldPtr(getExternalSchema( subField.getType(), subColumn, nameMapping, hasNameMapping, - base64InitialDefaults)); + base64InitialDefaults, binaryLikeFieldIds, fieldOptionality)); structField.addToFields(fieldPtr); } @@ -227,7 +253,7 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, - hasNameMapping, base64InitialDefaults)); + hasNameMapping, base64InitialDefaults, binaryLikeFieldIds, fieldOptionality)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -238,13 +264,13 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, - hasNameMapping, base64InitialDefaults)); + hasNameMapping, base64InitialDefaults, binaryLikeFieldIds, fieldOptionality)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, - hasNameMapping, base64InitialDefaults)); + hasNameMapping, base64InitialDefaults, binaryLikeFieldIds, fieldOptionality)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 0b375c70d6791e..d078c649e1e9f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -94,6 +94,7 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionsTable; import org.apache.iceberg.Schema; +import org.apache.iceberg.SingleValueParser; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.StructLike; @@ -729,6 +730,13 @@ public static boolean containsVariant(Type type) { } public static void validateWriteSchema(Table table, List columns) { + try { + validateNestedPartitionWriteBackendCompatibility(table.spec(), table.schema(), + Env.getCurrentSystemInfo().getBackendsByCurrentCluster().values()); + } catch (AnalysisException e) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Failed to check backend compatibility for nested Iceberg partition writes", e); + } if (columns.stream().noneMatch(column -> containsVariant(column.getType()))) { return; } @@ -784,6 +792,26 @@ static void validateVariantWriteBackendCompatibility(List columns, Itera } } + @VisibleForTesting + static void validateNestedPartitionWriteBackendCompatibility( + PartitionSpec spec, Schema schema, Iterable backends) throws AnalysisException { + Set topLevelIds = schema.columns().stream() + .map(Types.NestedField::fieldId).collect(Collectors.toSet()); + boolean hasNestedSource = spec.fields().stream() + .anyMatch(field -> !topLevelIds.contains(field.sourceId())); + if (!hasNestedSource) { + return; + } + for (Backend backend : backends) { + if (backend.isQueryAvailable() && backend.isSmoothUpgradeSrc()) { + // Old writers index partition sources only by top-level field ID and can route a + // nested source through column zero, so mixed-version scheduling is unsafe. + throw new AnalysisException("Nested Iceberg partition writes are unavailable while backend " + + backend.getId() + " is a smooth upgrade source"); + } + } + } + /** * Get partition info map for identity partitions only, considering partition * evolution. @@ -1338,8 +1366,12 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi return resSchema; } - private static String serializeInitialDefault(org.apache.iceberg.types.Type type, Object value, + static String serializeInitialDefault(org.apache.iceberg.types.Type type, Object value, boolean enableMappingTimestampTz) { + if (type.isNestedType()) { + // BE consumes Iceberg's field-id keyed single-value representation for complex defaults. + return SingleValueParser.toJson(type, value); + } String humanValue = Transforms.identity(type).toHumanString(type, value); if (type.typeId() == TypeID.TIMESTAMP) { // Iceberg formats timestamps as ISO-8601 (for example 2024-01-01T00:00:00), while @@ -1378,6 +1410,23 @@ public static Map getBase64EncodedInitialDefaults(Schema schema return result; } + /** + * Return every binary-like source field, including leaves that only occur inside a parent + * complex default. BE needs this independent marker when VARBINARY mapping is disabled. + */ + public static Set getBinaryLikeFieldIds(Schema schema) { + return TypeUtil.indexById(schema.asStruct()).values().stream() + .filter(field -> isBinaryLike(field.type())) + .map(Types.NestedField::fieldId) + .collect(Collectors.toSet()); + } + + /** Keep Iceberg requiredness request-scoped instead of changing cached Doris Column semantics. */ + public static Map getFieldOptionality(Schema schema) { + return TypeUtil.indexById(schema.asStruct()).values().stream() + .collect(Collectors.toMap(Types.NestedField::fieldId, Types.NestedField::isOptional)); + } + private static boolean isBinaryLike(org.apache.iceberg.types.Type type) { return type.typeId() == TypeID.UUID || type.typeId() == TypeID.BINARY || type.typeId() == TypeID.FIXED; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 56457819a2a3de..11e36fe8183835 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -814,7 +814,8 @@ void initializeIcebergSchemaInfo(Optional>> nameMappin : source.getTargetTable().getColumns(); ExternalUtil.initSchemaInfoForAllColumn(params, -1L, columns, nameMapping.orElse(Collections.emptyMap()), nameMapping.isPresent(), - getBase64EncodedInitialDefaultsForScan()); + getBase64EncodedInitialDefaultsForScan(), + getBinaryLikeFieldIdsForScan(), getFieldOptionalityForScan()); } @VisibleForTesting @@ -826,11 +827,25 @@ void enableCurrentIcebergScanSemantics() { @VisibleForTesting Map getBase64EncodedInitialDefaultsForScan() throws UserException { + return IcebergUtils.getBase64EncodedInitialDefaults(getSchemaForInitialDefaultTransport()); + } + + @VisibleForTesting + Set getBinaryLikeFieldIdsForScan() throws UserException { + return IcebergUtils.getBinaryLikeFieldIds(getSchemaForInitialDefaultTransport()); + } + + @VisibleForTesting + Map getFieldOptionalityForScan() throws UserException { + return IcebergUtils.getFieldOptionality(getSchemaForInitialDefaultTransport()); + } + + private Schema getSchemaForInitialDefaultTransport() throws UserException { if (isSystemTable) { // System-table columns are derived from the metadata table schema. Some metadata // tables, such as position_deletes, do not support Table.newScan(). Use the same // schema that produced source.getTargetTable().getColumns() to keep defaults aligned. - return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); + return icebergTable.schema(); } IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); Schema scanSchema = null; @@ -846,8 +861,7 @@ Map getBase64EncodedInitialDefaultsForScan() throws UserExcepti } // A branch can expose a schema newer than its data snapshot. The statement-pinned schema // produced the target columns, so default markers must not be recomputed from that snapshot. - return IcebergUtils.getBase64EncodedInitialDefaults( - Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null")); + return Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java index 243d64c26c5787..e7264bc8151475 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java @@ -31,9 +31,11 @@ import org.apache.doris.datasource.hive.HivePartition; import org.apache.doris.datasource.hive.HiveProperties; import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.trees.plans.commands.insert.HiveInsertCommandContext; import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; import org.apache.doris.thrift.TExplainLevel; @@ -47,6 +49,7 @@ import org.apache.doris.thrift.THiveSerDeProperties; import org.apache.doris.thrift.THiveTableSink; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -55,6 +58,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -93,9 +97,6 @@ public String getExplainString(String prefix, TExplainLevel explainLevel) { public void bindDataSink(Optional insertCtx) throws AnalysisException { THiveTableSink tSink = new THiveTableSink(); - // The legacy planner uses the same deferred Azure protocol as the connector planner, so the BE - // must not fall back to publishing blocks before FE has durably accepted the commit records. - tSink.setSupportsDeferredAzureMultipart(true); tSink.setDbName(targetTable.getDbName()); tSink.setTableName(targetTable.getName()); Set partNames = new HashSet<>(targetTable.getPartitionColumnNames()); @@ -136,6 +137,14 @@ public void bindDataSink(Optional insertCtx) LocationPath locationPath = LocationPath.of(sd.getLocation(), targetTable.getStoragePropertiesMap()); String location = sd.getLocation(); TFileType fileType = locationPath.getTFileTypeForBE(); + Map backendStorageProperties = targetTable.getBackendStorageProperties(); + boolean isAzureObjectStorage = fileType == TFileType.FILE_S3 + && "azure".equalsIgnoreCase( + backendStorageProperties.get(StorageProperties.FS_PROVIDER_KEY)); + validateDeferredAzureMultipartBackendCompatibility(isAzureObjectStorage, + Env.getCurrentSystemInfo().getBackendsByCurrentCluster().values()); + // The flag is safe only after every eligible BE is known to understand deferred block IDs. + tSink.setSupportsDeferredAzureMultipart(true); if (fileType == TFileType.FILE_S3) { locationParams.setWritePath(locationPath.getNormalizedLocation()); locationParams.setOriginalWritePath(originalLocation); @@ -164,12 +173,26 @@ public void bindDataSink(Optional insertCtx) tSink.setBrokerAddresses(getBrokerAddresses(targetTable.getCatalog().bindBrokerName())); } - tSink.setHadoopConfig(targetTable.getBackendStorageProperties()); + tSink.setHadoopConfig(backendStorageProperties); tDataSink = new TDataSink(getDataSinkType()); tDataSink.setHiveTableSink(tSink); } + @VisibleForTesting + static void validateDeferredAzureMultipartBackendCompatibility( + boolean isAzureObjectStorage, Iterable backends) throws AnalysisException { + if (!isAzureObjectStorage) { + return; + } + for (Backend backend : backends) { + if (backend.isQueryAvailable() && backend.isSmoothUpgradeSrc()) { + throw new AnalysisException("Azure Hive writes are unavailable while backend " + + backend.getId() + " is a smooth upgrade source"); + } + } + } + private String createTempPath(String location) { String user = ConnectContext.get().getCurrentUserIdentity().getUser(); String stagingBaseDir = targetTable.getCatalog().getCatalogProperty() diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java index afe2a541abe0d6..8a64f8c85c1528 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java @@ -90,9 +90,11 @@ public final boolean updateFragmentExecStatus(TReportExecStatusParams params) { } SingleFragmentPipelineTask fragmentTask = backendFragmentTasks.get().get( new BackendFragmentId(params.getBackendId(), params.getFragmentId())); + boolean transfersExternalFileOwnership = params.isDone() + && (params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() + || params.isSetMcCommitDatas() || params.isSetPaimonCommitMessages()); if (fragmentTask == null) { - if (params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() - || params.isSetMcCommitDatas() || params.isSetPaimonCommitMessages()) { + if (transfersExternalFileOwnership) { throw new IllegalStateException("Missing fragment handler for external-file report"); } return false; @@ -121,9 +123,8 @@ public final boolean updateFragmentExecStatus(TReportExecStatusParams params) { } } doProcessReportExecStatus(params, fragmentTask); - return (!params.isSetHivePartitionUpdates() && !params.isSetIcebergCommitDatas() - && !params.isSetMcCommitDatas() && !params.isSetPaimonCommitMessages()) - || fragmentTask.isDone(); + // A legacy periodic vector is informational; only EOS requires durable acceptance. + return !transfersExternalFileOwnership || fragmentTask.isDone(); } private Map buildBackendFragmentTasks( diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 1b752281e15996..5295d162884a62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -2591,16 +2591,15 @@ public boolean updateFragmentExecStatus(TReportExecStatusParams params) { boolean hasExternalCommitData = params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() || params.isSetMcCommitDatas() || params.isSetPaimonCommitMessages(); + boolean transfersExternalFileOwnership = hasExternalCommitData && params.isDone(); if (ctx == null) { - if (hasExternalCommitData) { + if (transfersExternalFileOwnership) { throw new IllegalStateException("Missing fragment handler for external-file report"); } return false; } if (!ctx.updatePipelineStatus(params)) { - if (hasExternalCommitData && !ctx.done) { - throw new IllegalStateException("External-file report was not a completed fragment report"); - } + // Old BEs retain periodic commit vectors and replay them at EOS, where ownership moves. LOG.debug("Fragment {} is not done, ignore report status: {}", params.getFragmentId(), params.toString()); return ctx.done; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java index c893e02ee3d909..4667c7588ec41d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java @@ -285,11 +285,14 @@ public TReportExecStatusResult reportExecStatus(TReportExecStatusParams params, } boolean hasExternalCommitData = hasExternalCommitData(params); - String reportKey = hasExternalCommitData ? externalFileReportKey(params) : null; - if (hasExternalCommitData && reportKey == null) { + // Legacy BEs may attach the same vectors to periodic reports and resend them at EOS. Only + // the final report transfers ownership, so rollout must not reject or cache the preview. + boolean transfersExternalFileOwnership = hasExternalCommitData && params.isDone(); + String reportKey = transfersExternalFileOwnership ? externalFileReportKey(params) : null; + if (transfersExternalFileOwnership && reportKey == null) { return rejectedExternalFileReport(result, "External-file report is missing its identity fields"); } - if (hasExternalCommitData && acceptedExternalFileReports.getIfPresent(reportKey) != null) { + if (transfersExternalFileOwnership && acceptedExternalFileReports.getIfPresent(reportKey) != null) { // Keep acceptance available after coordinator removal so a lost response is retry-safe. result.setStatus(new TStatus(TStatusCode.OK)); result.setExternalFileCommitDataAccepted(true); @@ -302,24 +305,24 @@ public TReportExecStatusResult reportExecStatus(TReportExecStatusParams params, // Currently, the execution of query is splited from the exec status process. // So, it is very likely that when exec status arrived on FE asynchronously, coordinator // has been removed from coordinatorMap. - return hasExternalCommitData + return transfersExternalFileOwnership ? rejectedExternalFileReport(result, "Coordinator no longer owns this external-file report") : result; } try { boolean accepted = info.getCoord().updateFragmentExecStatus(params); - if (hasExternalCommitData && !accepted) { + if (transfersExternalFileOwnership && !accepted) { return rejectedExternalFileReport(result, "FE has not accepted the external-file report"); } } catch (Exception e) { LOG.warn("Exception during handle report, response: {}, query: {}, instance: {}", result.toString(), DebugUtil.printId(params.query_id), DebugUtil.printId(params.fragment_instance_id), e); - return hasExternalCommitData + return transfersExternalFileOwnership ? rejectedExternalFileReport(result, "FE did not accept the external-file report") : result; } result.setStatus(new TStatus(TStatusCode.OK)); - if (hasExternalCommitData) { + if (transfersExternalFileOwnership) { // Publish the retry token before replying; a transport loss cannot revoke FE ownership. acceptedExternalFileReports.put(reportKey, Boolean.TRUE); result.setExternalFileCommitDataAccepted(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java index 6fa059ed18c783..a9ce5f7621fa28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java @@ -183,11 +183,7 @@ protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleF } if (!fragmentTask.processReportExecStatus(params, () -> acceptFinalReport(params))) { - if ((params.isSetHivePartitionUpdates() || params.isSetIcebergCommitDatas() - || params.isSetMcCommitDatas() || params.isSetPaimonCommitMessages()) - && !fragmentTask.isDone()) { - throw new IllegalStateException("External-file report was not a completed fragment report"); - } + // Old BEs retain periodic commit vectors and replay them at EOS, where ownership moves. LOG.debug("Fragment {} is not done, ignore report status: {}", params.getFragmentId(), params.toString()); return; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 9422f08812de2d..6ab4432b68cbac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -234,7 +234,8 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Map base64InitialDefaults = new HashMap<>(); base64InitialDefaults.put(col2.getUniqueId(), "AAEC/w=="); ExternalUtil.initSchemaInfoForAllColumn( - params, schemaId, columns, nameMapping, base64InitialDefaults); + params, schemaId, columns, nameMapping, true, base64InitialDefaults, + Collections.emptySet(), Collections.singletonMap(col1.getUniqueId(), false)); Assert.assertEquals(schemaId.longValue(), params.getCurrentSchemaId()); List history = params.getHistorySchemaInfo(); @@ -252,7 +253,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.getName(), field1.getName()); Assert.assertEquals(col1.getUniqueId(), field1.getId()); - Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); + Assert.assertFalse(field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); Assert.assertTrue(field1.isNameMappingIsAuthoritative()); @@ -269,6 +270,22 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } + @Test + public void testBinarySourceMarkerDoesNotRequireDirectDefault() { + TFileScanRangeParams params = new TFileScanRangeParams(); + Column column = new Column("binary_leaf", Type.VARCHAR, false); + column.setUniqueId(103); + + ExternalUtil.initSchemaInfoForAllColumn(params, 501L, Collections.singletonList(column), + Collections.emptyMap(), false, Collections.emptyMap(), + Collections.singleton(column.getUniqueId()), Collections.emptyMap()); + + TField field = params.getHistorySchemaInfo().get(0).getRootField() + .getFields().get(0).getFieldPtr(); + Assert.assertFalse(field.isSetInitialDefaultValue()); + Assert.assertTrue(field.isInitialDefaultValueIsBase64()); + } + @Test public void testInitSchemaInfoForAllColumnSerializesNestedNonBinaryDefault() { StructType structType = new StructType( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 45619851cdc78a..8f6c3c974c3e86 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -48,6 +48,7 @@ import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.UnboundPredicate; @@ -396,7 +397,8 @@ public void testRejectVariantWritesWhenParquetShreddingIsEnabled() { IcebergUtils.validateVariantWriteProperties(variantColumns, ImmutableMap.of(shredVariantsProperty, "false")); - AnalysisException exception = Assert.assertThrows(AnalysisException.class, + AnalysisException exception = Assert.assertThrows( + AnalysisException.class, () -> IcebergUtils.validateVariantWriteProperties(variantColumns, ImmutableMap.of(shredVariantsProperty, "true"))); Assert.assertTrue(exception.getMessage().contains("only unshredded Iceberg VARIANT writes")); @@ -465,6 +467,24 @@ public void testRejectSmoothUpgradeSourceBackendForVariantWrite() { variantColumns, ImmutableList.of(currentBackend, smoothUpgradeSource)); } + @Test + public void testRejectNestedPartitionWriteWhenOldBackendIsEligible() { + Schema schema = new Schema(Types.NestedField.optional(1, "payload", Types.StructType.of( + Types.NestedField.optional(2, "part", Types.IntegerType.get())))); + PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("payload.part", 8).build(); + Backend oldBackend = Mockito.mock(Backend.class); + Mockito.when(oldBackend.isQueryAvailable()).thenReturn(true); + Mockito.when(oldBackend.isSmoothUpgradeSrc()).thenReturn(true); + Mockito.when(oldBackend.getId()).thenReturn(10005L); + + org.apache.doris.common.AnalysisException exception = Assert.assertThrows( + org.apache.doris.common.AnalysisException.class, + () -> IcebergUtils.validateNestedPartitionWriteBackendCompatibility( + spec, schema, Collections.singletonList(oldBackend))); + + Assert.assertTrue(exception.getMessage().contains("backend 10005")); + } + @Test public void testIcebergVariantEnablesParquetMetricsCollection() { Table table = Mockito.mock(Table.class); @@ -532,6 +552,41 @@ public void testParseSchemaPreservesNestedNonBinaryInitialDefault() { Assert.assertEquals("7", columns.get(0).getChildren().get(0).getDefaultValue()); } + @Test + public void testIcebergRequirednessIsCollectedRecursively() { + Schema schema = new Schema(Types.NestedField.required(10, "s", Types.StructType.of( + Types.NestedField.required(11, "required_child", Types.IntegerType.get()), + Types.NestedField.optional(12, "optional_child", Types.IntegerType.get())))); + + Map optionality = IcebergUtils.getFieldOptionality(schema); + + Assert.assertFalse(optionality.get(10)); + Assert.assertFalse(optionality.get(11)); + Assert.assertTrue(optionality.get(12)); + } + + @Test + public void testComplexInitialDefaultUsesIcebergSingleValueJson() { + Types.StructType structType = Types.StructType.of( + Types.NestedField.optional(11, "bytes", Types.BinaryType.get()), + Types.NestedField.optional(12, "uuid", Types.UUIDType.get())); + GenericRecord value = GenericRecord.create(structType); + value.set(0, ByteBuffer.wrap(new byte[] {0, 1, (byte) 0xFF})); + value.set(1, UUID.fromString("00112233-4455-6677-8899-aabbccddeeff")); + + Assert.assertEquals("{\"11\":\"0001FF\",\"12\":\"00112233-4455-6677-8899-aabbccddeeff\"}", + IcebergUtils.serializeInitialDefault(structType, value, false)); + } + + @Test + public void testBinaryLikeFieldIdsIncludeNestedLeavesWithoutDefaults() { + Schema schema = new Schema(Types.NestedField.optional(10, "s", Types.StructType.of( + Types.NestedField.optional(11, "bytes", Types.BinaryType.get()), + Types.NestedField.optional(12, "text", Types.StringType.get())))); + + Assert.assertEquals(Collections.singleton(11), IcebergUtils.getBinaryLikeFieldIds(schema)); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 94d6ea42717ff5..cb728f13242b10 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -235,6 +235,8 @@ public void testPartitionEvolutionKeepsNonFileSlotInReaderSchema() throws Except node.addSlot(1, projectedColumn); setIcebergSource(node, source); Mockito.doReturn(Collections.emptyMap()).when(node).getBase64EncodedInitialDefaultsForScan(); + Mockito.doReturn(Collections.emptySet()).when(node).getBinaryLikeFieldIdsForScan(); + Mockito.doReturn(Collections.emptyMap()).when(node).getFieldOptionalityForScan(); TFileScanRangeParams scanParams = node.initializeAndGetIcebergSchemaInfo(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java index 8ade2d52d2b04e..b371372e546698 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java @@ -29,6 +29,7 @@ import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.foundation.util.PathUtils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; import mockit.Mock; import mockit.MockUp; @@ -38,8 +39,10 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -52,6 +55,21 @@ public class HiveTableSinkTest { + @Test + public void testRejectDeferredAzureWriteWhenOldBackendIsEligible() { + Backend oldBackend = Mockito.mock(Backend.class); + Mockito.when(oldBackend.isQueryAvailable()).thenReturn(true); + Mockito.when(oldBackend.isSmoothUpgradeSrc()).thenReturn(true); + Mockito.when(oldBackend.getId()).thenReturn(10006L); + + org.apache.doris.common.AnalysisException exception = Assert.assertThrows( + org.apache.doris.common.AnalysisException.class, + () -> HiveTableSink.validateDeferredAzureMultipartBackendCompatibility( + true, Collections.singletonList(oldBackend))); + + Assert.assertTrue(exception.getMessage().contains("backend 10006")); + } + @Test public void testBindDataSink() throws UserException { ConnectContext ctx = new ConnectContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java index 1b9d461e00ea4c..1372fc137f4d8e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/QeProcessorImplReportAckTest.java @@ -58,6 +58,29 @@ void rejectsExternalReportWithoutCoordinator() { Assertions.assertFalse(result.isExternalFileCommitDataAccepted()); } + @Test + void acceptsLegacyPeriodicExternalReportWithoutTakingOwnership() { + TReportExecStatusParams params = params(new TUniqueId(12345, 7)).setDone(false); + + TReportExecStatusResult result = report(params); + + Assertions.assertEquals(TStatusCode.OK, result.getStatus().getStatusCode()); + Assertions.assertFalse(result.isSetExternalFileCommitDataAccepted()); + } + + @Test + void forwardsLegacyPeriodicExternalReportWithoutRequiringAcceptance() throws Exception { + TUniqueId queryId = new TUniqueId(12345, 8); + Coordinator coordinator = register(queryId); + TReportExecStatusParams params = params(queryId).setDone(false); + + TReportExecStatusResult result = report(params); + + Assertions.assertEquals(TStatusCode.OK, result.getStatus().getStatusCode()); + Assertions.assertFalse(result.isSetExternalFileCommitDataAccepted()); + Mockito.verify(coordinator).updateFragmentExecStatus(params); + } + @Test void rejectsExternalReportWhenHandlerThrows() throws Exception { TUniqueId queryId = new TUniqueId(12345, 2); diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 86915e46d28bfb..2e8d88c17dd3d8 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -55,9 +55,9 @@ struct TField { // values use Doris' FE string representation. An old data file that predates this field // logically contains this value rather than NULL. 7: optional string initial_default_value, - // True when initial_default_value is Base64 and must be decoded before constructing the Doris - // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg - // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. + // True for an Iceberg UUID/BINARY/FIXED source field. A direct initial_default_value is Base64, + // and the marker also lets BE decode this leaf inside a parent complex JSON default. This + // cannot be inferred from the Doris type because the source may map to STRING/CHAR. 8: optional bool initial_default_value_is_base64, // Version marker for authoritative Iceberg mapping semantics. Its absence preserves the // legacy name fallback when a new BE executes a plan produced by an older FE during rollout. From cc71881412c4ed0b486570809cc787bf9b767fd6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 17 Aug 2026 18:02:18 +0800 Subject: [PATCH 3/4] [fix](iceberg) Address remaining branch-4.1 review feedback --- .../operator/iceberg_sorter_reserve_memory.h | 12 ++ .../spill_iceberg_table_sink_operator.cpp | 18 ++- .../writer/iceberg/viceberg_sort_writer.cpp | 12 +- .../writer/paimon/paimon_table_writer.cpp | 82 +++++++++----- .../sink/writer/paimon/paimon_table_writer.h | 20 +++- .../sink/writer/paimon/paimon_write_backend.h | 6 +- be/src/runtime/runtime_state.cpp | 27 +++-- be/src/runtime/runtime_state.h | 9 +- ...spill_iceberg_table_sink_operator_test.cpp | 6 + .../paimon/paimon_table_writer_test.cpp | 103 ++++++++++++++++++ .../runtime_state_block_budget_test.cpp | 34 ++++++ .../apache/doris/datasource/FileScanNode.java | 30 ++++- .../iceberg/IcebergCommitCoordinator.java | 90 +++++++++++++++ .../iceberg/IcebergTransaction.java | 52 +++++++-- .../IcebergRemoveOrphanFilesAction.java | 68 +++++++----- .../iceberg/source/IcebergScanNode.java | 12 +- .../datasource/FileQueryScanNodeTest.java | 32 ++++++ .../iceberg/IcebergTransactionTest.java | 29 +++++ .../iceberg/source/IcebergScanNodeTest.java | 19 ++++ .../test_iceberg_write_complex_evolution.out | 8 +- 20 files changed, 576 insertions(+), 93 deletions(-) create mode 100644 be/test/exec/sink/writer/paimon/paimon_table_writer_test.cpp create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCommitCoordinator.java diff --git a/be/src/exec/operator/iceberg_sorter_reserve_memory.h b/be/src/exec/operator/iceberg_sorter_reserve_memory.h index 8b09a0af4dfbfc..1c974b707ab84c 100644 --- a/be/src/exec/operator/iceberg_sorter_reserve_memory.h +++ b/be/src/exec/operator/iceberg_sorter_reserve_memory.h @@ -35,6 +35,12 @@ inline size_t iceberg_saturating_add(size_t lhs, size_t rhs) { return std::min(std::numeric_limits::max() - lhs, rhs) + lhs; } +inline size_t iceberg_saturating_multiply(size_t lhs, size_t rhs) { + return lhs != 0 && rhs > std::numeric_limits::max() / lhs + ? std::numeric_limits::max() + : lhs * rhs; +} + inline size_t bounded_iceberg_reserve_size( const std::vector& per_partition_reservations, size_t incoming_rows = std::numeric_limits::max(), @@ -129,4 +135,10 @@ inline size_t iceberg_spill_merge_workspace(size_t spill_file_count, size_t spil : input_bytes + spill_buffer_bytes; } +inline size_t iceberg_final_merge_batch_rows(size_t spill_buffer_rows, size_t runtime_batch_rows) { + // The output block is covered by one spill-buffer reservation, so its row count must use the + // observed spill bound instead of the unrelated query-wide batch size. + return std::max(1, std::min(spill_buffer_rows, runtime_batch_rows)); +} + } // namespace doris diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 8ad3b86f132c7a..59c6ce530b0f16 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -17,6 +17,7 @@ #include "exec/operator/spill_iceberg_table_sink_operator.h" +#include "common/config.h" #include "common/status.h" #include "core/block/block.h" #include "exec/operator/iceberg_table_sink_operator.h" @@ -36,7 +37,22 @@ size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_worksp ? std::numeric_limits::max() : block_bytes * 4; size_t reserve = iceberg_saturating_add(writer_workspace_bytes, dispatch_copies); - // Transform, selected blocks, and retained sorters coexist during high-cardinality dispatch. + if (block.rows() > 0) { + size_t minimum_selected_block_bytes = 0; + for (const auto& column : block.get_columns_with_type_and_name()) { + minimum_selected_block_bytes = + iceberg_saturating_add(minimum_selected_block_bytes, + column.column->clone_resized(1)->allocated_bytes()); + } + const size_t max_partition_count = static_cast( + std::max(1, config::table_sink_partition_write_max_partition_nums_per_writer)); + const size_t touched_partitions = std::min(block.rows(), max_partition_count); + const size_t retained_sorter_capacity = iceberg_saturating_multiply( + iceberg_saturating_multiply(minimum_selected_block_bytes, touched_partitions), 2); + // Sorter append growth can retain twice the minimum selected-column capacity for every new + // partition even though only the current selection itself is temporary during dispatch. + reserve = iceberg_saturating_add(reserve, retained_sorter_capacity); + } return iceberg_saturating_add(reserve, row_index_bytes); } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp index 32f195366dc300..572445f38b139b 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp @@ -181,8 +181,9 @@ void VIcebergSortWriter::_update_spill_block_batch_row_count(const Block& block) if (rows > 0 && 0 == _avg_row_bytes) { _avg_row_bytes = std::max(1UL, block.bytes() / rows); int64_t spill_batch_bytes = _runtime_state->spill_buffer_size_bytes(); // default 8MB - // Calculate how many rows fit in one spill batch (ceiling division) - _spill_block_batch_row_count = (spill_batch_bytes + _avg_row_bytes - 1) / _avg_row_bytes; + // Keep the merge output inside the spill-buffer reservation; a single oversized row is the + // only unavoidable exception and is still admitted as one row. + _spill_block_batch_row_count = std::max(1, spill_batch_bytes / _avg_row_bytes); } } @@ -384,8 +385,11 @@ Status VIcebergSortWriter::_create_merger(bool is_final_merge, size_t batch_size } Status VIcebergSortWriter::_create_final_merger() { - // Final merger uses the runtime batch size and merges all remaining streams - return _create_merger(true, _runtime_state->batch_size(), 1); + return _create_merger( + true, + iceberg_final_merge_batch_rows(_spill_block_batch_row_count, + static_cast(_runtime_state->batch_size())), + 1); } void VIcebergSortWriter::_cleanup_spill_streams() { diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp index 95ddca671097bd..38069545da7db9 100644 --- a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp @@ -24,6 +24,39 @@ namespace doris { +PaimonPreparedCommitOwner::PaimonPreparedCommitOwner(std::unique_ptr writer, + std::unique_ptr backend) + : _writer(std::move(writer)), _backend(std::move(backend)) {} + +PaimonPreparedCommitOwner::~PaimonPreparedCommitOwner() { + _close(); +} + +void PaimonPreparedCommitOwner::finalize(ExternalFileReportOutcome outcome) { + if (_finalized || outcome == ExternalFileReportOutcome::AMBIGUOUS) { + return; + } + _finalized = true; + if (outcome == ExternalFileReportOutcome::REJECTED && _writer) { + Status abort_status = _writer->abort(); + if (!abort_status.ok()) { + LOG(WARNING) << "Paimon prepared writer abort failed: " << abort_status.to_string(); + } + } + _close(); +} + +void PaimonPreparedCommitOwner::_close() { + _writer.reset(); + if (_backend) { + Status close_status = _backend->close(); + if (!close_status.ok()) { + LOG(WARNING) << "Paimon prepared backend close failed: " << close_status.to_string(); + } + _backend.reset(); + } +} + PaimonTableWriter::PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs, std::shared_ptr dep, std::shared_ptr fin_dep) @@ -99,8 +132,8 @@ Status PaimonTableWriter::write(RuntimeState* state, Block& block) { Status PaimonTableWriter::close(Status status) { SCOPED_TIMER(_close_timer); - // Prepare messages first, but do not publish them until the backend confirms - // that every SDK user has stopped and its native backing memory is safe to release. + // Prepare messages while retaining the backend: report rejection still needs the live Java + // writer to abort, and the final report outcome becomes the backend close boundary. std::vector messages; if (status.ok()) { DCHECK(_writer); @@ -125,38 +158,35 @@ Status PaimonTableWriter::close(Status status) { } } - // The adapter only owns Arrow conversion resources. Release it before closing - // the backend, whose Java close is the authoritative SDK shutdown boundary. - _writer.reset(); - - if (_backend) { - Status close_st = _backend->close(); - if (!close_st.ok()) { - if (status.ok()) { - status = close_st; - } else { - LOG(WARNING) << "Paimon backend close also failed: " << close_st.to_string(); + if (!status.ok() || messages.empty()) { + _writer.reset(); + if (_backend) { + Status close_st = _backend->close(); + if (!close_st.ok()) { + if (status.ok()) { + status = close_st; + } else { + LOG(WARNING) << "Paimon backend close also failed: " << close_st.to_string(); + } } + _backend.reset(); } - } - - // Only a fully prepared and cleanly stopped writer may contribute payloads - // to the FE transaction. A Java close failure therefore aborts the Doris - // transaction instead of allowing it to commit potentially unsafe output. - if (status.ok()) { + } else { COUNTER_UPDATE(_commit_payload_count, static_cast(messages.size())); for (const auto& msg : messages) { DORIS_CHECK(msg.__isset.payload); COUNTER_UPDATE(_commit_payload_bytes_counter, static_cast(msg.payload.size())); } - if (!messages.empty()) { - _state->add_paimon_commit_messages(messages); - LOG(INFO) << "Paimon writer closed: " << messages.size() - << " commit messages, total rows=" << _written_rows; - } + auto owner = std::make_shared(std::move(_writer), + std::move(_backend)); + // Paimon's abort API needs the prepared Java writer. Retain that owner until the shared + // final report is accepted or rejected instead of publishing an irreversible payload. + _state->add_external_file_report_finalizer( + [owner](ExternalFileReportOutcome outcome) { owner->finalize(outcome); }); + _state->add_paimon_commit_messages(messages); + LOG(INFO) << "Paimon writer prepared: " << messages.size() + << " commit messages, total rows=" << _written_rows; } - - _backend.reset(); return status; } diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.h b/be/src/exec/sink/writer/paimon/paimon_table_writer.h index cbfc3209e80285..b003187c4785eb 100644 --- a/be/src/exec/sink/writer/paimon/paimon_table_writer.h +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.h @@ -32,6 +32,23 @@ namespace doris { class RuntimeState; +enum class ExternalFileReportOutcome; + +class PaimonPreparedCommitOwner { +public: + PaimonPreparedCommitOwner(std::unique_ptr writer, + std::unique_ptr backend); + ~PaimonPreparedCommitOwner(); + + void finalize(ExternalFileReportOutcome outcome); + +private: + void _close(); + + std::unique_ptr _writer; + std::unique_ptr _backend; + bool _finalized = false; +}; /// Each PaimonTableSinkLocalState owns one PaimonTableWriter, which in turn /// owns one IPaimonWriteBackend and one IPaimonWriter. Pipeline parallelism @@ -57,11 +74,12 @@ class RuntimeState; /// │ → selected SDK owns row normalization, routing, buffering, /// │ file writing, and compaction /// â–¼ -/// close() → prepareCommit() → CommitMessage[] +/// close() → prepareCommit() → retain abort owner → CommitMessage[] /// /// Commit flow (BE only prepares messages; FE is the commit coordinator): /// close() → writer->prepare_commit() /// → collect TPaimonCommitMessage[] (DPCM-framed serialized messages) +/// → retain writer/backend until final report ACK or rejection /// → RuntimeState::add_paimon_commit_messages() /// → RPC to FE Coordinator → PaimonTransaction class PaimonTableWriter final : public AsyncResultWriter { diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend.h b/be/src/exec/sink/writer/paimon/paimon_write_backend.h index d28b204a4c1aac..9993581b9fe894 100644 --- a/be/src/exec/sink/writer/paimon/paimon_write_backend.h +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend.h @@ -54,7 +54,7 @@ class IPaimonWriter { /// messages (DPCM-framed). Called once at EOS. virtual Status prepare_commit(std::vector& messages) = 0; - /// Discard written data files on error. Called when write or prepare_commit fails. + /// Discard written data files on error or when FE rejects the final prepared-file report. virtual Status abort() = 0; }; @@ -83,8 +83,8 @@ class IPaimonWriteBackend { /// Stop all SDK users and release backend resources. /// /// A successful return is the ownership boundary after which native memory - /// backing SDK buffers can be reclaimed safely. Callers must not publish - /// prepared commit messages until this succeeds. + /// backing SDK buffers can be reclaimed safely. A prepared writer may defer + /// this call while the final report outcome is pending so rejection can abort it. virtual Status close() = 0; virtual PaimonBackendType type() const = 0; diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 3b697b36aaa0c5..e5ea016763a37c 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -126,30 +126,37 @@ void RuntimeState::append_external_file_commit_data(TReportExecStatusParams* par } void RuntimeState::add_rejected_external_file_report_cleanup(std::function cleanup) { + add_external_file_report_finalizer( + [cleanup = std::move(cleanup)](ExternalFileReportOutcome outcome) { + if (outcome == ExternalFileReportOutcome::REJECTED) { + cleanup(); + } + }); +} + +void RuntimeState::add_external_file_report_finalizer( + std::function finalizer) { std::lock_guard lock(_external_file_report_state->mutex); - _external_file_report_state->rejected_report_cleanups.emplace_back(std::move(cleanup)); + _external_file_report_state->report_finalizers.emplace_back(std::move(finalizer)); } void RuntimeState::finalize_external_file_report_cleanup(ExternalFileReportOutcome outcome) { - std::vector> cleanups; + std::vector> finalizers; { std::lock_guard lock(_external_file_report_state->mutex); - if (outcome == ExternalFileReportOutcome::ACKNOWLEDGED) { - _external_file_report_state->rejected_report_cleanups.clear(); - return; - } if (outcome == ExternalFileReportOutcome::AMBIGUOUS) { // Once an ACK can have been lost, a later rejection cannot prove FE never accepted the files. _external_file_report_state->ownership_may_have_transferred = true; return; } - if (_external_file_report_state->ownership_may_have_transferred) { + if (outcome == ExternalFileReportOutcome::REJECTED && + _external_file_report_state->ownership_may_have_transferred) { return; } - cleanups.swap(_external_file_report_state->rejected_report_cleanups); + finalizers.swap(_external_file_report_state->report_finalizers); } - for (auto& cleanup : cleanups) { - cleanup(); + for (auto& finalizer : finalizers) { + finalizer(outcome); } } diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 3748e1c31bc503..a9a452bd7f28c4 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -78,6 +78,8 @@ class TaskExecutionContext; // Keep RuntimeState self-contained without importing the full frontend Thrift service header. class TReportExecStatusParams; +enum class ExternalFileReportOutcome { ACKNOWLEDGED, REJECTED, AMBIGUOUS }; + class ExternalFileReportState { friend class RuntimeState; @@ -85,11 +87,9 @@ class ExternalFileReportState { std::mutex mutex; size_t iceberg_serialized_bytes = 0; bool ownership_may_have_transferred = false; - std::vector> rejected_report_cleanups; + std::vector> report_finalizers; }; -enum class ExternalFileReportOutcome { ACKNOWLEDGED, REJECTED, AMBIGUOUS }; - // A collection of items that are part of the global state of a // query and shared across all execution nodes of that query. class RuntimeState { @@ -560,6 +560,9 @@ class RuntimeState { void add_rejected_external_file_report_cleanup(std::function cleanup); + void add_external_file_report_finalizer( + std::function finalizer); + void finalize_external_file_report_cleanup(ExternalFileReportOutcome outcome); void set_external_file_report_state(std::shared_ptr report_state) { diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index 3e004051220504..740b098373000f 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -95,6 +95,12 @@ TEST(SpillIcebergTableSinkOperatorTest, ReservesAllMergeInputsAndOutputAtEos) { EXPECT_EQ(32 * MB, iceberg_spill_merge_workspace(3, 8 * MB, 64 * MB)); } +TEST(SpillIcebergTableSinkOperatorTest, FinalMergeBatchFitsTheReservedSpillOutputBuffer) { + EXPECT_EQ(128, iceberg_final_merge_batch_rows(128, 4062)); + EXPECT_EQ(64, iceberg_final_merge_batch_rows(128, 64)); + EXPECT_EQ(1, iceberg_final_merge_batch_rows(0, 4062)); +} + TEST(SpillIcebergTableSinkOperatorTest, WaitsUntilDequeuedBlockUpdatesSorterState) { AsyncWriterQueueAdmission stateful_admission; stateful_admission.wait_for_processing_before_next_sink(); diff --git a/be/test/exec/sink/writer/paimon/paimon_table_writer_test.cpp b/be/test/exec/sink/writer/paimon/paimon_table_writer_test.cpp new file mode 100644 index 00000000000000..afdd2f765e9f1f --- /dev/null +++ b/be/test/exec/sink/writer/paimon/paimon_table_writer_test.cpp @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_table_writer.h" + +#include + +#include + +#include "runtime/runtime_state.h" + +namespace doris { +namespace { + +class FakePaimonWriter final : public IPaimonWriter { +public: + explicit FakePaimonWriter(int* abort_count) : _abort_count(abort_count) {} + + Status write(RuntimeState*, Block&) override { return Status::OK(); } + Status prepare_commit(std::vector&) override { return Status::OK(); } + Status abort() override { + ++*_abort_count; + return Status::OK(); + } + +private: + int* _abort_count; +}; + +class FakePaimonBackend final : public IPaimonWriteBackend { +public: + explicit FakePaimonBackend(int* close_count) : _close_count(close_count) {} + + Status open(const TPaimonTableSink&, RuntimeState*, RuntimeProfile*) override { + return Status::OK(); + } + Status create_writer(std::unique_ptr*) override { return Status::OK(); } + Status close() override { + ++*_close_count; + return Status::OK(); + } + PaimonBackendType type() const override { return PaimonBackendType::JNI; } + +private: + int* _close_count; +}; + +} // namespace + +TEST(PaimonPreparedCommitOwnerTest, RejectedReportAbortsAndClosesBackend) { + int abort_count = 0; + int close_count = 0; + { + PaimonPreparedCommitOwner owner(std::make_unique(&abort_count), + std::make_unique(&close_count)); + owner.finalize(ExternalFileReportOutcome::REJECTED); + } + EXPECT_EQ(1, abort_count); + EXPECT_EQ(1, close_count); +} + +TEST(PaimonPreparedCommitOwnerTest, AcknowledgedReportClosesWithoutAbort) { + int abort_count = 0; + int close_count = 0; + { + PaimonPreparedCommitOwner owner(std::make_unique(&abort_count), + std::make_unique(&close_count)); + owner.finalize(ExternalFileReportOutcome::ACKNOWLEDGED); + } + EXPECT_EQ(0, abort_count); + EXPECT_EQ(1, close_count); +} + +TEST(PaimonPreparedCommitOwnerTest, AmbiguousReportRetainsOwnerUntilAcknowledged) { + int abort_count = 0; + int close_count = 0; + { + PaimonPreparedCommitOwner owner(std::make_unique(&abort_count), + std::make_unique(&close_count)); + owner.finalize(ExternalFileReportOutcome::AMBIGUOUS); + EXPECT_EQ(0, abort_count); + EXPECT_EQ(0, close_count); + owner.finalize(ExternalFileReportOutcome::ACKNOWLEDGED); + } + EXPECT_EQ(0, abort_count); + EXPECT_EQ(1, close_count); +} + +} // namespace doris diff --git a/be/test/runtime/runtime_state_block_budget_test.cpp b/be/test/runtime/runtime_state_block_budget_test.cpp index 5a384378ec382b..ef46b369a05564 100644 --- a/be/test/runtime/runtime_state_block_budget_test.cpp +++ b/be/test/runtime/runtime_state_block_budget_test.cpp @@ -143,6 +143,40 @@ TEST(RuntimeStateIcebergCommitDataTest, AmbiguousOwnershipCannotBecomeRejected) EXPECT_EQ(0, cleanup_count); } +TEST(RuntimeStateIcebergCommitDataTest, FinalizersKeepOwnersUntilAReportOutcome) { + RuntimeState state; + int acknowledged_count = 0; + int rejected_count = 0; + state.add_external_file_report_finalizer([&](ExternalFileReportOutcome outcome) { + if (outcome == ExternalFileReportOutcome::ACKNOWLEDGED) { + ++acknowledged_count; + } else if (outcome == ExternalFileReportOutcome::REJECTED) { + ++rejected_count; + } + }); + + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::ACKNOWLEDGED); + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::REJECTED); + + EXPECT_EQ(1, acknowledged_count); + EXPECT_EQ(0, rejected_count); +} + +TEST(RuntimeStateIcebergCommitDataTest, AcknowledgementAfterAmbiguityReleasesOwner) { + RuntimeState state; + int acknowledged_count = 0; + state.add_external_file_report_finalizer([&](ExternalFileReportOutcome outcome) { + if (outcome == ExternalFileReportOutcome::ACKNOWLEDGED) { + ++acknowledged_count; + } + }); + + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::AMBIGUOUS); + state.finalize_external_file_report_cleanup(ExternalFileReportOutcome::ACKNOWLEDGED); + + EXPECT_EQ(1, acknowledged_count); +} + // --------------------------------------------------------------------------- // RuntimeState::batch_size() // --------------------------------------------------------------------------- diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index 0469c0ce1e389d..7e3458cfaaf809 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.UserException; import org.apache.doris.nereids.CascadesContext; @@ -30,6 +31,8 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.VarcharType; @@ -304,11 +307,7 @@ protected void setDefaultValueExprs(TableIf tbl, Expr expr; Expression expression; if (column.getDefaultValue() != null) { - expression = new NereidsParser().parseExpression( - column.getDefaultValueSql()); - ExpressionAnalyzer analyzer = new ExpressionAnalyzer( - null, new Scope(ImmutableList.of()), null, true, true); - expression = analyzer.analyze(expression); + expression = buildDefaultValueExpression(column); } else { if (column.isAllowNull()) { // For load, use Varchar as Null, for query, use column type. @@ -354,6 +353,27 @@ protected void setDefaultValueExprs(TableIf tbl, } } + private Expression buildDefaultValueExpression(Column column) { + if (column.getType().isFloatingPointType()) { + try { + double value = Double.parseDouble(column.getDefaultValue()); + if (!Double.isFinite(value)) { + // NaN and infinities are identifiers in SQL text, so construct a typed literal + // before expression name resolution while leaving other SQL defaults intact. + return column.getType().getPrimitiveType() == PrimitiveType.FLOAT + ? new FloatLiteral(Float.parseFloat(column.getDefaultValue())) + : new DoubleLiteral(value); + } + } catch (NumberFormatException ignored) { + // A floating-point column may still use a general SQL default expression. + } + } + Expression expression = new NereidsParser().parseExpression(column.getDefaultValueSql()); + ExpressionAnalyzer analyzer = new ExpressionAnalyzer( + null, new Scope(ImmutableList.of()), null, true, true); + return analyzer.analyze(expression); + } + protected void addFileCacheAdmissionLog(String userIdentity, Boolean admitted, String reason, double durationMs) { String admissionStatus = admitted ? "ADMITTED" : "DENIED"; String admissionLog = String.format("file cache request %s: user_identity:%s, reason:%s, cost:%.6f ms", diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCommitCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCommitCoordinator.java new file mode 100644 index 00000000000000..fcc581134289da --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCommitCoordinator.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import java.net.URI; +import java.util.Locale; +import java.util.concurrent.locks.StampedLock; + +/** Serializes destructive maintenance with Doris-coordinated commits for one Iceberg table. */ +public final class IcebergCommitCoordinator { + private static final int STRIPE_COUNT = 1024; + private static final StampedLock[] TABLE_LOCKS = new StampedLock[STRIPE_COUNT]; + + static { + for (int i = 0; i < TABLE_LOCKS.length; i++) { + TABLE_LOCKS[i] = new StampedLock(); + } + } + + private IcebergCommitCoordinator() { + } + + public static Guard beginCommit(String tableLocation) { + StampedLock lock = lockFor(tableLocation); + return new Guard(lock, lock.readLock(), false); + } + + public static Guard beginMaintenance(String tableLocation) { + StampedLock lock = lockFor(tableLocation); + return new Guard(lock, lock.writeLock(), true); + } + + static StampedLock lockFor(String tableLocation) { + URI uri = URI.create(tableLocation).normalize(); + String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT); + if (scheme.equals("s3a") || scheme.equals("s3n")) { + scheme = "s3"; + } + String authority = uri.getAuthority() == null + ? "" : uri.getAuthority().toLowerCase(Locale.ROOT); + String path = uri.getPath() == null ? "" : uri.getPath(); + while (path.length() > 1 && path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + String normalizedLocation = scheme + "://" + authority + path; + return TABLE_LOCKS[Math.floorMod(normalizedLocation.hashCode(), STRIPE_COUNT)]; + } + + public static final class Guard implements AutoCloseable { + private final StampedLock lock; + private final long stamp; + private final boolean write; + private boolean closed; + + private Guard(StampedLock lock, long stamp, boolean write) { + this.lock = lock; + this.stamp = stamp; + this.write = write; + } + + @Override + public synchronized void close() { + if (!closed) { + closed = true; + // Transaction begin and completion may run on different FE worker threads, so the + // fence must be released by its stamp instead of by thread ownership. + if (write) { + lock.unlockWrite(stamp); + } else { + lock.unlockRead(stamp); + } + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index e5b16817becc69..1385b152a92447 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -79,6 +79,7 @@ public class IcebergTransaction implements Transaction { private Table table; private org.apache.iceberg.Transaction transaction; + private IcebergCommitCoordinator.Guard commitGuard; private final List commitDataList = Lists.newArrayList(); private Optional conflictDetectionFilter = Optional.empty(); @@ -133,6 +134,7 @@ public void updateRewriteFiles(List filesToDelete) { public void beginInsert(ExternalTable dorisTable, Table targetTable, Optional ctx) throws UserException { ctx.ifPresent(c -> this.insertCtx = (IcebergInsertCommandContext) c); + acquireCommitFence(targetTable); try { ops.getExecutionAuthenticator().execute(() -> { // Planning, BE serialization, and commit must share one Iceberg metadata @@ -162,6 +164,7 @@ public void beginInsert(ExternalTable dorisTable, Table targetTable, this.rewrittenDeleteFilesByReferencedDataFile = Collections.emptyMap(); }); } catch (Exception e) { + releaseCommitFence(); throw new UserException("Failed to begin insert for iceberg table " + dorisTable.getName() + " because: " + e.getMessage(), e); } @@ -173,6 +176,7 @@ public void beginRewrite(ExternalTable dorisTable, Table targetTable) throws Use // For rewrite operations, we work directly on the main table this.branchName = null; this.isRewriteMode = true; + acquireCommitFence(targetTable); try { ops.getExecutionAuthenticator().execute(() -> { @@ -192,6 +196,7 @@ public void beginRewrite(ExternalTable dorisTable, Table targetTable) throws Use return null; }); } catch (Exception e) { + releaseCommitFence(); throw new UserException("Failed to begin rewrite for iceberg table " + dorisTable.getName() + " because: " + e.getMessage(), e); } @@ -277,6 +282,7 @@ private void updateManifestAfterRewrite() { * Begin delete operation for Iceberg table */ public void beginDelete(ExternalTable dorisTable, Table targetTable) throws UserException { + acquireCommitFence(targetTable); try { ops.getExecutionAuthenticator().execute(() -> { // RowDelta's validation base must match the generation used to select row IDs; @@ -296,6 +302,7 @@ public void beginDelete(ExternalTable dorisTable, Table targetTable) throws User LOG.info("Started delete transaction for table: {}", dorisTable.getName()); }); } catch (Exception e) { + releaseCommitFence(); throw new UserException("Failed to begin delete for iceberg table " + dorisTable.getName() + " because: " + e.getMessage(), e); } @@ -313,6 +320,7 @@ private Table createTransactionTable(ExternalTable dorisTable, Table retainedTab /** Begin UPDATE/MERGE against the metadata generation retained by the merge sink. */ public void beginMerge(ExternalTable dorisTable, Table targetTable) throws UserException { + acquireCommitFence(targetTable); try { ops.getExecutionAuthenticator().execute(() -> { this.branchName = null; @@ -333,6 +341,7 @@ public void beginMerge(ExternalTable dorisTable, Table targetTable) throws UserE return null; }); } catch (Exception e) { + releaseCommitFence(); throw new UserException("Failed to begin merge for iceberg table " + dorisTable.getName() + " because: " + e.getMessage(), e); } @@ -561,23 +570,44 @@ private void updateManifestAfterInsert(TUpdateMode updateMode) { @Override public void commit() throws UserException { - // commit the iceberg transaction - transaction.commitTransaction(); + try { + transaction.commitTransaction(); + } finally { + releaseCommitFence(); + } } @Override public void rollback() { - if (isRewriteMode) { - // Clear the collected files for rewrite mode - synchronized (filesToDelete) { - filesToDelete.clear(); - } - synchronized (filesToAdd) { - filesToAdd.clear(); + try { + if (isRewriteMode) { + // Clear the collected files for rewrite mode + synchronized (filesToDelete) { + filesToDelete.clear(); + } + synchronized (filesToAdd) { + filesToAdd.clear(); + } + LOG.info("Rewrite transaction rolled back"); } - LOG.info("Rewrite transaction rolled back"); + // For insert mode, do nothing as original implementation + } finally { + releaseCommitFence(); + } + } + + private void acquireCommitFence(Table targetTable) { + Preconditions.checkState(commitGuard == null, "Iceberg transaction fence is already held"); + // Hold a shared fence before files can be selected or written. Destructive maintenance + // takes the exclusive side, while unrelated and concurrent table writes remain parallel. + commitGuard = IcebergCommitCoordinator.beginCommit(targetTable.location()); + } + + private void releaseCommitFence() { + if (commitGuard != null) { + commitGuard.close(); + commitGuard = null; } - // For insert mode, do nothing as original implementation } public long getUpdateCnt() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesAction.java index 90939455ba041f..a08d8af1344479 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRemoveOrphanFilesAction.java @@ -23,6 +23,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ArgumentParsers; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.IcebergCommitCoordinator; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.info.PartitionNamesInfo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -100,6 +101,30 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf tableIf) throws UserException { Table table = ((IcebergExternalTable) tableIf).getIcebergTable(); + long olderThan = namedArguments.getLong(OLDER_THAN); + // Reject an unsafe cutoff before opening any metadata or manifest file. + if (olderThan > System.currentTimeMillis() - MIN_RETENTION_MS) { + throw new UserException("older_than must retain at least 24 hours of files"); + } + + try { + if (namedArguments.getBoolean(DRY_RUN)) { + return scanAndDeleteOrphans(table, olderThan, true); + } + // Refresh only after acquiring the commit fence, then keep the same metadata generation + // stable until deletion finishes so an old imported file cannot become reachable midway. + try (IcebergCommitCoordinator.Guard ignored = + IcebergCommitCoordinator.beginMaintenance(table.location())) { + table.refresh(); + return scanAndDeleteOrphans(table, olderThan, false); + } + } catch (Exception e) { + throw new UserException("Failed to remove orphan files: " + e.getMessage(), e); + } + } + + private List scanAndDeleteOrphans(Table table, long olderThan, boolean dryRun) + throws IOException, UserException { if (!(table.io() instanceof SupportsPrefixOperations)) { throw new UserException("remove_orphan_files requires FileIO prefix listing support"); } @@ -108,38 +133,27 @@ protected List executeAction(TableIf tableIf) throws UserException { // A GC-disabled table may share files with another table, so no destructive scan is safe. throw new UserException("Cannot remove orphan files: Iceberg GC is disabled"); } - long olderThan = namedArguments.getLong(OLDER_THAN); - // Reject an unsafe cutoff before opening any metadata or manifest file. - if (olderThan > System.currentTimeMillis() - MIN_RETENTION_MS) { - throw new UserException("older_than must retain at least 24 hours of files"); - } List scanScopes = resolveScanScopes(table); - - try { - ReachableIndex reachable = collectReachableFiles(table); - long orphanCount = 0; - long deletedCount = 0; - boolean dryRun = namedArguments.getBoolean(DRY_RUN); - for (ScanScope scope : scanScopes) { - // Object stores use raw prefix matching, so the separator excludes sibling prefixes. - String listingPrefix = scope.root.endsWith("/") ? scope.root : scope.root + "/"; - for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { - // Unknown creation time cannot prove the file predates every in-flight writer. - if (scope.owns(file.location()) && file.createdAtMillis() > 0 - && file.createdAtMillis() < olderThan - && !isReachable(file.location(), reachable)) { - orphanCount++; - if (!dryRun) { - table.io().deleteFile(file.location()); - deletedCount++; - } + ReachableIndex reachable = collectReachableFiles(table); + long orphanCount = 0; + long deletedCount = 0; + for (ScanScope scope : scanScopes) { + // Object stores use raw prefix matching, so the separator excludes sibling prefixes. + String listingPrefix = scope.root.endsWith("/") ? scope.root : scope.root + "/"; + for (FileInfo file : ((SupportsPrefixOperations) table.io()).listPrefix(listingPrefix)) { + // Unknown creation time cannot prove the file predates every in-flight writer. + if (scope.owns(file.location()) && file.createdAtMillis() > 0 + && file.createdAtMillis() < olderThan + && !isReachable(file.location(), reachable)) { + orphanCount++; + if (!dryRun) { + table.io().deleteFile(file.location()); + deletedCount++; } } } - return Lists.newArrayList(String.valueOf(orphanCount), String.valueOf(deletedCount)); - } catch (Exception e) { - throw new UserException("Failed to remove orphan files: " + e.getMessage(), e); } + return Lists.newArrayList(String.valueOf(orphanCount), String.valueOf(deletedCount)); } private List resolveScanScopes(Table table) throws UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 11e36fe8183835..94399c98575031 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -342,9 +342,19 @@ private boolean requiresIcebergScanSemanticsV2() throws UserException { private boolean hasApplicableEqualityDeletes(TableScan scan) throws UserException { Snapshot snapshot = scan.snapshot(); - if (snapshot == null || "0".equals(snapshot.summary().get("total-equality-deletes"))) { + if (snapshot == null) { return false; } + String equalityDeleteCount = snapshot.summary().get("total-equality-deletes"); + if (equalityDeleteCount != null) { + try { + // A positive snapshot total proves V2 semantics are required without opening every + // delete manifest; only old summaries that omit the counter need the fallback. + return Long.parseLong(equalityDeleteCount) > 0; + } catch (NumberFormatException ignored) { + // Fall through for non-standard summaries instead of weakening compatibility. + } + } // Inspect only delete manifests, not data tasks: equality-delete semantics are snapshot-wide // compatibility state even when the current predicate happens to prune their partitions. for (ManifestFile manifest : snapshot.deleteManifests(icebergTable.io())) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java index f7a55ae2f6c3c4..61fb5a74962c82 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java @@ -229,4 +229,36 @@ public void testDefaultValueUsesRelationSnapshotSchema() throws Exception { Mockito.verify(externalTable, Mockito.never()).getFullSchema(); } + @Test + public void testNonFiniteFloatingDefaultsBuildTypedScanExpressions() throws Exception { + TestFileQueryScanNode node = new TestFileQueryScanNode(new SessionVariable()); + IcebergExternalTable externalTable = Mockito.mock(IcebergExternalTable.class); + Column floatColumn = new Column("float_default", Type.FLOAT, false, + AggregateType.NONE, true, "NaN", ""); + Column doubleColumn = new Column("double_default", Type.DOUBLE, false, + AggregateType.NONE, true, "Infinity", ""); + List columns = Arrays.asList(floatColumn, doubleColumn); + node.setTargetTable(externalTable); + node.getTupleDescriptor().setTable(externalTable); + Mockito.when(externalTable.getBaseSchema(Optional.empty(), false)).thenReturn(columns); + Mockito.when(externalTable.getFullSchema(Optional.empty())).thenReturn(columns); + + SlotDescriptor floatSlot = new SlotDescriptor(new SlotId(1), node.getTupleDescriptor()); + floatSlot.setColumn(floatColumn); + node.getTupleDescriptor().addSlot(floatSlot); + SlotDescriptor doubleSlot = new SlotDescriptor(new SlotId(2), node.getTupleDescriptor()); + doubleSlot.setColumn(doubleColumn); + node.getTupleDescriptor().addSlot(doubleSlot); + + node.initSchemaParams(); + + TExpr floatExpr = node.params.getDefaultValueOfSrcSlot().get(floatSlot.getId().asInt()); + TExpr doubleExpr = node.params.getDefaultValueOfSrcSlot().get(doubleSlot.getId().asInt()); + Assert.assertEquals(TExprNodeType.FLOAT_LITERAL, floatExpr.getNodes().get(0).getNodeType()); + Assert.assertTrue(Double.isNaN(floatExpr.getNodes().get(0).getFloatLiteral().getValue())); + Assert.assertEquals(TExprNodeType.FLOAT_LITERAL, doubleExpr.getNodes().get(0).getNodeType()); + Assert.assertEquals(Double.POSITIVE_INFINITY, + doubleExpr.getNodes().get(0).getFloatLiteral().getValue(), 0.0); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index 830f9047e135ba..5dca9c3df5ac55 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -70,9 +70,30 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.StampedLock; public class IcebergTransactionTest { + @Test + public void testCleanupAndCommitAliasesShareCrossThreadFence() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergCommitCoordinator.Guard commit = + IcebergCommitCoordinator.beginCommit("s3://bucket/path/table"); + try { + StampedLock fence = IcebergCommitCoordinator.lockFor("s3a://BUCKET/path/table/"); + Assert.assertEquals(0, fence.tryWriteLock()); + executor.submit(commit::close).get(5, TimeUnit.SECONDS); + long maintenanceStamp = fence.tryWriteLock(); + Assert.assertNotEquals(0, maintenanceStamp); + fence.unlockWrite(maintenanceStamp); + } finally { + commit.close(); + executor.shutdownNow(); + } + } private static String dbName = "db3"; private static String tbWithPartition = "tbWithPartition"; @@ -601,6 +622,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use IcebergExternalTable icebergExternalTable = Mockito.mock(IcebergExternalTable.class); PartitionSpec spec = PartitionSpec.unpartitioned(); + Mockito.when(icebergTable.location()).thenReturn("s3a://warehouse/wh/db3/tbWithoutPartition"); Mockito.when(icebergTable.newTransaction()).thenReturn(icebergTxn); Mockito.when(icebergTable.currentSnapshot()).thenReturn(null); Mockito.when(icebergTable.spec()).thenReturn(spec); @@ -649,6 +671,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use Mockito.verify(rowDelta).removeDeletes(oldDeleteFile1); Mockito.verify(rowDelta).removeDeletes(oldDeleteFile2); Mockito.verify(rowDelta).commit(); + txn.rollback(); } private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expectRewrite) @@ -663,6 +686,7 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect IcebergExternalTable icebergExternalTable = Mockito.mock(IcebergExternalTable.class); PartitionSpec spec = PartitionSpec.unpartitioned(); + Mockito.when(icebergTable.location()).thenReturn("s3a://warehouse/wh/db3/tbWithoutPartition"); Mockito.when(icebergTable.newTransaction()).thenReturn(icebergTxn); Mockito.when(icebergTable.currentSnapshot()).thenReturn(null); Mockito.when(icebergTable.spec()).thenReturn(spec); @@ -713,6 +737,7 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect Mockito.verify(rowDelta, Mockito.never()).removeDeletes(ArgumentMatchers.any(DeleteFile.class)); } Mockito.verify(rowDelta).commit(); + txn.rollback(); } @Test @@ -722,12 +747,14 @@ public void testBeginInsertUsesRetainedTargetTable() throws UserException { Table retainedTable = Mockito.mock(Table.class); org.apache.iceberg.Transaction retainedTransaction = Mockito.mock(org.apache.iceberg.Transaction.class); + Mockito.when(retainedTable.location()).thenReturn("s3a://warehouse/retained_target"); Mockito.when(retainedTable.newTransaction()).thenReturn(retainedTransaction); IcebergTransaction txn = getTxn(); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); Mockito.verify(retainedTable).newTransaction(); + txn.rollback(); } @Test @@ -737,12 +764,14 @@ public void testBeginDeleteUsesRetainedTargetTable() throws UserException { Table retainedTable = Mockito.mock(Table.class); org.apache.iceberg.Transaction retainedTransaction = Mockito.mock(org.apache.iceberg.Transaction.class); + Mockito.when(retainedTable.location()).thenReturn("s3a://warehouse/retained_delete_target"); Mockito.when(retainedTable.newTransaction()).thenReturn(retainedTransaction); IcebergTransaction txn = getTxn(); txn.beginDelete(dorisTable, retainedTable); Mockito.verify(retainedTable).newTransaction(); + txn.rollback(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index cb728f13242b10..72539d190cc114 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -116,6 +116,13 @@ private static Table useFrozenTableGeneration(IcebergScanNode node, Table table) return (Table) method.invoke(node, table); } + private static boolean hasApplicableEqualityDeletes(IcebergScanNode node, TableScan scan) + throws Exception { + Method method = IcebergScanNode.class.getDeclaredMethod("hasApplicableEqualityDeletes", TableScan.class); + method.setAccessible(true); + return (boolean) method.invoke(node, scan); + } + private static class TestIcebergScanNode extends IcebergScanNode { private final boolean enableMappingVarbinary; private final boolean batchMode; @@ -217,6 +224,18 @@ public void testEmitsCurrentIcebergScanSemanticsCapability() { node.enableAndGetIcebergScanSemanticsVersion()); } + @Test + public void testPositiveEqualityDeleteSummaryAvoidsManifestIo() throws Exception { + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.summary()).thenReturn(ImmutableMap.of("total-equality-deletes", "1")); + TableScan scan = Mockito.mock(TableScan.class); + Mockito.when(scan.snapshot()).thenReturn(snapshot); + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + + Assert.assertTrue(hasApplicableEqualityDeletes(node, scan)); + Mockito.verify(snapshot, Mockito.never()).deleteManifests(Mockito.any()); + } + @Test public void testPartitionEvolutionKeepsNonFileSlotInReaderSchema() throws Exception { Column evolvedIdentityColumn = new Column("int_col", Type.BIGINT, true); diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out index f3f51aac96e1c5..673d6c879cb824 100644 --- a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out @@ -13,16 +13,22 @@ 3 \N \N \N \N \N 4 6000000000 7000000000 7.5 ["x", null, "z"] {"a":8000000000, "b":null} 5 50 5 \N \N {"null-value":null} +6 \N \N \N \N \N -- !complex_nulls -- 1 2 3 5 +6 -- !complex_partition_specs -- 0 3 -2 2 +3 5 + +-- !complex_nested_partition_pruning -- +4 +6 -- !complex_base_tag -- 1 [1, null, 3] {"x":10, "null-value":null} 10 old-a 1 \N From 175f1235691235deba57a9d6ea600781084294e3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 17 Aug 2026 23:06:19 +0800 Subject: [PATCH 4/4] fix: address remaining external write review feedback --- .../spill_iceberg_table_sink_operator.cpp | 49 +++++- .../writer/iceberg/viceberg_sort_writer.cpp | 12 +- .../paimon/ffi_paimon_write_backend.cpp | 4 + .../writer/paimon/ffi_paimon_write_backend.h | 1 + .../paimon/jni_paimon_write_backend.cpp | 14 ++ .../writer/paimon/jni_paimon_write_backend.h | 2 + .../writer/paimon/paimon_table_writer.cpp | 10 ++ .../sink/writer/paimon/paimon_table_writer.h | 1 + .../sink/writer/paimon/paimon_write_backend.h | 4 + be/src/exec/sort/sorter.cpp | 19 +- be/src/exec/sort/sorter.h | 7 + be/src/format_v2/table_reader.h | 164 ++++++++++++++++-- ...spill_iceberg_table_sink_operator_test.cpp | 50 ++++++ .../iceberg/iceberg_partition_writer_test.cpp | 28 +++ .../paimon/paimon_table_writer_test.cpp | 21 ++- be/test/format_v2/table_reader_test.cpp | 39 +++++ .../apache/doris/paimon/PaimonJniWriter.java | 41 +++++ .../rewrite/RewriteDataFileExecutor.java | 157 +++++++++-------- .../iceberg/rewrite/RewriteGroupTask.java | 49 ++++-- .../iceberg/source/IcebergScanNode.java | 59 ++++++- .../rewrite/RewriteDataFileExecutorTest.java | 35 +++- .../iceberg/source/IcebergScanNodeTest.java | 43 +++++ .../commands/IcebergUpdateCommandTest.java | 4 + 23 files changed, 707 insertions(+), 106 deletions(-) diff --git a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp index 59c6ce530b0f16..61daa07e5167e8 100644 --- a/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp +++ b/be/src/exec/operator/spill_iceberg_table_sink_operator.cpp @@ -20,6 +20,12 @@ #include "common/config.h" #include "common/status.h" #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_const.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" #include "exec/operator/iceberg_table_sink_operator.h" #include "exec/operator/spill_utils.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" @@ -28,6 +34,44 @@ namespace doris { #include "common/compile_check_begin.h" +namespace { +constexpr size_t MIN_POD_ARRAY_CAPACITY = 4096; + +// Admission runs before a writer may allocate; model the structural minimum recursively so a +// single oversized value cannot be cloned and multiplied by the cold-partition fan-out. +size_t minimum_selected_column_capacity(const IColumn& column) { + if (const auto* constant = check_and_get_column(column)) { + return minimum_selected_column_capacity(constant->get_data_column()); + } + if (const auto* nullable = check_and_get_column(column)) { + return iceberg_saturating_add( + MIN_POD_ARRAY_CAPACITY, + minimum_selected_column_capacity(nullable->get_nested_column())); + } + if (const auto* array = check_and_get_column(column)) { + return iceberg_saturating_add(MIN_POD_ARRAY_CAPACITY, + minimum_selected_column_capacity(array->get_data())); + } + if (const auto* map = check_and_get_column(column)) { + size_t capacity = iceberg_saturating_add(MIN_POD_ARRAY_CAPACITY, + minimum_selected_column_capacity(map->get_keys())); + return iceberg_saturating_add(capacity, + minimum_selected_column_capacity(map->get_values())); + } + if (const auto* structure = check_and_get_column(column)) { + size_t capacity = 0; + for (const auto& child : structure->get_columns()) { + capacity = iceberg_saturating_add(capacity, minimum_selected_column_capacity(*child)); + } + return capacity; + } + if (check_and_get_column(column) != nullptr) { + return 2 * MIN_POD_ARRAY_CAPACITY; + } + return MIN_POD_ARRAY_CAPACITY; +} +} // namespace + size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_workspace_bytes) { const size_t block_bytes = block.allocated_bytes(); const size_t row_index_bytes = @@ -40,9 +84,8 @@ size_t iceberg_cold_writer_reserve_size(const Block& block, size_t writer_worksp if (block.rows() > 0) { size_t minimum_selected_block_bytes = 0; for (const auto& column : block.get_columns_with_type_and_name()) { - minimum_selected_block_bytes = - iceberg_saturating_add(minimum_selected_block_bytes, - column.column->clone_resized(1)->allocated_bytes()); + minimum_selected_block_bytes = iceberg_saturating_add( + minimum_selected_block_bytes, minimum_selected_column_capacity(*column.column)); } const size_t max_partition_count = static_cast( std::max(1, config::table_sink_partition_write_max_partition_nums_per_writer)); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp index 572445f38b139b..d610725047b37c 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp @@ -92,7 +92,10 @@ SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components(RuntimeS if (_sorter == nullptr) { return {}; } - auto reservation = _sorter->get_reserve_mem_size_components(state, eos); + const size_t target = _target_file_size_bytes >= 0 + ? static_cast(_target_file_size_bytes) + : std::numeric_limits::max(); + auto reservation = _sorter->get_reserve_mem_size_components(state, eos, target); _include_spill_merge_reservation(state, eos, &reservation); return reservation; } @@ -103,8 +106,11 @@ SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components( if (_sorter == nullptr) { return {}; } - auto reservation = - _sorter->get_reserve_mem_size_components(state, eos, incoming_rows, incoming_bytes); + const size_t target = _target_file_size_bytes >= 0 + ? static_cast(_target_file_size_bytes) + : std::numeric_limits::max(); + auto reservation = _sorter->get_reserve_mem_size_components(state, eos, incoming_rows, + incoming_bytes, target); _include_spill_merge_reservation(state, eos, &reservation); return reservation; } diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp index a5abfcdc15c41c..13c979f094124b 100644 --- a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp @@ -27,6 +27,10 @@ Status FfiPaimonWriteBackend::create_writer(std::unique_ptr*) { return Status::NotSupported("Paimon Rust FFI writer is not implemented"); } +Status FfiPaimonWriteBackend::prepare_close_for_commit() { + return Status::NotSupported("Paimon Rust FFI writer is not implemented"); +} + Status FfiPaimonWriteBackend::close() { return Status::OK(); } diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h index be833d53b79bcd..54d5c593661207 100644 --- a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h @@ -29,6 +29,7 @@ class FfiPaimonWriteBackend final : public IPaimonWriteBackend { Status open(const TPaimonTableSink& sink, RuntimeState* state, RuntimeProfile* profile) override; Status create_writer(std::unique_ptr* writer) override; + Status prepare_close_for_commit() override; Status close() override; PaimonBackendType type() const override { return PaimonBackendType::FFI; } }; diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp index e59668759df65b..f7ca5e3ea31765 100644 --- a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp @@ -234,6 +234,18 @@ Status JniPaimonWriteBackend::close() { return close_status; } +Status JniPaimonWriteBackend::prepare_close_for_commit() { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(_get_jni_env(&env)); + if (_jni_writer_obj == nullptr || _prepare_close_for_commit_id == nullptr) { + return Status::InternalError("Paimon prepared writer close method is unavailable"); + } + env->CallVoidMethod(_jni_writer_obj, _prepare_close_for_commit_id); + Status status = _check_jni_exception(env, "prepare-close PaimonJniWriter"); + _refresh_memory_profile(); + return status; +} + Status JniPaimonWriteBackend::_check_jni_exception(JNIEnv* env, const std::string& method_name) { if (env->ExceptionCheck()) { Status st = @@ -326,6 +338,8 @@ Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* s _write_id = env->GetMethodID(_jni_writer_cls, "write", "(Ljava/nio/ByteBuffer;)V"); _prepare_commit_id = env->GetMethodID(_jni_writer_cls, "prepareCommit", "()[[B"); _abort_id = env->GetMethodID(_jni_writer_cls, "abort", "()V"); + _prepare_close_for_commit_id = + env->GetMethodID(_jni_writer_cls, "prepareCloseForCommit", "()V"); _close_id = env->GetMethodID(_jni_writer_cls, "close", "()V"); RETURN_IF_ERROR(_check_jni_exception(env, "GetMethodID")); diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h index ae36bdf37a3582..72e75b1f67bc96 100644 --- a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h @@ -49,6 +49,7 @@ class JniPaimonWriteBackend final : public IPaimonWriteBackend { Status open(const TPaimonTableSink& sink, RuntimeState* state, RuntimeProfile* profile) override; Status create_writer(std::unique_ptr* writer) override; + Status prepare_close_for_commit() override; Status close() override; PaimonBackendType type() const override { return PaimonBackendType::JNI; } @@ -65,6 +66,7 @@ class JniPaimonWriteBackend final : public IPaimonWriteBackend { jmethodID _write_id = nullptr; jmethodID _prepare_commit_id = nullptr; jmethodID _abort_id = nullptr; + jmethodID _prepare_close_for_commit_id = nullptr; jmethodID _close_id = nullptr; TPaimonTableSink _sink; diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp index 38069545da7db9..37e07f9566baf8 100644 --- a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp @@ -32,6 +32,10 @@ PaimonPreparedCommitOwner::~PaimonPreparedCommitOwner() { _close(); } +Status PaimonPreparedCommitOwner::prepare_for_report() { + return _backend == nullptr ? Status::OK() : _backend->prepare_close_for_commit(); +} + void PaimonPreparedCommitOwner::finalize(ExternalFileReportOutcome outcome) { if (_finalized || outcome == ExternalFileReportOutcome::AMBIGUOUS) { return; @@ -179,6 +183,12 @@ Status PaimonTableWriter::close(Status status) { } auto owner = std::make_shared(std::move(_writer), std::move(_backend)); + Status close_st = owner->prepare_for_report(); + if (!close_st.ok()) { + // Commit messages cannot become coordinator-owned until every SDK user has stopped. + owner->finalize(ExternalFileReportOutcome::REJECTED); + return close_st; + } // Paimon's abort API needs the prepared Java writer. Retain that owner until the shared // final report is accepted or rejected instead of publishing an irreversible payload. _state->add_external_file_report_finalizer( diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.h b/be/src/exec/sink/writer/paimon/paimon_table_writer.h index b003187c4785eb..7a78f14ecd29cb 100644 --- a/be/src/exec/sink/writer/paimon/paimon_table_writer.h +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.h @@ -40,6 +40,7 @@ class PaimonPreparedCommitOwner { std::unique_ptr backend); ~PaimonPreparedCommitOwner(); + Status prepare_for_report(); void finalize(ExternalFileReportOutcome outcome); private: diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend.h b/be/src/exec/sink/writer/paimon/paimon_write_backend.h index 9993581b9fe894..2af3ffe42db9ae 100644 --- a/be/src/exec/sink/writer/paimon/paimon_write_backend.h +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend.h @@ -80,6 +80,10 @@ class IPaimonWriteBackend { /// Create a lightweight writer adapter that delegates to this backend. virtual Status create_writer(std::unique_ptr* writer) = 0; + /// Stop SDK writers before commit metadata can be accepted while preserving enough metadata + /// for abort(messages) if the coordinator rejects the final report. + virtual Status prepare_close_for_commit() = 0; + /// Stop all SDK users and release backend resources. /// /// A successful return is the ownership boundary after which native memory diff --git a/be/src/exec/sort/sorter.cpp b/be/src/exec/sort/sorter.cpp index 8a7f5809e8c1ae..2e3128c7c71938 100644 --- a/be/src/exec/sort/sorter.cpp +++ b/be/src/exec/sort/sorter.cpp @@ -204,17 +204,30 @@ size_t FullSorter::get_reserve_mem_size(RuntimeState* state, bool eos) const { SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, bool eos) const { + return get_reserve_mem_size_components(state, eos, std::numeric_limits::max()); +} + +SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t sort_threshold_bytes) const { const auto rows = _state->unsorted_block()->rows(); const auto bytes = _state->unsorted_block()->bytes(); const auto bytes_per_row = rows == 0 ? 0 : bytes / rows; return get_reserve_mem_size_components( state, eos, state->batch_size(), - saturating_multiply_size(bytes_per_row, state->batch_size())); + saturating_multiply_size(bytes_per_row, state->batch_size()), sort_threshold_bytes); } SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, bool eos, size_t incoming_rows, size_t incoming_bytes) const { + return get_reserve_mem_size_components(state, eos, incoming_rows, incoming_bytes, + std::numeric_limits::max()); +} + +SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t incoming_rows, + size_t incoming_bytes, + size_t sort_threshold_bytes) const { SorterReserveMemory reserve; const auto rows = _state->unsorted_block()->rows(); if (rows != 0) { @@ -234,8 +247,10 @@ SorterReserveMemory FullSorter::get_reserve_mem_size_components(RuntimeState* st } // Iceberg close forces every nonempty pending run to sort at EOS, even when the generic // append thresholds are not reached, so admission must cover that final allocation too. + // The reservation must mirror every caller-side rollover that immediately invokes do_sort(). auto sort = (eos && new_rows > 0) || new_rows > _buffered_block_size || - new_block_bytes > _buffered_block_bytes; + new_block_bytes > _buffered_block_bytes || + new_block_bytes >= sort_threshold_bytes; if (sort) { // sort_block keeps the source columns live while materializing a fully permuted destination. reserve.transient_workspace = diff --git a/be/src/exec/sort/sorter.h b/be/src/exec/sort/sorter.h index 5d3401e52aae21..a7aa650f5a2dbe 100644 --- a/be/src/exec/sort/sorter.h +++ b/be/src/exec/sort/sorter.h @@ -212,10 +212,17 @@ class FullSorter final : public Sorter { SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos) const; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t sort_threshold_bytes) const; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos, size_t incoming_rows, size_t incoming_bytes) const; + SorterReserveMemory get_reserve_mem_size_components(RuntimeState* state, bool eos, + size_t incoming_rows, size_t incoming_bytes, + size_t sort_threshold_bytes) const; + Status merge_sort_read_for_spill(RuntimeState* state, doris::Block* block, int batch_size, bool* eos) override; void reset() override; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index e8b7225ad0b691..45d3c9b4d986d8 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1716,6 +1716,133 @@ class TableReader { return false; } + static bool _mapping_requires_parent_null_map_at(const ColumnMapping& mapping, + const ColumnPtr& column, + const DataTypePtr& table_type, + const size_t row) { + DORIS_CHECK(column.get() != nullptr); + DORIS_CHECK(table_type != nullptr); + DORIS_CHECK(row < column->size()); + if (table_type->is_nullable()) { + const auto& nested_type = + assert_cast(*table_type).get_nested_type(); + if (const auto* nullable = check_and_get_column(*column)) { + if (nullable->is_null_at(row)) { + return false; + } + return _mapping_requires_parent_null_map_at( + mapping, nullable->get_nested_column_ptr(), nested_type, row); + } + return _mapping_requires_parent_null_map_at(mapping, column, nested_type, row); + } + if (const auto* nullable = check_and_get_column(*column)) { + if (nullable->is_null_at(row)) { + return true; + } + return _mapping_requires_parent_null_map_at(mapping, nullable->get_nested_column_ptr(), + table_type, row); + } + if (mapping.is_trivial) { + return _requires_parent_null_map_for_alignment_at(column, table_type, row); + } + if (mapping.child_mappings.empty()) { + if (is_complex_type(table_type->get_primitive_type())) { + // A fully pruned complex subtree has no materialization consumer for this mask. + return false; + } + return _requires_parent_null_map_for_alignment_at(column, table_type, row); + } + if (typeid_cast(table_type.get()) != nullptr) { + const auto& struct_column = assert_cast(*column); + const auto file_ordered_children = + _present_child_mappings_in_file_order(mapping.child_mappings); + for (const auto* child_mapping : file_ordered_children) { + const size_t ordinal = _file_child_ordinal_for_mapping(mapping, *child_mapping, + file_ordered_children); + DORIS_CHECK(ordinal < struct_column.tuple_size()); + if (_mapping_requires_parent_null_map_at(*child_mapping, + struct_column.get_column_ptr(ordinal), + child_mapping->table_type, row)) { + return true; + } + } + return false; + } + if (const auto* array_type = typeid_cast(table_type.get())) { + const auto& array_column = assert_cast(*column); + const auto& element_mapping = mapping.child_mappings[0]; + if (!element_mapping.file_local_id.has_value()) { + return false; + } + const auto& offsets = array_column.get_offsets(); + const size_t begin = row == 0 ? 0 : offsets[row - 1]; + const size_t end = offsets[row]; + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_mapping_requires_parent_null_map_at( + element_mapping, array_column.get_data_ptr(), + array_type->get_nested_type(), child_row)) { + return true; + } + } + return false; + } + if (const auto* map_type = typeid_cast(table_type.get())) { + const auto& map_column = assert_cast(*column); + const auto& offsets = map_column.get_offsets(); + const size_t begin = row == 0 ? 0 : offsets[row - 1]; + const size_t end = offsets[row]; + for (const auto& child_mapping : mapping.child_mappings) { + if (!child_mapping.file_local_id.has_value()) { + continue; + } + const bool is_key = *child_mapping.file_local_id == 0; + const ColumnPtr& child_column = + is_key ? map_column.get_keys_ptr() : map_column.get_values_ptr(); + const DataTypePtr& child_type = + is_key ? map_type->get_key_type() : map_type->get_value_type(); + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_mapping_requires_parent_null_map_at(child_mapping, child_column, + child_type, child_row)) { + return true; + } + } + } + return false; + } + return false; + } + + template + static bool _mapping_requires_collection_parent_null_map(const NullMap* container_null_map, + const NullMap* ancestor_null_map, + const ColumnMapping& mapping, + const ColumnPtr& column, + const size_t rows, + const Offsets& offsets) { + DORIS_CHECK(container_null_map == nullptr || container_null_map->size() == rows); + DORIS_CHECK(ancestor_null_map == nullptr || ancestor_null_map->size() == rows); + DORIS_CHECK(offsets.size() == rows); + DORIS_CHECK(offsets.empty() || offsets.back() == column->size()); + size_t begin = 0; + for (size_t row = 0; row < rows; ++row) { + const size_t end = offsets[row]; + const bool hidden = (container_null_map != nullptr && (*container_null_map)[row]) || + (ancestor_null_map != nullptr && (*ancestor_null_map)[row]); + if (hidden) { + // Only mapped descendants can consume the projected mask; probing physical + // siblings would reintroduce an entry-sized allocation for pruned schemas. + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_mapping_requires_parent_null_map_at(mapping, column, mapping.table_type, + child_row)) { + return true; + } + } + } + begin = end; + } + return false; + } + template static const NullMap* _project_collection_parent_null_map_for_hidden_entries( const NullMap* container_null_map, const NullMap* ancestor_null_map, const size_t rows, @@ -1867,10 +1994,14 @@ class TableReader { // storage invariant, so add it only at the materialization boundary. element_mapping.table_type = make_nullable(element_mapping.table_type); NullMap descendant_parent_null_map; - const NullMap* descendant_parent_null_map_ptr = - _project_collection_parent_null_map_for_hidden_entries( - parent_null_map, nullable_parent_null_map, rows, file_array->get_offsets(), - nested_column->size(), &descendant_parent_null_map); + const NullMap* descendant_parent_null_map_ptr = nullptr; + if (_mapping_requires_collection_parent_null_map(parent_null_map, nullable_parent_null_map, + element_mapping, nested_column, rows, + file_array->get_offsets())) { + descendant_parent_null_map_ptr = _project_collection_parent_null_map_for_hidden_entries( + parent_null_map, nullable_parent_null_map, rows, file_array->get_offsets(), + nested_column->size(), &descendant_parent_null_map); + } RETURN_IF_ERROR(_materialize_present_child_mapping_column( element_mapping, nested_column, nested_column->size(), &nested_column, descendant_parent_null_map_ptr)); @@ -1921,12 +2052,6 @@ class TableReader { ColumnPtr key_column = file_map->get_keys_ptr(); ColumnPtr value_column = file_map->get_values_ptr(); DORIS_CHECK(key_column->size() == value_column->size()); - NullMap descendant_parent_null_map; - const NullMap* descendant_parent_null_map_ptr = - _project_collection_parent_null_map_for_hidden_entries( - parent_null_map, nullable_parent_null_map, rows, file_map->get_offsets(), - key_column->size(), &descendant_parent_null_map); - const ColumnMapping* key_mapping = nullptr; const ColumnMapping* value_mapping = nullptr; for (const auto& child_mapping : mapping.child_mappings) { @@ -1940,6 +2065,25 @@ class TableReader { } } + bool requires_parent_null_map = false; + if (key_mapping != nullptr) { + requires_parent_null_map = _mapping_requires_collection_parent_null_map( + parent_null_map, nullable_parent_null_map, *key_mapping, key_column, rows, + file_map->get_offsets()); + } + if (!requires_parent_null_map && value_mapping != nullptr) { + requires_parent_null_map = _mapping_requires_collection_parent_null_map( + parent_null_map, nullable_parent_null_map, *value_mapping, value_column, rows, + file_map->get_offsets()); + } + NullMap descendant_parent_null_map; + const NullMap* descendant_parent_null_map_ptr = nullptr; + if (requires_parent_null_map) { + descendant_parent_null_map_ptr = _project_collection_parent_null_map_for_hidden_entries( + parent_null_map, nullable_parent_null_map, rows, file_map->get_offsets(), + key_column->size(), &descendant_parent_null_map); + } + if (key_mapping != nullptr) { RETURN_IF_ERROR(_materialize_present_child_mapping_column( *key_mapping, key_column, key_column->size(), &key_column, diff --git a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp index 740b098373000f..5effca9dba3416 100644 --- a/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp +++ b/be/test/exec/operator/spill_iceberg_table_sink_operator_test.cpp @@ -22,7 +22,14 @@ #include #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "exec/operator/iceberg_sorter_reserve_memory.h" #include "exec/sink/writer/async_writer_queue_admission.h" @@ -88,6 +95,49 @@ TEST(SpillIcebergTableSinkOperatorTest, ColdWriterReserveUsesFirstBlockLargerTha 4 * block.allocated_bytes() + operator_floor); } +TEST(SpillIcebergTableSinkOperatorTest, ColdWriterReserveDoesNotAmplifyHugeFirstValuePerPartition) { + constexpr size_t operator_floor = 32 * 1024 * 1024; + constexpr size_t payload_size = 8 * 1024 * 1024; + std::string payload(payload_size, 'x'); + + auto strings = ColumnString::create(); + strings->insert_data(payload.data(), payload.size()); + + auto array_strings = ColumnString::create(); + array_strings->insert_data(payload.data(), payload.size()); + auto array_nulls = ColumnUInt8::create(1, 0); + auto array_offsets = ColumnArray::ColumnOffsets::create(); + array_offsets->get_data().push_back(1); + auto array = ColumnArray::create( + ColumnNullable::create(std::move(array_strings), std::move(array_nulls)), + std::move(array_offsets)); + + auto map_keys = ColumnString::create(); + map_keys->insert_data(payload.data(), payload.size()); + auto map_values = ColumnInt32::create(1, 7); + auto map_offsets = ColumnArray::ColumnOffsets::create(); + map_offsets->get_data().push_back(1); + auto map = + ColumnMap::create(std::move(map_keys), std::move(map_values), std::move(map_offsets)); + + Block block; + block.insert({std::move(strings), std::make_shared(), "payload"}); + block.insert( + {std::move(array), + std::make_shared(make_nullable(std::make_shared())), + "items"}); + block.insert({std::move(map), + std::make_shared(std::make_shared(), + std::make_shared()), + "attributes"}); + + const size_t reserve = iceberg_cold_writer_reserve_size(block, operator_floor); + + // The input payload is covered by dispatch_copies; only structural column capacity is retained + // once per touched partition. + EXPECT_LT(reserve, 8 * block.allocated_bytes() + operator_floor); +} + TEST(SpillIcebergTableSinkOperatorTest, ReservesAllMergeInputsAndOutputAtEos) { constexpr size_t MB = 1024 * 1024; diff --git a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp index d5a7e3f9ab3351..062939f7710df9 100644 --- a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp @@ -21,6 +21,7 @@ #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/iceberg/viceberg_sort_writer.h" +#include "testutil/column_helper.h" #include "testutil/mock/mock_descriptors.h" #include "testutil/mock/mock_runtime_state.h" #include "testutil/mock/mock_slot_ref.h" @@ -158,4 +159,31 @@ TEST_F(VIcebergPartitionWriterTest, EosReservationIncludesActualSpillFanIn) { EXPECT_EQ(72 * 1024 * 1024, reservation.transient_workspace); } +TEST_F(VIcebergPartitionWriterTest, TargetSizeReservationIncludesImmediateSortBelowGenericLimit) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto partition_writer = std::shared_ptr( + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf)); + VIcebergSortWriter sort_writer(partition_writer, TSortInfo(), 64); + MockRuntimeState state; + ObjectPool pool; + auto row_desc = std::make_unique( + std::vector {std::make_shared()}, &pool); + auto ordering_expr_ctxs = + MockSlotRef::create_mock_contexts(0, std::make_shared()); + std::vector is_asc_order {true}; + std::vector nulls_first {false}; + sort_writer._sorter = FullSorter::create_unique(ordering_expr_ctxs, -1, 0, &pool, is_asc_order, + nulls_first, *row_desc, &state, nullptr); + Block block = ColumnHelper::create_block({10, 9, 8, 7, 6, 5, 4, 3, 2, 1}); + ASSERT_LT(block.bytes(), 256 * 1024 * 1024); + ASSERT_TRUE(sort_writer._sorter->append_block(&block).ok()); + + const auto reservation = sort_writer.get_reserve_mem_size_components(&state, false, 0, 0); + + EXPECT_GT(reservation.transient_workspace, 0); +} + } // namespace doris diff --git a/be/test/exec/sink/writer/paimon/paimon_table_writer_test.cpp b/be/test/exec/sink/writer/paimon/paimon_table_writer_test.cpp index afdd2f765e9f1f..62377352963534 100644 --- a/be/test/exec/sink/writer/paimon/paimon_table_writer_test.cpp +++ b/be/test/exec/sink/writer/paimon/paimon_table_writer_test.cpp @@ -43,12 +43,14 @@ class FakePaimonWriter final : public IPaimonWriter { class FakePaimonBackend final : public IPaimonWriteBackend { public: - explicit FakePaimonBackend(int* close_count) : _close_count(close_count) {} + explicit FakePaimonBackend(int* close_count, Status prepare_close_status = Status::OK()) + : _close_count(close_count), _prepare_close_status(std::move(prepare_close_status)) {} Status open(const TPaimonTableSink&, RuntimeState*, RuntimeProfile*) override { return Status::OK(); } Status create_writer(std::unique_ptr*) override { return Status::OK(); } + Status prepare_close_for_commit() override { return _prepare_close_status; } Status close() override { ++*_close_count; return Status::OK(); @@ -57,6 +59,7 @@ class FakePaimonBackend final : public IPaimonWriteBackend { private: int* _close_count; + Status _prepare_close_status; }; } // namespace @@ -100,4 +103,20 @@ TEST(PaimonPreparedCommitOwnerTest, AmbiguousReportRetainsOwnerUntilAcknowledged EXPECT_EQ(1, close_count); } +TEST(PaimonPreparedCommitOwnerTest, FailedSdkShutdownRejectsCommitBeforeAcknowledgement) { + int abort_count = 0; + int close_count = 0; + PaimonPreparedCommitOwner owner( + std::make_unique(&abort_count), + std::make_unique( + &close_count, Status::InternalError("injected SDK shutdown failure"))); + + Status status = owner.prepare_for_report(); + owner.finalize(ExternalFileReportOutcome::REJECTED); + + EXPECT_FALSE(status.ok()); + EXPECT_EQ(1, abort_count); + EXPECT_EQ(1, close_count); +} + } // namespace doris diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 923caf2f8daab7..7f9e762f657104 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1116,6 +1116,7 @@ class TableReaderCastTestHelper final : public TableReader { using TableReader::_materialize_map_mapping_column; using TableReader::_materialize_present_child_mapping_column; using TableReader::_materialize_struct_mapping_column; + using TableReader::_mapping_requires_collection_parent_null_map; using TableReader::_project_collection_parent_null_map_for_hidden_entries; using TableReader::_requires_collection_parent_null_map; using TableReader::_requires_parent_null_map_for_alignment; @@ -6566,5 +6567,43 @@ TEST(TableReaderTest, CollectionParentMaskSkipsLargeMapWhenOnlyEmptyRowIsHidden) EXPECT_TRUE(projected_null_map.empty()); } +TEST(TableReaderTest, MappingProbeSkipsLargeEvolvedArrayWithoutRequiredConsumer) { + constexpr size_t visible_entries = 500000; + const size_t entries = visible_entries + 1; + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto struct_type = + std::make_shared(DataTypes {nullable_int_type}, Strings {"kept"}); + + ColumnMapping child_mapping; + child_mapping.table_column_name = "kept"; + child_mapping.table_type = nullable_int_type; + child_mapping.file_local_id = 0; + ColumnMapping element_mapping; + element_mapping.table_type = make_nullable(struct_type); + element_mapping.child_mappings = {child_mapping}; + element_mapping.is_trivial = false; + + auto values = ColumnInt32::create(entries, 0); + auto child_null_map = ColumnUInt8::create(entries, 0); + child_null_map->get_data()[0] = 1; + MutableColumns physical_children; + physical_children.push_back( + ColumnNullable::create(std::move(values), std::move(child_null_map))); + ColumnPtr physical_elements = ColumnStruct::create(std::move(physical_children)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {1, entries}; + + EXPECT_FALSE(TableReaderCastTestHelper::_mapping_requires_collection_parent_null_map( + nullptr, &parent_null_map, element_mapping, physical_elements, 2, offsets)); + + child_mapping.table_type = int_type; + element_mapping.table_type = + make_nullable(std::make_shared(DataTypes {int_type}, Strings {"kept"})); + element_mapping.child_mappings = {child_mapping}; + EXPECT_TRUE(TableReaderCastTestHelper::_mapping_requires_collection_parent_null_map( + nullptr, &parent_null_map, element_mapping, physical_elements, 2, offsets)); +} + } // namespace } // namespace doris::format diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java index 21c5682f638576..3b8b7316386ef6 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java @@ -278,6 +278,23 @@ public void abort() throws Exception { } } + /** + * Stop SDK writers before the coordinator can accept commit messages, while retaining the + * table and prepared messages needed to abort a rejected report. + */ + public void prepareCloseForCommit() throws Exception { + try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) { + if (preExecutionAuthenticator != null) { + preExecutionAuthenticator.execute(() -> { + closePreparedResources(); + return null; + }); + } else { + closePreparedResources(); + } + } + } + /** * Close: release all resources. */ @@ -501,6 +518,30 @@ private void closeResources() throws Exception { } } + private void closePreparedResources() throws Exception { + if (sdkCloseFailed) { + throw new IllegalStateException( + "A previous Paimon SDK close failed; native memory cannot be released safely"); + } + Exception failure = closeResource(writer, null); + failure = closeResource(globalIndexAssigner, failure); + failure = closeResource(ioManager, failure); + writer = null; + hashBucketAssigner = null; + dynamicBucketExtractor = null; + globalIndexAssigner = null; + ioManager = null; + fullCompactionBuckets.clear(); + if (allocator != null) { + failure = closeResource(allocator, failure); + allocator = null; + } + if (failure != null) { + sdkCloseFailed = true; + throw failure; + } + } + private List prepareCommitMessages() throws Exception { if (writer == null) { throw new IllegalStateException("Paimon writer is not open"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java index 095bad988849ef..fd38a0b5a0a1ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java @@ -27,7 +27,9 @@ import org.apache.doris.resource.computegroup.ComputeGroup; import org.apache.doris.scheduler.exception.JobException; import org.apache.doris.scheduler.executor.TransientTaskExecutor; +import org.apache.doris.scheduler.manager.TransientTaskManager; import org.apache.doris.system.Backend; +import org.apache.doris.transaction.TransactionManager; import com.google.common.collect.Lists; // Keep third-party imports lexical to preserve the repository's CustomImportOrder invariant. @@ -61,88 +63,103 @@ public RewriteDataFileExecutor(IcebergExternalTable dorisTable, */ public RewriteResult executeGroupsConcurrently(List groups, long targetFileSizeBytes) throws UserException { - // Begin transaction - long transactionId = dorisTable.getCatalog().getTransactionManager().begin(); - IcebergTransaction transaction = (IcebergTransaction) dorisTable.getCatalog().getTransactionManager() - .getTransaction(transactionId); - MvccSnapshot targetSnapshot = dorisTable.loadSnapshot(Optional.empty(), Optional.empty()); - Table targetIcebergTable = ((IcebergMvccSnapshot) targetSnapshot).getSnapshotCacheValue() - .getIcebergTable().orElseThrow( - () -> new UserException("Iceberg rewrite target metadata is not available")); - transaction.beginRewrite(dorisTable, targetIcebergTable); - - // Register files to delete - for (RewriteDataGroup group : groups) { - transaction.updateRewriteFiles(Lists.newArrayList(group.getDataFiles())); - } - - // Create result collector and tasks + TransactionManager transactionManager = dorisTable.getCatalog().getTransactionManager(); + long transactionId = transactionManager.begin(); List tasks = Lists.newArrayList(); - RewriteResultCollector resultCollector = new RewriteResultCollector(groups.size(), tasks); - - // Get available BE count once before creating tasks - // This avoids calling getBackendsNumber() in each task during multi-threaded execution. - // Use compute group from connect context to align with actual BE selection for queries. - int availableBeCount = getAvailableBeCount(); - - // Create tasks with callbacks - for (RewriteDataGroup group : groups) { - RewriteGroupTask task = new RewriteGroupTask( - group, - transactionId, - dorisTable, - targetSnapshot, - connectContext, - targetFileSizeBytes, - availableBeCount, - new RewriteGroupTask.RewriteResultCallback() { - @Override - public void onTaskCompleted(Long taskId) { - resultCollector.onTaskCompleted(taskId); - } - - @Override - public void onTaskFailed(Long taskId, Exception error) { - resultCollector.onTaskFailed(taskId, error); - } - }); - tasks.add(task); - } - - // Submit tasks to TransientTaskManager + boolean committed = false; try { - for (TransientTaskExecutor task : tasks) { - Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(task); + IcebergTransaction transaction = (IcebergTransaction) transactionManager + .getTransaction(transactionId); + MvccSnapshot targetSnapshot = dorisTable.loadSnapshot(Optional.empty(), Optional.empty()); + Table targetIcebergTable = ((IcebergMvccSnapshot) targetSnapshot).getSnapshotCacheValue() + .getIcebergTable().orElseThrow( + () -> new UserException("Iceberg rewrite target metadata is not available")); + transaction.beginRewrite(dorisTable, targetIcebergTable); + + for (RewriteDataGroup group : groups) { + transaction.updateRewriteFiles(Lists.newArrayList(group.getDataFiles())); } - } catch (JobException e) { - throw new UserException("Failed to submit rewrite tasks: " + e.getMessage(), e); - } - // Wait for all tasks to complete - waitForTasksCompletion(resultCollector, groups.size()); - - // Finish rewrite operation - transaction.finishRewrite(); - - // Collect statistics from transaction after all tasks are completed - int rewrittenDataFilesCount = groups.stream().mapToInt(group -> group.getDataFiles().size()).sum(); - // this should after finishRewrite - int addedDataFilesCount = transaction.getFilesToAddCount(); - long rewrittenBytesCount = groups.stream().mapToLong(group -> group.getTotalSize()).sum(); - int removedDeleteFilesCount = groups.stream().mapToInt(group -> group.getDeleteFileCount()).sum(); + RewriteResultCollector resultCollector = new RewriteResultCollector(groups.size(), tasks); + int availableBeCount = getAvailableBeCount(); + for (RewriteDataGroup group : groups) { + RewriteGroupTask task = new RewriteGroupTask( + group, transactionId, dorisTable, targetSnapshot, connectContext, + targetFileSizeBytes, availableBeCount, + new RewriteGroupTask.RewriteResultCallback() { + @Override + public void onTaskCompleted(Long taskId) { + resultCollector.onTaskCompleted(taskId); + } + + @Override + public void onTaskFailed(Long taskId, Exception error) { + resultCollector.onTaskFailed(taskId, error); + } + }); + tasks.add(task); + } - commitAndInvalidate(transaction); + try { + for (TransientTaskExecutor task : tasks) { + Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(task); + } + } catch (JobException e) { + throw new UserException("Failed to submit rewrite tasks: " + e.getMessage(), e); + } - return new RewriteResult(rewrittenDataFilesCount, addedDataFilesCount, - rewrittenBytesCount, removedDeleteFilesCount); + waitForTasksCompletion(resultCollector, groups.size()); + transaction.finishRewrite(); + + int rewrittenDataFilesCount = groups.stream() + .mapToInt(group -> group.getDataFiles().size()).sum(); + int addedDataFilesCount = transaction.getFilesToAddCount(); + long rewrittenBytesCount = groups.stream().mapToLong(group -> group.getTotalSize()).sum(); + int removedDeleteFilesCount = groups.stream() + .mapToInt(group -> group.getDeleteFileCount()).sum(); + + commitAndInvalidate(transactionManager, transactionId); + committed = true; + return new RewriteResult(rewrittenDataFilesCount, addedDataFilesCount, + rewrittenBytesCount, removedDeleteFilesCount); + } finally { + if (!committed) { + cancelAndQuiesce(tasks); + // No task may update the transaction after rollback releases its rewrite fence. + transactionManager.rollback(transactionId); + } + } } - void commitAndInvalidate(IcebergTransaction transaction) throws UserException { - transaction.commit(); + void commitAndInvalidate(TransactionManager transactionManager, long transactionId) + throws UserException { + transactionManager.commit(transactionId); // Rewrite commits bypass the external-table DDL path, so evict the pre-rewrite snapshot before reuse. Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable); } + private void cancelAndQuiesce(List tasks) { + if (tasks.isEmpty()) { + return; + } + TransientTaskManager taskManager = Env.getCurrentEnv().getTransientTaskManager(); + for (RewriteGroupTask task : tasks) { + try { + taskManager.cancelMemoryTask(task.getId()); + } catch (JobException e) { + LOG.warn("Failed to remove rewrite task {} from the transient queue", task.getId(), e); + } + try { + task.cancel(); + } catch (JobException e) { + LOG.warn("Failed to cancel rewrite task {}", task.getId(), e); + } + } + for (RewriteGroupTask task : tasks) { + task.awaitCompletionUninterruptibly(); + } + } + /** * Wait for all tasks to complete using notification mechanism */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java index d20931a83b309c..2d51e93bd3c098 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -67,10 +68,12 @@ public class RewriteGroupTask implements TransientTaskExecutor { private final Long taskId; private final AtomicBoolean isCanceled; private final AtomicBoolean isFinished; + private final AtomicBoolean isStarted; + private final CountDownLatch completionLatch; private final int availableBeCount; // for canceling the task - private StmtExecutor stmtExecutor; + private volatile StmtExecutor stmtExecutor; public RewriteGroupTask(RewriteDataGroup group, long transactionId, @@ -91,6 +94,8 @@ public RewriteGroupTask(RewriteDataGroup group, this.taskId = UUID.randomUUID().getMostSignificantBits(); this.isCanceled = new AtomicBoolean(false); this.isFinished = new AtomicBoolean(false); + this.isStarted = new AtomicBoolean(false); + this.completionLatch = new CountDownLatch(1); } // Tests that only exercise scheduling strategy do not create an Iceberg metadata snapshot. @@ -115,17 +120,15 @@ public void execute() throws JobException { LOG.debug("[Rewrite Task] taskId: {} starting execution for group with {} tasks", taskId, group.getTaskCount()); - if (isCanceled.get()) { - LOG.debug("[Rewrite Task] taskId: {} was already canceled before execution", taskId); - throw new JobException("Rewrite task has been canceled, task id: " + taskId); - } - - if (isFinished.get()) { - LOG.debug("[Rewrite Task] taskId: {} was already finished", taskId); + if (!isStarted.compareAndSet(false, true)) { return; } try { + if (isCanceled.get()) { + LOG.debug("[Rewrite Task] taskId: {} was already canceled before execution", taskId); + throw new JobException("Rewrite task has been canceled, task id: " + taskId); + } // Step 1: Create and customize a new ConnectContext for this task ConnectContext taskConnectContext = buildConnectContext(); // Set target file size for Iceberg write @@ -161,6 +164,7 @@ public void execute() throws JobException { throw new JobException("Rewrite group execution failed: " + e.getMessage(), e); } finally { isFinished.set(true); + completionLatch.countDown(); } } @@ -172,12 +176,32 @@ public void cancel() throws JobException { } isCanceled.set(true); - if (stmtExecutor != null) { - stmtExecutor.cancel(new Status(TStatusCode.CANCELLED, "rewrite task cancelled")); + StmtExecutor executor = stmtExecutor; + if (executor != null) { + executor.cancel(new Status(TStatusCode.CANCELLED, "rewrite task cancelled")); + } + if (!isStarted.get()) { + // A task removed from the transient queue will never enter execute(). + completionLatch.countDown(); } LOG.info("[Rewrite Task] taskId: {} cancelled", taskId); } + void awaitCompletionUninterruptibly() { + boolean interrupted = false; + while (true) { + try { + completionLatch.await(); + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + /** * Execute rewrite group with task-specific logical plan and parsed statement */ @@ -186,6 +210,11 @@ private void executeGroup(ConnectContext taskConnectContext, StatementBase taskParsedStmt) throws Exception { // Step 1: Create stmt executor stmtExecutor = new StmtExecutor(taskConnectContext, taskParsedStmt); + if (isCanceled.get()) { + // Recheck after publishing the executor to close the cancel-before-assignment race. + stmtExecutor.cancel(new Status(TStatusCode.CANCELLED, "rewrite task cancelled")); + throw new JobException("Rewrite task has been canceled, task id: " + taskId); + } // Step 2: Create insert executor AbstractInsertExecutor insertExecutor = taskLogicalPlan.initPlan(taskConnectContext, stmtExecutor); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 94399c98575031..baef37472e8272 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -22,8 +22,12 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MapType; +import org.apache.doris.catalog.StructField; +import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; @@ -337,7 +341,7 @@ private boolean requiresIcebergScanSemanticsV2() throws UserException { return true; } return schemaHistoryRequiresMissingRequiredFieldRejection( - scanSchema, projectedFieldIds, icebergTable.schemas().values()); + scanSchema, projectedFieldIds, reachableSchemas(icebergTable, snapshot)); } private boolean hasApplicableEqualityDeletes(TableScan scan) throws UserException { @@ -386,20 +390,67 @@ private static boolean hasSmoothUpgradeSource(Iterable backends) { } private Set projectedFieldIds(Schema scanSchema) { + return projectedFieldIds(scanSchema, desc.getSlots()); + } + + @VisibleForTesting + static Set projectedFieldIds(Schema scanSchema, Iterable slots) { Set projected = new HashSet<>(); - for (SlotDescriptor slot : desc.getSlots()) { + for (SlotDescriptor slot : slots) { int fieldId = slot.getColumn().getUniqueId(); // Stable Iceberg IDs prevent a dropped-and-readded name from selecting the wrong history. NestedField field = fieldId >= 0 ? scanSchema.findField(fieldId) : scanSchema.caseInsensitiveFindField(slot.getColumn().getName()); if (field != null) { - projected.addAll(TypeUtil.indexById( - org.apache.iceberg.types.Types.StructType.of(field)).keySet()); + collectProjectedFieldIds(field, slot.getType(), projected); } } return projected; } + private static void collectProjectedFieldIds( + NestedField field, org.apache.doris.catalog.Type projectedType, + Set projected) { + projected.add(field.fieldId()); + if (projectedType instanceof StructType && field.type().isStructType()) { + for (StructField projectedChild : ((StructType) projectedType).getFields()) { + NestedField icebergChild = field.type().asStructType().fields().stream() + .filter(child -> child.name().equalsIgnoreCase(projectedChild.getName())) + .findFirst().orElse(null); + if (icebergChild != null) { + collectProjectedFieldIds(icebergChild, projectedChild.getType(), projected); + } + } + } else if (projectedType instanceof ArrayType && field.type().isListType()) { + collectProjectedFieldIds(field.type().asListType().fields().get(0), + ((ArrayType) projectedType).getItemType(), projected); + } else if (projectedType instanceof MapType && field.type().isMapType()) { + collectProjectedFieldIds(field.type().asMapType().fields().get(0), + ((MapType) projectedType).getKeyType(), projected); + collectProjectedFieldIds(field.type().asMapType().fields().get(1), + ((MapType) projectedType).getValueType(), projected); + } + } + + @VisibleForTesting + static Iterable reachableSchemas(Table table, Snapshot selectedSnapshot) { + Map schemas = table.schemas(); + List reachable = new ArrayList<>(); + Set visitedSnapshots = new HashSet<>(); + Set visitedSchemaIds = new HashSet<>(); + Snapshot snapshot = selectedSnapshot; + while (snapshot != null && visitedSnapshots.add(snapshot.snapshotId())) { + Schema schema = schemas.get(snapshot.schemaId()); + if (schema != null && visitedSchemaIds.add(snapshot.schemaId())) { + reachable.add(schema); + } + Long parentId = snapshot.parentId(); + snapshot = parentId == null ? null : table.snapshot(parentId); + } + // Only ancestors of the selected ref can have produced files visible to this scan. + return reachable; + } + @VisibleForTesting static void checkIcebergScanSemanticsV2Compatibility( boolean requiresV2, Iterable backends) throws UserException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java index e297ba478e46bf..ab237b85d84409 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java @@ -18,10 +18,13 @@ package org.apache.doris.datasource.iceberg.rewrite; import org.apache.doris.catalog.Env; +import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergTransaction; +import org.apache.doris.transaction.TransactionManager; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.InOrder; import org.mockito.MockedStatic; @@ -33,18 +36,44 @@ class RewriteDataFileExecutorTest { void testInvalidateTableCacheAfterCommit() throws Exception { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); IcebergTransaction transaction = Mockito.mock(IcebergTransaction.class); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + TransactionManager transactionManager = Mockito.mock(TransactionManager.class); Env env = Mockito.mock(Env.class); ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getTransactionManager()).thenReturn(transactionManager); try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { mockedEnv.when(Env::getCurrentEnv).thenReturn(env); Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); - new RewriteDataFileExecutor(table, null).commitAndInvalidate(transaction); + new RewriteDataFileExecutor(table, null).commitAndInvalidate(transactionManager, 7L); - InOrder inOrder = Mockito.inOrder(transaction, cacheMgr); - inOrder.verify(transaction).commit(); + InOrder inOrder = Mockito.inOrder(transactionManager, cacheMgr); + inOrder.verify(transactionManager).commit(7L); inOrder.verify(cacheMgr).invalidateTableCache(table); + Mockito.verify(transaction, Mockito.never()).commit(); } } + + @Test + void testRollbackThroughManagerWhenSnapshotLoadFails() throws Exception { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + IcebergTransaction transaction = Mockito.mock(IcebergTransaction.class); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + TransactionManager transactionManager = Mockito.mock(TransactionManager.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getTransactionManager()).thenReturn(transactionManager); + Mockito.when(transactionManager.begin()).thenReturn(7L); + Mockito.when(transactionManager.getTransaction(7L)).thenReturn(transaction); + Mockito.when(table.loadSnapshot(Mockito.any(), Mockito.any())) + .thenThrow(new IllegalStateException("injected snapshot failure")); + + Assertions.assertThrows(IllegalStateException.class, + () -> new RewriteDataFileExecutor(table, null) + .executeGroupsConcurrently(java.util.Collections.emptyList(), 1024)); + + Mockito.verify(transactionManager).rollback(7L); + Mockito.verify(transactionManager, Mockito.never()).commit(7L); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 72539d190cc114..ae571118aa6ae0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -97,6 +97,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; public class IcebergScanNodeTest { @@ -1362,6 +1363,48 @@ public void testRequirednessHistoryTriggersCurrentScanSemantics() { current, ImmutableList.of(historicalRequired))); } + @Test + public void testReachableSchemasExcludeLaterAndUnrelatedHistory() { + Schema selectedSchema = new Schema(1, ImmutableList.of( + Types.NestedField.required(1, "id", Types.LongType.get()))); + Schema unrelatedSchema = new Schema(2, ImmutableList.of( + Types.NestedField.optional(1, "id", Types.LongType.get()))); + Snapshot selectedSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(selectedSnapshot.snapshotId()).thenReturn(10L); + Mockito.when(selectedSnapshot.schemaId()).thenReturn(1); + Mockito.when(selectedSnapshot.parentId()).thenReturn(null); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(ImmutableMap.of( + 1, selectedSchema, 2, unrelatedSchema)); + + List reachable = new ArrayList<>(); + IcebergScanNode.reachableSchemas(table, selectedSnapshot).forEach(reachable::add); + + Assert.assertEquals(ImmutableList.of(selectedSchema), reachable); + Assert.assertFalse(IcebergScanNode.schemaHistoryRequiresMissingRequiredFieldRejection( + selectedSchema, reachable)); + } + + @Test + public void testProjectedFieldIdsExcludePrunedSibling() { + Schema schema = new Schema(ImmutableList.of(Types.NestedField.required( + 1, "payload", Types.StructType.of( + Types.NestedField.required(2, "keep", Types.StringType.get()), + Types.NestedField.required(3, "added", Types.IntegerType.get()))))); + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Column column = new Column("payload", new StructType( + new StructField("keep", Type.STRING), new StructField("added", Type.INT))); + column.setUniqueId(1); + SlotDescriptor slot = node.addSlot(1, column); + slot.setType(new StructType(new StructField("keep", Type.STRING))); + + Set projected = IcebergScanNode.projectedFieldIds( + schema, ImmutableList.of(slot)); + + Assert.assertEquals(ImmutableList.of(1, 2), projected.stream().sorted() + .collect(java.util.stream.Collectors.toList())); + } + @Test public void testBatchVariantProjectionUsesSharedCompatibilityGate() throws Exception { TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable(), false, true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommandTest.java index 6011b25ff0e7a5..82825822f48158 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommandTest.java @@ -116,6 +116,10 @@ public void testMergeSinkChecksVariantWriteCapability() { // Satisfy the branch-4.1 sink invariants so the assertion exercises Variant write validation. org.apache.iceberg.Table icebergTable = Mockito.mock(org.apache.iceberg.Table.class); + Mockito.when(icebergTable.schema()).thenReturn(new org.apache.iceberg.Schema( + org.apache.iceberg.types.Types.NestedField.optional( + 1, "payload", org.apache.iceberg.types.Types.VariantType.get()))); + Mockito.when(icebergTable.spec()).thenReturn(org.apache.iceberg.PartitionSpec.unpartitioned()); Mockito.when(icebergTable.properties()).thenReturn(ImmutableMap.of("format-version", "2", "write.format.default", "parquet")); Assertions.assertThrows(AnalysisException.class, () -> new LogicalIcebergMergeSink<>(