From a3bf75bdc741d449443a5256b25c982738de73ff Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 16 Aug 2026 17:17:04 +0800 Subject: [PATCH 1/4] [improvement](external) Refine Parquet splits by row group in BE --- be/src/exec/operator/file_scan_operator.cpp | 51 ++- be/src/exec/operator/file_scan_operator.h | 1 + be/src/exec/scan/file_scanner_v2.cpp | 163 +++++++- be/src/exec/scan/file_scanner_v2.h | 39 ++ be/src/exec/scan/split_source_connector.cpp | 107 +++++ be/src/exec/scan/split_source_connector.h | 22 ++ be/src/format_v2/file_reader.h | 14 + be/src/format_v2/file_scan_context.cpp | 93 +++++ be/src/format_v2/file_scan_context.h | 109 +++++ .../parquet/parquet_file_context.cpp | 91 ++++- .../format_v2/parquet/parquet_file_context.h | 28 +- be/src/format_v2/parquet/parquet_reader.cpp | 85 +++- be/src/format_v2/parquet/parquet_reader.h | 12 +- be/src/format_v2/parquet/parquet_scan.cpp | 48 ++- be/src/format_v2/parquet/parquet_scan.h | 9 +- be/src/format_v2/table/hudi_reader.cpp | 8 + be/src/format_v2/table/hudi_reader.h | 2 + be/src/format_v2/table/paimon_reader.cpp | 8 + be/src/format_v2/table/paimon_reader.h | 2 + be/src/format_v2/table_reader.cpp | 41 +- be/src/format_v2/table_reader.h | 12 + be/test/exec/scan/file_scanner_v2_test.cpp | 371 ++++++++++++++++++ .../format_v2/jni/jni_table_reader_test.cpp | 6 + .../format_v2/parquet/parquet_reader_test.cpp | 109 ++++- .../format_v2/parquet/parquet_scan_test.cpp | 20 + be/test/format_v2/table/lance_reader_test.cpp | 2 + be/test/format_v2/table_reader_test.cpp | 68 ++++ 27 files changed, 1452 insertions(+), 69 deletions(-) create mode 100644 be/src/format_v2/file_scan_context.cpp create mode 100644 be/src/format_v2/file_scan_context.h diff --git a/be/src/exec/operator/file_scan_operator.cpp b/be/src/exec/operator/file_scan_operator.cpp index 66ee94ee48e4f3..34deecab13d56b 100644 --- a/be/src/exec/operator/file_scan_operator.cpp +++ b/be/src/exec/operator/file_scan_operator.cpp @@ -258,9 +258,54 @@ void FileScanLocalState::set_scan_ranges(RuntimeState* state, if (_split_source == nullptr) { _split_source = std::make_shared(scan_ranges, _max_scanners); } - // currently the total number of splits in the bach split mode cannot be accurately obtained, - // so we don't do it in the batch split mode. - _max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges()); + // A single FE Parquet split can publish many row-group children after its footer is read. + // Keep the requested scanner concurrency in that case; capping it to the initial range + // count would leave the generated children serial even though they share one footer. + bool can_generate_parquet_splits = false; + const TFileScanRangeParams* common_params = nullptr; + if (state->get_query_ctx() != nullptr && + state->get_query_ctx()->file_scan_range_params_map.contains(parent_id())) { + common_params = &state->get_query_ctx()->file_scan_range_params_map[parent_id()]; + } + for (const auto& scan_range_params : scan_ranges) { + const auto& file_scan_range = + scan_range_params.scan_range.ext_scan_range.file_scan_range; + const auto* params = + file_scan_range.__isset.params ? &file_scan_range.params : common_params; + if (params == nullptr) { + continue; + } + const bool is_load = + state->desc_tbl().get_tuple_descriptor(params->src_tuple_id) != nullptr; + if (!_should_use_file_scanner_v2(state->query_options(), is_load, *params)) { + continue; + } + can_generate_parquet_splits = + std::ranges::any_of(file_scan_range.ranges, [&](const auto& range) { + const auto format = + range.__isset.format_type ? range.format_type : params->format_type; + if (format == TFileFormatType::FORMAT_PARQUET) { + return true; + } + if (format != TFileFormatType::FORMAT_JNI || + !range.__isset.table_format_params || + range.table_format_params.table_format_type != "paimon" || + !range.table_format_params.__isset.paimon_params) { + return false; + } + const auto& paimon = range.table_format_params.paimon_params; + return paimon.__isset.file_format && paimon.file_format == "parquet" && + !paimon.__isset.paimon_split; + }); + if (can_generate_parquet_splits) { + break; + } + } + // Currently the total number of remote splits cannot be accurately obtained, so batch + // mode already skips this cap. + if (!can_generate_parquet_splits) { + _max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges()); + } } if (!scan_ranges.empty() && diff --git a/be/src/exec/operator/file_scan_operator.h b/be/src/exec/operator/file_scan_operator.h index 2aaeebe049cf64..91407faa50e262 100644 --- a/be/src/exec/operator/file_scan_operator.h +++ b/be/src/exec/operator/file_scan_operator.h @@ -94,6 +94,7 @@ class FileScanLocalState final : public ScanLocalState { // 2. parquet file meta // KVCache _kv_cache; std::unique_ptr _kv_cache; + FileContextRegistry _file_context_registry; TupleId _output_tuple_id = -1; RuntimeProfile::Counter* _condition_cache_hit_counter = nullptr; RuntimeProfile::Counter* _condition_cache_filtered_rows_counter = nullptr; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index be17a9920e6ac6..be2843d6474f5e 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -281,6 +281,27 @@ Status adapt_runtime_filter_for_table_reader(VExprSPtr* expr) { } // namespace +int64_t FileScannerV2::_cumulative_profile_delta(int64_t current, int64_t* reported) { + DORIS_CHECK(reported != nullptr); + DORIS_CHECK(current >= *reported); + const int64_t delta = current - *reported; + *reported = current; + return delta; +} + +FileScannerV2::FileReaderProfileDeltas FileScannerV2::_collect_file_reader_profile_deltas( + const io::FileReaderStats& stats, int64_t* reported_bytes, int64_t* reported_calls, + int64_t* reported_time) { + return { + .read_bytes = + _cumulative_profile_delta(cast_set(stats.read_bytes), reported_bytes), + .read_calls = + _cumulative_profile_delta(cast_set(stats.read_calls), reported_calls), + .read_time_ns = + _cumulative_profile_delta(cast_set(stats.read_time_ns), reported_time), + }; +} + #ifdef BE_TEST FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile, std::unique_ptr table_reader) @@ -380,7 +401,8 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat const std::unordered_map* colname_to_slot_id) : Scanner(state, local_state, limit, profile), _split_source(std::move(split_source)), - _kv_cache(kv_cache) { + _kv_cache(kv_cache), + _file_context_registry(&local_state->_file_context_registry) { (void)colname_to_slot_id; if (state->get_query_ctx() != nullptr && state->get_query_ctx()->file_scan_range_params_map.count(local_state->parent_id()) > 0) { @@ -448,6 +470,11 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { DORIS_CHECK(_table_reader != nullptr); RETURN_IF_ERROR(_init_expr_ctxes()); RETURN_IF_ERROR(_init_table_reader(_current_range)); + // Refine the first source split before yielding the scanner worker. Other scanners may be + // waiting for its row-group children, so deferring publication until a later get_block() + // turn could let those waiters occupy the scan thread pool ahead of the producer. + bool eos = false; + RETURN_IF_ERROR(_prepare_next_split(&eos)); } return Status::OK(); } @@ -455,9 +482,38 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { Status FileScannerV2::_get_next_scan_range(bool* has_next) { SCOPED_TIMER(_get_next_range_timer); DORIS_CHECK(has_next != nullptr); - RETURN_IF_ERROR(_split_source->get_next(has_next, &_current_range)); + RETURN_IF_ERROR(_split_source->get_next_split(has_next, &_current_split)); if (*has_next) { + _current_range = _current_split.materialize_range(); RETURN_IF_ERROR(_validate_scan_range(*_params, _current_range)); + } else { + _current_split = {}; + _current_range = {}; + } + return Status::OK(); +} + +Status FileScannerV2::_retire_current_source_split(std::vector generated_splits) { + if (!_current_split.is_source_split) { + return Status::OK(); + } + const bool transferred_progress = !generated_splits.empty(); + RETURN_IF_ERROR( + _split_source->finish_source_split(_current_split, std::move(generated_splits))); + _current_split.is_source_split = false; + if (transferred_progress) { + _current_split.source_progress.reset(); + } + return Status::OK(); +} + +Status FileScannerV2::_complete_current_split() { + RETURN_IF_ERROR(_retire_current_source_split()); + if (_current_split.source_progress != nullptr) { + if (_current_split.source_progress->complete_one()) { + _state->update_num_finished_scan_range(1); + } + _current_split.source_progress.reset(); } return Status::OK(); } @@ -488,7 +544,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e if (_should_skip_not_found(status, config::ignore_not_found_file_in_external_table)) { RETURN_IF_ERROR(_table_reader->abort_split()); COUNTER_UPDATE(_not_found_file_counter, 1); - _state->update_num_finished_scan_range(1); + RETURN_IF_ERROR(_complete_current_split()); _has_prepared_split = false; block->clear_column_data(cast_set(_projected_columns.size())); *eof = false; @@ -502,7 +558,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e // discard partial reader state, and let the loop fetch the next split. RETURN_IF_ERROR(_table_reader->abort_split()); COUNTER_UPDATE(_empty_file_counter, 1); - _state->update_num_finished_scan_range(1); + RETURN_IF_ERROR(_complete_current_split()); _has_prepared_split = false; block->clear_column_data(cast_set(_projected_columns.size())); *eof = false; @@ -511,7 +567,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e RETURN_IF_ERROR(status); } if (*eof) { - _state->update_num_finished_scan_range(1); + RETURN_IF_ERROR(_complete_current_split()); _has_prepared_split = false; *eof = false; continue; @@ -552,6 +608,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { } _first_scan_range = false; if (!has_next || _should_stop) { + RETURN_IF_ERROR(_complete_current_split()); *eos = true; return Status::OK(); } @@ -575,7 +632,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { if (_should_skip_not_found(status, config::ignore_not_found_file_in_external_table)) { RETURN_IF_ERROR(_table_reader->abort_split()); COUNTER_UPDATE(_not_found_file_counter, 1); - _state->update_num_finished_scan_range(1); + RETURN_IF_ERROR(_complete_current_split()); continue; } if (_should_skip_empty(status, _should_stop || _io_ctx->should_stop)) { @@ -585,14 +642,51 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { // advance exactly one scan range and preserve later files in the same scan. RETURN_IF_ERROR(_table_reader->abort_split()); COUNTER_UPDATE(_empty_file_counter, 1); - _state->update_num_finished_scan_range(1); + RETURN_IF_ERROR(_complete_current_split()); continue; } RETURN_IF_ERROR(status); if (_table_reader->current_split_pruned()) { - _state->update_num_finished_scan_range(1); + RETURN_IF_ERROR(_complete_current_split()); continue; } + if (_current_split.is_source_split && _can_refine_source_split(_current_range)) { + std::vector generated_splits; + bool was_split = false; + const auto split_status = _table_reader->build_physical_splits( + _current_split, &generated_splits, &was_split); + const auto ignored_split_status = _classify_ignored_split_status( + split_status, config::ignore_not_found_file_in_external_table, + _should_stop || _io_ctx->should_stop); + if (ignored_split_status == IgnoredSplitStatus::NOT_FOUND) { + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_not_found_file_counter, 1); + RETURN_IF_ERROR(_complete_current_split()); + continue; + } + if (ignored_split_status == IgnoredSplitStatus::EMPTY) { + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_empty_file_counter, 1); + RETURN_IF_ERROR(_complete_current_split()); + continue; + } + RETURN_IF_ERROR(split_status); + if (was_split) { + RETURN_IF_ERROR(_table_reader->abort_split()); + const bool has_children = !generated_splits.empty(); + RETURN_IF_ERROR(_retire_current_source_split(std::move(generated_splits))); + if (!has_children) { + RETURN_IF_ERROR(_complete_current_split()); + } + continue; + } + } + if (_current_split.is_source_split) { + // Non-Parquet and metadata-only source splits are already fully prepared and cannot + // publish children. Release the source reservation before yielding the scanner worker + // so waiters never depend on a later get_block() turn to observe completion. + RETURN_IF_ERROR(_retire_current_source_split()); + } COUNTER_UPDATE(_file_counter, 1); _has_prepared_split = true; _table_reader_rf_num = _applied_rf_num; @@ -636,6 +730,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .push_down_agg_type = _local_state->get_push_down_agg_type(), .push_down_count_columns = std::move(push_down_count_columns), .condition_cache_digest = _local_state->get_condition_cache_digest(), + .file_context_registry = _file_context_registry, })); return Status::OK(); } @@ -709,6 +804,8 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range, .cache = _kv_cache, .current_range = range, .current_split_format = current_split_format, + .file_context = _current_split.file_context, + .format_split_id = _current_split.format_split_id, .global_rowid_context = _create_global_rowid_context(range), })); return Status::OK(); @@ -726,6 +823,27 @@ bool FileScannerV2::_should_skip_empty(const Status& status, bool stopped) { return !stopped && status.is(); } +FileScannerV2::IgnoredSplitStatus FileScannerV2::_classify_ignored_split_status( + const Status& status, bool ignore_not_found, bool stopped) { + if (_should_skip_not_found(status, ignore_not_found)) { + return IgnoredSplitStatus::NOT_FOUND; + } + if (_should_skip_empty(status, stopped)) { + return IgnoredSplitStatus::EMPTY; + } + return IgnoredSplitStatus::NONE; +} + +bool FileScannerV2::_can_refine_source_split(const TFileRangeDesc& range) { + if (!range.__isset.table_format_params || !range.table_format_params.__isset.iceberg_params) { + return true; + } + const auto& iceberg = range.table_format_params.iceberg_params; + // Iceberg delete readers build split-local delete state. Refining the data file would rebuild + // that state for every row-group child, so keep the already prepared source reader instead. + return !iceberg.__isset.delete_files || iceberg.delete_files.empty(); +} + bool FileScannerV2::_should_enable_file_meta_cache() const { return ExecEnv::GetInstance()->file_meta_cache()->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; @@ -1053,16 +1171,24 @@ Status FileScannerV2::close(RuntimeState* state) { if (_table_reader != nullptr) { const auto close_status = _table_reader->close(); if (!close_status.ok()) { + // A failed reader close remains retryable, but this scanner can no longer publish + // useful children. Retire its source reservation so peer scanners are not stranded. + const auto split_status = _complete_current_split(); + if (!split_status.ok()) { + return split_status; + } return close_status; } _report_condition_cache_profile(); _table_reader.reset(); } + RETURN_IF_ERROR(_complete_current_split()); return Scanner::close(state); } void FileScannerV2::try_stop() { Scanner::try_stop(); + _split_source->stop(); if (_io_ctx) { _io_ctx->should_stop = true; } @@ -1073,7 +1199,6 @@ void FileScannerV2::update_realtime_counters() { return; } DORIS_CHECK(_file_cache_statistics != nullptr); - const int64_t bytes_read = cast_set(_file_reader_stats->read_bytes); auto* local_state = static_cast(_local_state); const auto file_type = _current_range.__isset.file_type @@ -1095,9 +1220,14 @@ void FileScannerV2::update_realtime_counters() { _state->get_query_ctx()->resource_ctx()->io_context()->update_scan_bytes_from_remote_storage( deltas.scan_bytes_from_remote_storage); - COUNTER_SET(_file_read_bytes_counter, bytes_read); - COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); - COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + // Scanner instances share these profile counters. Publish only scanner-private deltas so one + // scanner cannot erase or double count values already contributed by a sibling. + const auto profile_deltas = _collect_file_reader_profile_deltas( + *_file_reader_stats, &_reported_file_read_bytes, &_reported_file_read_calls, + &_reported_file_read_time); + COUNTER_UPDATE(_file_read_bytes_counter, profile_deltas.read_bytes); + COUNTER_UPDATE(_file_read_calls_counter, profile_deltas.read_calls); + COUNTER_UPDATE(_file_read_time_counter, profile_deltas.read_time_ns); DorisMetrics::instance()->query_scan_bytes->increment(deltas.scan_bytes); DorisMetrics::instance()->query_scan_rows->increment(deltas.scan_rows); @@ -1192,9 +1322,12 @@ void FileScannerV2::_collect_profile_before_close() { _reported_file_cache_statistics = *_file_cache_statistics; } if (_file_reader_stats != nullptr) { - COUNTER_SET(_file_read_bytes_counter, cast_set(_file_reader_stats->read_bytes)); - COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); - COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + const auto profile_deltas = _collect_file_reader_profile_deltas( + *_file_reader_stats, &_reported_file_read_bytes, &_reported_file_read_calls, + &_reported_file_read_time); + COUNTER_UPDATE(_file_read_bytes_counter, profile_deltas.read_bytes); + COUNTER_UPDATE(_file_read_calls_counter, profile_deltas.read_calls); + COUNTER_UPDATE(_file_read_time_counter, profile_deltas.read_time_ns); const auto read_time = cast_set(_file_reader_stats->read_time_ns); DORIS_CHECK(read_time >= _reported_io_read_time); // Some transports (for example Arrow Flight) record directly into IO, while filesystem diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index fbf630731ad889..a19439edd28472 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -61,6 +61,14 @@ class FileScannerV2 final : public Scanner { int64_t scan_bytes_from_remote_storage = 0; }; + struct FileReaderProfileDeltas { + int64_t read_bytes = 0; + int64_t read_calls = 0; + int64_t read_time_ns = 0; + }; + + enum class IgnoredSplitStatus { NONE, NOT_FOUND, EMPTY }; + enum class UncachedReaderBytesStorage { LOCAL, REMOTE, NONE }; static bool is_supported(const TFileScanRangeParams& params, const TFileRangeDesc& range); @@ -87,6 +95,23 @@ class FileScannerV2 final : public Scanner { int64_t* last_bytes_read_from_remote); static void TEST_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); + static int64_t TEST_cumulative_profile_delta(int64_t current, int64_t* reported) { + return _cumulative_profile_delta(current, reported); + } + static FileReaderProfileDeltas TEST_collect_file_reader_profile_deltas( + const io::FileReaderStats& stats, int64_t* reported_bytes, int64_t* reported_calls, + int64_t* reported_time) { + return _collect_file_reader_profile_deltas(stats, reported_bytes, reported_calls, + reported_time); + } + static IgnoredSplitStatus TEST_classify_ignored_split_status(const Status& status, + bool ignore_not_found, + bool stopped) { + return _classify_ignored_split_status(status, ignore_not_found, stopped); + } + static bool TEST_can_refine_source_split(const TFileRangeDesc& range) { + return _can_refine_source_split(range); + } static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); static bool TEST_should_skip_empty(const Status& status, bool stopped); static Status TEST_contextualize_output_filter_status(Status status, @@ -124,6 +149,8 @@ class FileScannerV2 final : public Scanner { static Status _validate_scan_range(const TFileScanRangeParams& params, const TFileRangeDesc& range); Status _get_next_scan_range(bool* has_next); + Status _retire_current_source_split(std::vector generated_splits = {}); + Status _complete_current_split(); TFileFormatType::type _get_current_format_type() const; Status _init_io_ctx(); Status _init_expr_ctxes(); @@ -135,8 +162,15 @@ class FileScannerV2 final : public Scanner { std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); static bool _should_skip_empty(const Status& status, bool stopped); + static IgnoredSplitStatus _classify_ignored_split_status(const Status& status, + bool ignore_not_found, bool stopped); + static bool _can_refine_source_split(const TFileRangeDesc& range); static Status _contextualize_output_filter_status(Status status, TFileFormatType::type format_type); + static int64_t _cumulative_profile_delta(int64_t current, int64_t* reported); + static FileReaderProfileDeltas _collect_file_reader_profile_deltas( + const io::FileReaderStats& stats, int64_t* reported_bytes, int64_t* reported_calls, + int64_t* reported_time); bool _should_enable_file_meta_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; @@ -181,6 +215,7 @@ class FileScannerV2 final : public Scanner { bool _has_prepared_split = false; int _table_reader_rf_num = 0; TFileRangeDesc _current_range; + FileScanSplit _current_split; std::string _current_range_path; std::unique_ptr _table_reader; @@ -200,6 +235,7 @@ class FileScannerV2 final : public Scanner { std::unique_ptr _file_reader_stats; std::shared_ptr _io_ctx; ShardedKVCache* _kv_cache = nullptr; + FileContextRegistry* _file_context_registry = nullptr; RuntimeProfile::Counter* _scanner_total_timer = nullptr; RuntimeProfile::Counter* _init_timer = nullptr; @@ -227,6 +263,9 @@ class FileScannerV2 final : public Scanner { int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; int64_t _reported_io_read_time = 0; + int64_t _reported_file_read_bytes = 0; + int64_t _reported_file_read_calls = 0; + int64_t _reported_file_read_time = 0; }; } // namespace doris diff --git a/be/src/exec/scan/split_source_connector.cpp b/be/src/exec/scan/split_source_connector.cpp index 685a2d50f0c1a3..b27e39e065aa5f 100644 --- a/be/src/exec/scan/split_source_connector.cpp +++ b/be/src/exec/scan/split_source_connector.cpp @@ -24,6 +24,113 @@ namespace doris { using apache::thrift::transport::TTransportException; +Status SplitSourceConnector::get_next_split(bool* has_next, FileScanSplit* split) { + DORIS_CHECK(has_next != nullptr); + DORIS_CHECK(split != nullptr); + std::unique_lock lock(_split_lock); + while (true) { + *has_next = false; + if (_stopped) { + return Status::OK(); + } + if (!_generated_splits.empty()) { + *split = std::move(_generated_splits.front()); + _generated_splits.pop_front(); + *has_next = true; + return Status::OK(); + } + if (_source_exhausted) { + if (_active_source_splits.empty()) { + return Status::OK(); + } + _split_ready.wait(lock); + continue; + } + if (_source_claim_in_progress) { + _split_ready.wait(lock); + continue; + } + + TFileRangeDesc range; + bool has_source = false; + // Record the in-flight claim before dropping the queue lock, so another scanner cannot + // report raw EOS before this claim becomes an active source. The potentially blocking + // remote RPC must not hold the queue lock because a footer producer may publish children + // while that fetch is in flight. + _source_claim_in_progress = true; + lock.unlock(); + const auto source_status = get_next(&has_source, &range); + lock.lock(); + _source_claim_in_progress = false; + if (source_status.ok() && !has_source) { + // Both local exhaustion and an empty final remote batch are terminal. Remember EOS so + // parent waiters do not wake each other into repeated empty source fetches. + _source_exhausted = true; + } + _split_ready.notify_all(); + RETURN_IF_ERROR(source_status); + if (_stopped) { + return Status::OK(); + } + if (has_source) { + // A scanner reuses its output envelope across files. Reset child-only shared range and + // context state before installing a raw FE range, or materialization can read the + // previous child's file instead of this source. + *split = {}; + split->range = std::move(range); + split->is_source_split = true; + split->source_split_id = _next_source_split_id++; + split->source_progress = std::make_shared(); + _active_source_splits.insert(split->source_split_id); + *has_next = true; + return Status::OK(); + } + if (_active_source_splits.empty()) { + return Status::OK(); + } + _split_ready.wait(lock); + } +} + +Status SplitSourceConnector::finish_source_split(const FileScanSplit& source_split, + std::vector generated_splits) { + if (!source_split.is_source_split || source_split.source_split_id == 0) { + return Status::InvalidArgument("Only an active source split can publish generated splits"); + } + { + std::lock_guard lock(_split_lock); + if (_active_source_splits.erase(source_split.source_split_id) != 1) { + return Status::InvalidArgument("Source split {} is not active", + source_split.source_split_id); + } + if (!_stopped) { + if (!generated_splits.empty()) { + DORIS_CHECK(source_split.source_progress != nullptr); + source_split.source_progress->reset_for_children(generated_splits.size()); + } + for (auto& split : generated_splits) { + split.is_source_split = false; + split.source_split_id = 0; + split.source_progress = source_split.source_progress; + _generated_splits.push_back(std::move(split)); + } + } + } + _split_ready.notify_all(); + return Status::OK(); +} + +void SplitSourceConnector::stop() { + { + std::lock_guard lock(_split_lock); + _stopped = true; + // Cancellation must release queued descriptors and footer contexts immediately; source + // producers finishing later will only retire their reservation without publishing work. + _generated_splits.clear(); + } + _split_ready.notify_all(); +} + Status LocalSplitSourceConnector::get_next(bool* has_next, TFileRangeDesc* range) { std::lock_guard l(_range_lock); *has_next = false; diff --git a/be/src/exec/scan/split_source_connector.h b/be/src/exec/scan/split_source_connector.h index 6d914ef4a1da4d..583d6b9eebe69a 100644 --- a/be/src/exec/scan/split_source_connector.h +++ b/be/src/exec/scan/split_source_connector.h @@ -18,9 +18,13 @@ #pragma once #include +#include +#include +#include #include "common/config.h" #include "core/custom_allocator.h" +#include "format_v2/file_scan_context.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" @@ -44,6 +48,14 @@ class SplitSourceConnector { */ virtual Status get_next(bool* has_next, TFileRangeDesc* range) = 0; + // FileScannerV2 may replace one FE split with several physical children after inspecting file + // metadata. These methods are shared by local and remote sources so neither source can report + // EOS while a claimed source split is still producing children. + Status get_next_split(bool* has_next, FileScanSplit* split); + Status finish_source_split(const FileScanSplit& source_split, + std::vector generated_splits); + void stop(); + virtual int num_scan_ranges() = 0; virtual TFileScanRangeParams* get_params() = 0; @@ -96,6 +108,16 @@ class SplitSourceConnector { protected: int _max_scanners; + +private: + std::mutex _split_lock; + std::condition_variable _split_ready; + std::deque _generated_splits; + std::unordered_set _active_source_splits; + uint64_t _next_source_split_id = 1; + bool _source_claim_in_progress = false; + bool _source_exhausted = false; + bool _stopped = false; }; /** diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index b9635b2f6f0a22..0c105bd0afcb9d 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -31,6 +31,7 @@ #include "core/field.h" #include "exprs/vexpr_fwd.h" #include "format_v2/column_data.h" +#include "format_v2/file_scan_context.h" #include "gen_cpp/PlanNodes_types.h" #include "io/file_factory.h" #include "io/fs/file_reader_writer_fwd.h" @@ -347,6 +348,19 @@ class FileReader { // Initialize file reader and parse file metadata. virtual Status init(RuntimeState* state); + // Optionally refine one scheduler split into format-specific physical children after metadata + // initialization. The default keeps non-columnar formats on their original split. + virtual Status build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, + bool* was_split) const { + (void)source_split; + DORIS_CHECK(splits != nullptr); + DORIS_CHECK(was_split != nullptr); + splits->clear(); + *was_split = false; + return Status::OK(); + } + // Set the maximum row count for the next physical read batch. Readers that do not batch by // rows may ignore it. virtual void set_batch_size(size_t batch_size) { (void)batch_size; } diff --git a/be/src/format_v2/file_scan_context.cpp b/be/src/format_v2/file_scan_context.cpp new file mode 100644 index 00000000000000..fc6ea94ff17214 --- /dev/null +++ b/be/src/format_v2/file_scan_context.cpp @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/file_scan_context.h" + +#include +#include + +namespace doris { + +Status FileContextRegistry::get_or_create(const std::string& key, const Loader& loader, + std::shared_ptr* context) { + DORIS_CHECK(context != nullptr); + context->reset(); + while (true) { + std::shared_ptr entry; + bool load = false; + { + std::lock_guard registry_lock(_lock); + auto it = _entries.find(key); + if (it == _entries.end()) { + entry = std::make_shared(); + _entries.emplace(key, entry); + load = true; + } else { + entry = it->second; + std::lock_guard entry_lock(entry->lock); + if (!entry->loading && entry->status.ok() && entry->context.expired()) { + // Weak values keep a long-running scan from retaining every remote footer it + // has ever seen. Replace only an inactive entry so concurrent users of the + // same file still share one single-flight load. + entry = std::make_shared(); + it->second = entry; + load = true; + } + } + } + + if (load) { + std::shared_ptr loaded_context; + Status status; + try { + status = loader(&loaded_context); + } catch (const std::exception& e) { + status = Status::InternalError("File context loader failed: {}", e.what()); + } catch (...) { + status = Status::InternalError("File context loader failed with an unknown error"); + } + if (status.ok() && loaded_context == nullptr) { + status = Status::InternalError("File context loader returned a null context"); + } + if (status.ok()) { + *context = loaded_context; + } + { + std::lock_guard lock(entry->lock); + entry->status = status; + entry->context = loaded_context; + entry->loading = false; + } + entry->ready.notify_all(); + return status; + } + + std::unique_lock lock(entry->lock); + entry->ready.wait(lock, [&]() { return !entry->loading; }); + if (!entry->status.ok()) { + return entry->status; + } + *context = entry->context.lock(); + if (*context != nullptr) { + return Status::OK(); + } + // The loading caller may already have released its result. Retry so this caller installs + // a fresh single-flight entry instead of returning a null context. + } +} + +} // namespace doris diff --git a/be/src/format_v2/file_scan_context.h b/be/src/format_v2/file_scan_context.h new file mode 100644 index 00000000000000..a3719cb81cc078 --- /dev/null +++ b/be/src/format_v2/file_scan_context.h @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { + +// Opaque immutable metadata shared by all physical splits of one file. Concrete file formats own +// the derived type so the scanner and split-source layers do not depend on format internals. +class FileContext { +public: + virtual ~FileContext() = default; +}; + +class FileContextRegistry { +public: + using Loader = std::function*)>; + + Status get_or_create(const std::string& key, const Loader& loader, + std::shared_ptr* context); + +private: + struct Entry { + std::mutex lock; + std::condition_variable ready; + bool loading = true; + Status status; + std::weak_ptr context; + }; + + std::mutex _lock; + std::unordered_map> _entries; +}; + +// Tracks completion of one FE source range after BE refinement. Generated children share this +// object, so the FE range is reported finished only when its final physical child completes. +class SourceSplitProgress { +public: + void reset_for_children(size_t children) { + DORIS_CHECK(children > 0); + _remaining.store(children, std::memory_order_release); + } + + bool complete_one() { + const size_t previous = _remaining.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(previous > 0); + return previous == 1; + } + +private: + std::atomic _remaining {1}; +}; + +// BE-local scheduling envelope. It deliberately carries no Thrift fields: FE ranges remain the +// source of table-format semantics, while generated physical children add only opaque metadata and +// a format-local subrange id. +struct FileScanSplit { + TFileRangeDesc range; + std::shared_ptr source_range; + int64_t start_offset = 0; + int64_t size = -1; + bool clear_table_level_row_count = false; + std::shared_ptr file_context; + int64_t format_split_id = -1; + bool is_source_split = false; + uint64_t source_split_id = 0; + std::shared_ptr source_progress; + + TFileRangeDesc materialize_range() const { + auto materialized = source_range == nullptr ? range : *source_range; + if (source_range != nullptr) { + materialized.__set_start_offset(start_offset); + materialized.__set_size(size); + } + if (clear_table_level_row_count && materialized.__isset.table_format_params) { + materialized.table_format_params.__isset.table_level_row_count = false; + } + return materialized; + } +}; + +} // namespace doris diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 8ba8cf94662f9a..0e994767b18c00 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -47,7 +47,9 @@ namespace doris::format::parquet { constexpr size_t V2_PARQUET_FOOTER_SIZE = 8; NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size) - : _metadata(std::move(metadata)), _parsed_size(parsed_size) { + : _metadata(std::move(metadata)), + _row_group_first_rows(_metadata.row_groups.size()), + _parsed_size(parsed_size) { ExecEnv::GetInstance()->parquet_meta_tracker()->consume(get_mem_size()); } @@ -63,12 +65,20 @@ Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, // Native readers address projected leaves by stable DFS IDs. Assign them only on the private // v2 schema object so v1's cached schema lifecycle and numbering remain untouched. _schema.assign_ids(); + int64_t next_first_row = 0; for (size_t row_group_idx = 0; row_group_idx < _metadata.row_groups.size(); ++row_group_idx) { const auto& row_group = _metadata.row_groups[row_group_idx]; if (row_group.num_rows < 0) { return Status::Corruption("Parquet row group {} has negative row count {}", row_group_idx, row_group.num_rows); } + if (row_group.num_rows > std::numeric_limits::max() - next_first_row) { + return Status::Corruption("Parquet row counts overflow at row group {}", row_group_idx); + } + // Generated children share this immutable prefix through the footer context. Recomputing + // all prefixes in every exact-row-group child would turn R-way refinement into O(R^2). + _row_group_first_rows[row_group_idx] = next_first_row; + next_first_row += row_group.num_rows; if (row_group.columns.size() != _schema.physical_fields_size()) { // All v2 planners index chunks by the native DFS leaf order, so validate cardinality // once before any projection, prefetch, or decoder can perform indexed access. @@ -271,7 +281,9 @@ std::string build_page_cache_file_key(const io::FileReader& file_reader, Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, const io::FileDescription& file_description, - bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) { + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary, + FileContextRegistry* file_context_registry, + std::shared_ptr file_context) { DORIS_CHECK(input_file_reader != nullptr); contains_variant = false; if (detail::should_stage_small_http_file(input_file_reader->path().native(), @@ -299,24 +311,66 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont meta_cache_key.push_back(static_cast(enable_mapping_varbinary)); meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); } - size_t native_footer_size = 0; - if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled() && - meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { - native_metadata = native_meta_cache_handle.data(); - ++native_footer_cache_hits; - } else { - RETURN_IF_ERROR(parse_native_parquet_footer( - native_file, &native_metadata_owner, &native_footer_size, io_ctx, - enable_mapping_varbinary, enable_mapping_timestamp_tz)); - ++native_footer_read_calls; - if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled()) { - meta_cache->insert(meta_cache_key, native_metadata_owner.release(), - &native_meta_cache_handle); - native_metadata = native_meta_cache_handle.data(); + // The registry is scoped to one scan instance, whose splits describe one planned snapshot. + // Normalize optional FE identity fields with reader values so equivalent splits cannot miss + // single-flight merely because only one of them carried file size or mtime. + const int64_t registry_mtime = + file_description.mtime != 0 ? file_description.mtime : native_file->mtime(); + const int64_t registry_file_size = file_description.file_size >= 0 + ? file_description.file_size + : cast_set(native_file->size()); + const auto registry_path = native_file->path().native(); + const std::string registry_key = fmt::format( + "fs[{}]={}::path[{}]={}::mtime={}::size={}::immutable={}::varbinary={}::timestamp_tz={" + "}", + file_description.fs_name.size(), file_description.fs_name, registry_path.size(), + registry_path, registry_mtime, registry_file_size, file_description.is_immutable, + enable_mapping_varbinary, enable_mapping_timestamp_tz); + auto load_context = [&](std::shared_ptr* result) -> Status { + auto loaded = std::make_shared(); + loaded->registry_key = registry_key; + loaded->has_stable_identity = has_stable_meta_cache_identity; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled() && + meta_cache->lookup(meta_cache_key, &loaded->metadata_cache_handle)) { + loaded->metadata = loaded->metadata_cache_handle.data(); + ++native_footer_cache_hits; + } else { + size_t native_footer_size = 0; + RETURN_IF_ERROR(parse_native_parquet_footer( + native_file, &loaded->metadata_owner, &native_footer_size, io_ctx, + enable_mapping_varbinary, enable_mapping_timestamp_tz)); + ++native_footer_read_calls; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled()) { + meta_cache->insert(meta_cache_key, loaded->metadata_owner.release(), + &loaded->metadata_cache_handle); + loaded->metadata = loaded->metadata_cache_handle.data(); + } else { + loaded->metadata = loaded->metadata_owner.get(); + } + } + DORIS_CHECK(loaded->metadata != nullptr); + *result = std::move(loaded); + return Status::OK(); + }; + + std::shared_ptr resolved_context = std::move(file_context); + if (resolved_context == nullptr) { + if (file_context_registry != nullptr && has_stable_meta_cache_identity) { + RETURN_IF_ERROR(file_context_registry->get_or_create(registry_key, load_context, + &resolved_context)); } else { - native_metadata = native_metadata_owner.get(); + RETURN_IF_ERROR(load_context(&resolved_context)); } } + shared_file_context = + std::dynamic_pointer_cast(resolved_context); + if (shared_file_context == nullptr) { + return Status::InvalidArgument("Parquet split has an incompatible file context"); + } + if (shared_file_context->registry_key != registry_key) { + return Status::InvalidArgument("Parquet split file context does not match file identity"); + } + native_metadata = shared_file_context->metadata; DORIS_CHECK(native_metadata != nullptr); auto page_cache_file_key = build_page_cache_file_key(*native_file, file_description); @@ -597,8 +651,7 @@ Status ParquetFileContext::close() { } native_row_group_file.reset(); native_metadata = nullptr; - native_metadata_owner.reset(); - native_meta_cache_handle = {}; + shared_file_context.reset(); native_file.reset(); native_io_ctx = nullptr; native_page_cache_enabled = false; diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 38e78438e39b29..a73f2b8b89225f 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -27,6 +27,7 @@ #include #include "common/status.h" +#include "format_v2/file_scan_context.h" #include "format_v2/parquet/native_schema_desc.h" #include "io/fs/file_reader.h" #include "util/obj_lru_cache.h" @@ -54,14 +55,28 @@ class NativeParquetMetadata { Status init_schema(bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); const tparquet::FileMetaData& to_thrift() const { return _metadata; } const NativeFieldDescriptor& schema() const { return _schema; } - size_t get_mem_size() const { return _parsed_size; } + const std::vector& row_group_first_rows() const { return _row_group_first_rows; } + size_t get_mem_size() const { + return _parsed_size + _row_group_first_rows.capacity() * sizeof(int64_t); + } private: tparquet::FileMetaData _metadata; NativeFieldDescriptor _schema; + std::vector _row_group_first_rows; size_t _parsed_size = 0; }; +// The registry shares only immutable footer metadata. Every ParquetReader still owns its file and +// row-group state, which avoids sharing mutable page-index and merge-reader state across scanners. +struct ParquetSharedFileContext final : public FileContext { + std::string registry_key; + bool has_stable_identity = false; + const NativeParquetMetadata* metadata = nullptr; + std::unique_ptr metadata_owner; + ObjLRUCache::CacheHandle metadata_cache_handle; +}; + struct ParquetPageCacheRange { int64_t offset = 0; int64_t size = 0; @@ -130,11 +145,10 @@ struct ParquetFileContext { // large chunks and in-memory files keep native_file. io::FileReaderSPtr native_row_group_file; io::IOContext* native_io_ctx = nullptr; - // V2-owned Thrift footer/schema used to construct native page/encoding readers. A cache hit is - // owned by native_meta_cache_handle; a miss without cache is owned by native_metadata_owner. + // V2-owned Thrift footer/schema used to construct native page/encoding readers. The shared + // registry context owns either the parsed value or its process-cache handle. const NativeParquetMetadata* native_metadata = nullptr; - std::unique_ptr native_metadata_owner; - ObjLRUCache::CacheHandle native_meta_cache_handle; + std::shared_ptr shared_file_context; int64_t native_footer_read_calls = 0; int64_t native_footer_cache_hits = 0; bool native_page_cache_enabled = false; @@ -145,7 +159,9 @@ struct ParquetFileContext { Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, const io::FileDescription& file_description, - bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false, + FileContextRegistry* file_context_registry = nullptr, + std::shared_ptr file_context = nullptr); Status load_native_offset_indexes( int row_group_id, const std::unordered_set& leaf_column_ids, std::unordered_map* offset_indexes) const; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 86a1da92ca993e..90422f16d7cd34 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -18,6 +18,7 @@ #include "format_v2/parquet/parquet_reader.h" #include +#include #include #include #include @@ -41,6 +42,7 @@ #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" #include "format_v2/parquet/reader/count_column_reader.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -478,11 +480,17 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary, + FileContextRegistry* file_context_registry, + std::shared_ptr file_context, + int64_t format_split_id) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), - _enable_mapping_varbinary(enable_mapping_varbinary) {} + _enable_mapping_varbinary(enable_mapping_varbinary), + _file_context_registry(file_context_registry), + _file_context(std::move(file_context)), + _format_split_id(format_split_id) {} ParquetReader::~ParquetReader() = default; @@ -523,7 +531,8 @@ Status ParquetReader::init(RuntimeState* state) { SCOPED_TIMER(_parquet_profile.parse_footer_time); RETURN_IF_ERROR(_state->file_context.open( _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, - _enable_mapping_timestamp_tz, _enable_mapping_varbinary)); + _enable_mapping_timestamp_tz, _enable_mapping_varbinary, _file_context_registry, + _file_context)); } if (_profile != nullptr) { COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, @@ -551,6 +560,75 @@ Status ParquetReader::init(RuntimeState* state) { return Status::OK(); } +Status ParquetReader::build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, + bool* was_split) const { + DORIS_CHECK(splits != nullptr); + DORIS_CHECK(was_split != nullptr); + splits->clear(); + *was_split = false; + if (_state == nullptr || _state->file_context.native_metadata == nullptr || + _state->file_context.shared_file_context == nullptr) { + return Status::Uninitialized("ParquetReader is not open"); + } + if (!_state->file_context.shared_file_context->has_stable_identity) { + // A path and size do not identify a mutable remote object. Keep the initialized parent + // reader instead of publishing children whose shared footer could become stale. + return Status::OK(); + } + + ParquetScanRange scan_range { + .start_offset = + source_split.range.__isset.start_offset ? source_split.range.start_offset : 0, + .size = source_split.range.__isset.size ? source_split.range.size : -1, + .file_size = source_split.range.__isset.file_size ? source_split.range.file_size + : _file_description->file_size, + }; + std::vector selected_row_groups; + RETURN_IF_ERROR(detail::select_native_row_groups_by_scan_range( + _state->file_context.native_metadata->to_thrift(), scan_range, + _state->file_context.native_metadata->row_group_first_rows(), &selected_row_groups)); + const auto& metadata = _state->file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + const size_t file_size = _state->file_context.native_file->size(); + auto shared_source_range = std::make_shared(source_split.range); + splits->reserve(selected_row_groups.size()); + for (const int row_group_id : selected_row_groups) { + const auto& row_group = metadata.row_groups[row_group_id]; + size_t group_start = std::numeric_limits::max(); + size_t group_end = 0; + for (size_t column_id = 0; column_id < row_group.columns.size(); ++column_id) { + const auto& chunk = row_group.columns[column_id]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_id, column_id); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, file_size, compat.parquet_816_padding, &chunk_range)); + group_start = std::min(group_start, chunk_range.offset); + group_end = std::max(group_end, chunk_range.offset + chunk_range.length); + } + if (group_end <= group_start) { + return Status::Corruption("Parquet row group {} has an empty physical byte range", + row_group_id); + } + FileScanSplit child; + child.source_range = shared_source_range; + child.start_offset = cast_set(group_start); + child.size = cast_set(group_end - group_start); + // A source-level count is not valid for one generated row group. Child readers can still + // derive an exact count from the shared footer when aggregate pushdown is eligible. + child.clear_table_level_row_count = true; + child.file_context = _state->file_context.shared_file_context; + child.format_split_id = row_group_id; + splits->push_back(std::move(child)); + } + *was_split = true; + return Status::OK(); +} + void ParquetReader::set_batch_size(size_t batch_size) { _batch_size = std::max(1, batch_size); if (_state != nullptr) { @@ -668,6 +746,7 @@ Status ParquetReader::open(std::shared_ptr request) { scan_range.start_offset = _file_description->range_start_offset; scan_range.size = _file_description->range_size; scan_range.file_size = _file_description->file_size; + scan_range.row_group_id = _format_split_id; // Get selected ranges in row groups according to metadata (Row-Group level index and Page Index including Zonemap, Dictionary, Bloom Filter). RETURN_IF_ERROR(plan_parquet_row_groups( *_state->file_context.native_metadata, _state->file_schema, *request_snapshot, diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index fe95b93a9e0101..b10d43828195fd 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -46,11 +46,18 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false, + FileContextRegistry* file_context_registry = nullptr, + std::shared_ptr file_context = nullptr, + int64_t format_split_id = -1); ~ParquetReader() override; Status init(RuntimeState* state) override; + Status build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, + bool* was_split) const override; + void set_batch_size(size_t batch_size) override; Status get_schema(std::vector* file_schema) const override; @@ -94,6 +101,9 @@ class ParquetReader : public format::FileReader { size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ bool _enable_mapping_varbinary = false; // whether raw BYTE_ARRAY is mapped to VARBINARY + FileContextRegistry* _file_context_registry = nullptr; + std::shared_ptr _file_context; + int64_t _format_split_id = -1; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index b5db28fd0edff2..f64d1609c11340 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -592,12 +592,42 @@ Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& meta std::vector* row_group_first_rows, std::vector* selected_row_groups) { DORIS_CHECK(row_group_first_rows != nullptr && selected_row_groups != nullptr); - if (scan_range.start_offset < 0 || scan_range.size < -1 || + row_group_first_rows->assign(metadata.row_groups.size(), 0); + int64_t next_first_row = 0; + for (size_t row_group_idx = 0; row_group_idx < metadata.row_groups.size(); ++row_group_idx) { + (*row_group_first_rows)[row_group_idx] = next_first_row; + const auto row_group_rows = metadata.row_groups[row_group_idx].num_rows; + if (row_group_rows < 0) { + return Status::Corruption("Invalid negative row count in parquet row group {}", + row_group_idx); + } + if (row_group_rows > std::numeric_limits::max() - next_first_row) { + return Status::Corruption("Parquet row counts overflow at row group {}", row_group_idx); + } + next_first_row += row_group_rows; + } + return select_native_row_groups_by_scan_range(metadata, scan_range, *row_group_first_rows, + selected_row_groups); +} + +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + const std::vector& row_group_first_rows, + std::vector* selected_row_groups) { + DORIS_CHECK(selected_row_groups != nullptr); + DORIS_CHECK(row_group_first_rows.size() == metadata.row_groups.size()); + if (scan_range.start_offset < 0 || scan_range.size < -1 || scan_range.row_group_id < -1 || + scan_range.row_group_id >= cast_set(metadata.row_groups.size()) || (scan_range.size >= 0 && scan_range.start_offset > std::numeric_limits::max() - scan_range.size)) { return Status::Corruption("Invalid Parquet scan range [{}, {})", scan_range.start_offset, scan_range.size); } + selected_row_groups->clear(); + if (scan_range.row_group_id >= 0) { + selected_row_groups->push_back(cast_set(scan_range.row_group_id)); + return Status::OK(); + } const uint64_t range_start = static_cast(scan_range.start_offset); const uint64_t range_end = scan_range.size < 0 ? std::numeric_limits::max() @@ -609,21 +639,9 @@ Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& meta range_end >= static_cast(scan_range.file_size)); const auto compat = native::parquet_reader_compat( metadata.__isset.created_by ? metadata.created_by : std::string {}); - row_group_first_rows->assign(metadata.row_groups.size(), 0); - selected_row_groups->clear(); selected_row_groups->reserve(metadata.row_groups.size()); - int64_t next_first_row = 0; for (size_t row_group_idx = 0; row_group_idx < metadata.row_groups.size(); ++row_group_idx) { - (*row_group_first_rows)[row_group_idx] = next_first_row; const auto& row_group = metadata.row_groups[row_group_idx]; - if (row_group.num_rows < 0) { - return Status::Corruption("Invalid negative row count in parquet row group {}", - row_group_idx); - } - if (row_group.num_rows > std::numeric_limits::max() - next_first_row) { - return Status::Corruption("Parquet row counts overflow at row group {}", row_group_idx); - } - next_first_row += row_group.num_rows; bool selected = full_file_range; if (!full_file_range) { if (row_group.columns.empty()) { @@ -781,10 +799,10 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, plan->pruning_stats = {}; plan->requested_leaf_column_ids = request_leaf_column_ids(file_schema, request); plan->enable_bloom_filter = enable_bloom_filter; - std::vector row_group_first_rows; + const auto& row_group_first_rows = metadata.row_group_first_rows(); std::vector scan_range_selected; RETURN_IF_ERROR(detail::select_native_row_groups_by_scan_range( - metadata.to_thrift(), scan_range, &row_group_first_rows, &scan_range_selected)); + metadata.to_thrift(), scan_range, row_group_first_rows, &scan_range_selected)); RETURN_IF_ERROR(build_native_row_group_read_plans(metadata, file_schema, request, scan_range_selected, row_group_first_rows, plan, timezone, runtime_state, file_context)); diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 06e0ada0caf1d1..c264cd4b3f8d87 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -105,6 +105,10 @@ Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& meta const ParquetScanRange& scan_range, std::vector* row_group_first_rows, std::vector* selected_row_groups); +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + const std::vector& row_group_first_rows, + std::vector* selected_row_groups); #ifdef BE_TEST void reset_physical_leaf_set_build_count(); size_t physical_leaf_set_build_count(); @@ -116,8 +120,9 @@ size_t physical_leaf_set_build_count(); struct ParquetScanRange { int64_t start_offset = 0; - int64_t size = -1; // -1 means read the whole file - int64_t file_size = -1; // -1 means unknown + int64_t size = -1; // -1 means read the whole file + int64_t file_size = -1; // -1 means unknown + int64_t row_group_id = -1; // BE-generated splits select one row group without Thrift changes }; struct RowGroupReadPlan { diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 838d98b5e52e14..429591f803369a 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -80,6 +80,13 @@ Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) return _current_split_reader->prepare_split(options); } +Status HudiHybridReader::build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, + bool* was_split) { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->build_physical_splits(source_split, splits, was_split); +} + Status HudiHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { RETURN_IF_ERROR(format::TableReader::refresh_conjuncts(std::move(conjuncts))); if (_current_split_reader == nullptr) { @@ -196,6 +203,7 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, .push_down_agg_type = _push_down_agg_type, .push_down_count_columns = _push_down_count_columns, .condition_cache_digest = _condition_cache_digest, + .file_context_registry = _file_context_registry, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal // runtime default until FileScannerV2 supplies the first positive prediction. diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index c06e1b238b62c4..f98eb766c65c3f 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -59,6 +59,8 @@ class HudiHybridReader final : public format::TableReader { Status init(format::TableReadOptions&& options) override; Status prepare_split(const format::SplitReadOptions& options) override; + Status build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, bool* was_split) override; Status refresh_conjuncts(VExprContextSPtrs conjuncts) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 7c7a3fa7f3a9be..6cf4bc1910ea7c 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -279,6 +279,13 @@ Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options return _current_split_reader->prepare_split(options); } +Status PaimonHybridReader::build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, + bool* was_split) { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->build_physical_splits(source_split, splits, was_split); +} + Status PaimonHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { RETURN_IF_ERROR(format::TableReader::refresh_conjuncts(std::move(conjuncts))); if (_current_split_reader == nullptr) { @@ -401,6 +408,7 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, .push_down_agg_type = _push_down_agg_type, .push_down_count_columns = _push_down_count_columns, .condition_cache_digest = _condition_cache_digest, + .file_context_registry = _file_context_registry, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal // runtime default until FileScannerV2 supplies the first positive prediction. diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index ed2b9e75c1c722..de28e2af39c470 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -74,6 +74,8 @@ class PaimonHybridReader final : public format::TableReader { Status init(format::TableReadOptions&& options) override; Status prepare_split(const format::SplitReadOptions& options) override; + Status build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, bool* was_split) override; Status refresh_conjuncts(VExprContextSPtrs conjuncts) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index bd7fe744110fdf..05cdde601b539c 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -766,6 +766,7 @@ Status TableReader::init(TableReadOptions&& options) { _push_down_count_columns = options.push_down_count_columns; _initial_condition_cache_digest = options.condition_cache_digest; _condition_cache_digest = _initial_condition_cache_digest; + _file_context_registry = options.file_context_registry; _projected_columns = std::move(options.projected_columns); if (supports_iceberg_scan_semantics_v1(_scan_params)) { for (auto& projected_column : _projected_columns) { @@ -1108,7 +1109,9 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { // match the table type. *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); + _global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary, + _file_context_registry, _current_task->file_context, + _current_task->format_split_id); return Status::OK(); } if (_format == FileFormat::ORC) { @@ -1225,6 +1228,8 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { } _current_task = std::make_unique(); _current_task->data_file = create_file_description(options.current_range); + _current_task->file_context = options.file_context; + _current_task->format_split_id = options.format_split_id; _current_file_description = *_current_task->data_file; // A table-level row count is only equivalent to scanning the split when no row predicate is // active and no predicate can arrive later. The metadata path can return several batches for @@ -1249,6 +1254,40 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { return _parse_delete_predicates(options); } +Status TableReader::build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, bool* was_split) { + DORIS_CHECK(splits != nullptr); + DORIS_CHECK(was_split != nullptr); + splits->clear(); + *was_split = false; + if (_format != FileFormat::PARQUET || _current_split_pruned || + _current_split_uses_metadata_count || _current_task == nullptr) { + return Status::OK(); + } + + std::unique_ptr reader; + RETURN_IF_ERROR(create_file_reader(&reader)); + DORIS_CHECK(reader != nullptr); + RETURN_IF_ERROR(reader->init(_runtime_state)); + const auto status = reader->build_physical_splits(source_split, splits, was_split); + if (!status.ok()) { + static_cast(reader->close()); + return status; + } + if (!*was_split || splits->size() == 1) { + // Reuse the planning reader when refinement is unnecessary. This avoids parsing the same + // footer again for an unsplit file or for the common single-row-group case. + splits->clear(); + *was_split = false; + _data_reader.reader = std::move(reader); + if (_batch_size > 0) { + _data_reader.reader->set_batch_size(_batch_size); + } + return open_reader(); + } + return reader->close(); +} + Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, bool* can_filter_all) { DORIS_CHECK(can_filter_all != nullptr); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 3ae2077ecbc53d..d80c5a7928e531 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -60,6 +60,7 @@ #include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" +#include "format_v2/file_scan_context.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/schema_projection.h" #include "gen_cpp/PlanNodes_types.h" @@ -89,6 +90,8 @@ struct ScanTask { virtual ~ScanTask() = default; std::unique_ptr data_file; + std::shared_ptr file_context; + int64_t format_split_id = -1; }; struct ProjectedColumnBuildContext { @@ -158,6 +161,7 @@ struct TableReadOptions { // with SplitReadOptions::condition_cache_digest after collecting late-arrival runtime filters. // A zero digest disables condition cache. uint64_t condition_cache_digest = 0; + FileContextRegistry* file_context_registry = nullptr; }; struct SplitReadOptions { @@ -182,6 +186,8 @@ struct SplitReadOptions { ShardedKVCache* cache = nullptr; TFileRangeDesc current_range; FileFormat current_split_format = FileFormat::PARQUET; + std::shared_ptr file_context; + int64_t format_split_id = -1; std::optional global_rowid_context; }; @@ -236,6 +242,11 @@ class TableReader { return _current_split_uses_metadata_count; } + // Refine a prepared source split into format-specific physical children. The default + // implementation currently handles native Parquet; wrappers dispatch to their active child. + virtual Status build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, bool* was_split); + // Discard the active split after the caller decides an error is ignorable, for example a // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start // with no concrete reader or split-local state left from the failed split. @@ -1924,6 +1935,7 @@ class TableReader { RuntimeProfile* _scanner_profile; const std::vector* _file_slot_descs = nullptr; FileFormat _format; + FileContextRegistry* _file_context_registry = nullptr; TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; std::optional> _push_down_count_columns; size_t _batch_size = 0; diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 6d10ab156cfe50..8e12e6d9211497 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -20,9 +20,16 @@ #include #include +#include +#include +#include +#include +#include #include +#include #include #include +#include #include #include #include @@ -48,6 +55,7 @@ #include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" #include "format_v2/expr/cast.h" +#include "format_v2/file_scan_context.h" #include "testutil/mock/mock_runtime_state.h" namespace doris { @@ -492,6 +500,306 @@ TEST(FileScannerV2Test, LegacyCountExemptionRequiresMetadataCountOnEveryRange) { EXPECT_FALSE(invalid.all_ranges_have_table_level_row_count()); } +TScanRangeParams scan_range_with_path(std::string path) { + TScanRangeParams params; + TFileRangeDesc range; + range.__set_path(std::move(path)); + params.scan_range.ext_scan_range.file_scan_range.ranges.push_back(std::move(range)); + return params; +} + +class RemoteStyleSplitSourceConnector final : public SplitSourceConnector { +public: + explicit RemoteStyleSplitSourceConnector(std::vector ranges) + : _ranges(std::move(ranges)) {} + + Status get_next(bool* has_next, TFileRangeDesc* range) override { + std::lock_guard lock(_lock); + ++_get_next_calls; + *has_next = _next < _ranges.size(); + if (*has_next) { + *range = _ranges[_next++]; + } + return Status::OK(); + } + + int num_scan_ranges() override { return cast_set(_ranges.size()); } + + TFileScanRangeParams* get_params() override { return &_params; } + + int get_next_calls() const { return _get_next_calls.load(); } + +private: + std::mutex _lock; + std::vector _ranges; + size_t _next = 0; + std::atomic _get_next_calls = 0; + TFileScanRangeParams _params; +}; + +class BlockingRemoteStyleSplitSourceConnector final : public SplitSourceConnector { +public: + explicit BlockingRemoteStyleSplitSourceConnector(TFileRangeDesc range) + : _range(std::move(range)) {} + + Status get_next(bool* has_next, TFileRangeDesc* range) override { + std::unique_lock lock(_lock); + if (!_source_returned) { + _source_returned = true; + *has_next = true; + *range = _range; + return Status::OK(); + } + _fetch_started = true; + _fetch_cv.notify_all(); + _fetch_cv.wait(lock, [&]() { return _release_fetch; }); + *has_next = false; + return Status::OK(); + } + + void wait_for_fetch() { + std::unique_lock lock(_lock); + _fetch_cv.wait(lock, [&]() { return _fetch_started; }); + } + + void release_fetch() { + { + std::lock_guard lock(_lock); + _release_fetch = true; + } + _fetch_cv.notify_all(); + } + + int num_scan_ranges() override { return 1; } + + TFileScanRangeParams* get_params() override { return &_params; } + +private: + std::mutex _lock; + std::condition_variable _fetch_cv; + TFileRangeDesc _range; + bool _source_returned = false; + bool _fetch_started = false; + bool _release_fetch = false; + TFileScanRangeParams _params; +}; + +void expect_generated_splits_keep_source_alive(SplitSourceConnector* connector) { + FileScanSplit source; + bool has_next = false; + ASSERT_TRUE(connector->get_next_split(&has_next, &source).ok()); + ASSERT_TRUE(has_next); + ASSERT_TRUE(source.is_source_split); + + auto waiter = std::async(std::launch::async, [&]() { + FileScanSplit split; + bool waiter_has_next = false; + auto status = connector->get_next_split(&waiter_has_next, &split); + return std::make_tuple(status, waiter_has_next, std::move(split)); + }); + EXPECT_EQ(waiter.wait_for(std::chrono::milliseconds(50)), std::future_status::timeout); + + FileScanSplit child; + child.range = source.range; + child.range.__set_start_offset(101); + child.range.__set_size(17); + ASSERT_TRUE(connector->finish_source_split(source, {child}).ok()); + + ASSERT_EQ(waiter.wait_for(std::chrono::seconds(2)), std::future_status::ready); + auto [status, waiter_has_next, generated] = waiter.get(); + ASSERT_TRUE(status.ok()) << status; + ASSERT_TRUE(waiter_has_next); + EXPECT_FALSE(generated.is_source_split); + EXPECT_EQ(generated.range.start_offset, 101); + EXPECT_EQ(generated.range.size, 17); + + FileScanSplit end; + ASSERT_TRUE(connector->get_next_split(&has_next, &end).ok()); + EXPECT_FALSE(has_next); +} + +TEST(FileScannerV2Test, LocalSplitSourceWaitsForGeneratedRowGroupSplits) { + LocalSplitSourceConnector connector({scan_range_with_path("local.parquet")}, 4); + expect_generated_splits_keep_source_alive(&connector); +} + +TEST(FileScannerV2Test, RemoteStyleSplitSourceWaitsForGeneratedRowGroupSplits) { + TFileRangeDesc range; + range.__set_path("remote.parquet"); + RemoteStyleSplitSourceConnector connector({range}); + expect_generated_splits_keep_source_alive(&connector); + EXPECT_EQ(connector.get_next_calls(), 2); +} + +TEST(FileScannerV2Test, RemoteFetchDoesNotBlockGeneratedSplitPublication) { + TFileRangeDesc range; + range.__set_path("remote.parquet"); + BlockingRemoteStyleSplitSourceConnector connector(std::move(range)); + FileScanSplit source; + bool has_next = false; + ASSERT_TRUE(connector.get_next_split(&has_next, &source).ok()); + ASSERT_TRUE(has_next); + + auto remote_fetch = std::async(std::launch::async, [&]() { + FileScanSplit split; + bool fetch_has_next = false; + auto status = connector.get_next_split(&fetch_has_next, &split); + return std::make_pair(status, fetch_has_next); + }); + connector.wait_for_fetch(); + auto child_waiter = std::async(std::launch::async, [&]() { + FileScanSplit split; + bool child_available = false; + auto status = connector.get_next_split(&child_available, &split); + return std::make_tuple(status, child_available, std::move(split)); + }); + EXPECT_EQ(child_waiter.wait_for(std::chrono::milliseconds(50)), std::future_status::timeout); + + FileScanSplit child; + child.range = source.range; + child.range.__set_start_offset(101); + auto publish = std::async(std::launch::async, + [&]() { return connector.finish_source_split(source, {child}); }); + const bool publish_completed_while_fetch_blocked = + publish.wait_for(std::chrono::seconds(2)) == std::future_status::ready; + const bool child_published_while_fetch_blocked = + child_waiter.wait_for(std::chrono::seconds(2)) == std::future_status::ready; + connector.release_fetch(); + + const auto publish_status = publish.get(); + ASSERT_TRUE(publish_status.ok()) << publish_status; + ASSERT_TRUE(publish_completed_while_fetch_blocked); + ASSERT_TRUE(child_published_while_fetch_blocked); + auto [child_status, child_available, generated] = child_waiter.get(); + ASSERT_TRUE(child_status.ok()) << child_status; + ASSERT_TRUE(child_available); + EXPECT_EQ(generated.range.start_offset, 101); + auto [fetch_status, fetch_has_next] = remote_fetch.get(); + ASSERT_TRUE(fetch_status.ok()) << fetch_status; + EXPECT_FALSE(fetch_has_next); +} + +TEST(FileScannerV2Test, RawSourceClearsGeneratedChildEnvelope) { + LocalSplitSourceConnector connector({scan_range_with_path("next-local.parquet")}, 1); + FileScanSplit reused; + auto stale_source_range = std::make_shared(); + stale_source_range->__set_path("stale-child.parquet"); + reused.source_range = std::move(stale_source_range); + reused.start_offset = 101; + reused.size = 17; + reused.clear_table_level_row_count = true; + reused.file_context = std::make_shared(); + reused.format_split_id = 3; + + bool has_next = false; + ASSERT_TRUE(connector.get_next_split(&has_next, &reused).ok()); + ASSERT_TRUE(has_next); + EXPECT_TRUE(reused.is_source_split); + EXPECT_EQ(reused.materialize_range().path, "next-local.parquet"); + EXPECT_EQ(reused.source_range, nullptr); + EXPECT_EQ(reused.file_context, nullptr); + EXPECT_EQ(reused.format_split_id, -1); + ASSERT_TRUE(connector.finish_source_split(reused, {}).ok()); +} + +TEST(FileScannerV2Test, GeneratedChildrenCompleteSourceProgressOnlyOnce) { + LocalSplitSourceConnector connector({scan_range_with_path("two-groups.parquet")}, 1); + FileScanSplit source; + bool has_next = false; + ASSERT_TRUE(connector.get_next_split(&has_next, &source).ok()); + ASSERT_TRUE(has_next); + ASSERT_NE(source.source_progress, nullptr); + + FileScanSplit first_child; + first_child.range = source.range; + first_child.range.__set_start_offset(0); + FileScanSplit second_child; + second_child.range = source.range; + second_child.range.__set_start_offset(100); + ASSERT_TRUE(connector.finish_source_split(source, {first_child, second_child}).ok()); + + FileScanSplit first; + ASSERT_TRUE(connector.get_next_split(&has_next, &first).ok()); + ASSERT_TRUE(has_next); + FileScanSplit second; + ASSERT_TRUE(connector.get_next_split(&has_next, &second).ok()); + ASSERT_TRUE(has_next); + ASSERT_NE(first.source_progress, nullptr); + EXPECT_EQ(first.source_progress, second.source_progress); + EXPECT_FALSE(first.source_progress->complete_one()); + EXPECT_TRUE(second.source_progress->complete_one()); +} + +class TestFileContext final : public FileContext {}; + +TEST(FileScannerV2Test, FileContextRegistryLoadsEachFileOnceConcurrently) { + FileContextRegistry registry; + std::mutex loader_lock; + std::condition_variable loader_cv; + bool release_loader = false; + std::atomic loader_calls = 0; + auto expected = std::make_shared(); + std::latch ready(8); + std::latch start(1); + + auto load = [&]() { + ready.count_down(); + start.wait(); + std::shared_ptr context; + auto status = registry.get_or_create( + "fs::file.parquet::mtime=7::size=11", + [&](std::shared_ptr* result) { + ++loader_calls; + std::unique_lock lock(loader_lock); + loader_cv.wait(lock, [&]() { return release_loader; }); + *result = expected; + return Status::OK(); + }, + &context); + return std::make_pair(status, std::move(context)); + }; + + std::vector>>> futures; + for (int i = 0; i < 8; ++i) { + futures.push_back(std::async(std::launch::async, load)); + } + ready.wait(); + start.count_down(); + while (loader_calls.load() == 0) { + std::this_thread::yield(); + } + EXPECT_EQ(loader_calls.load(), 1); + { + std::lock_guard lock(loader_lock); + release_loader = true; + } + loader_cv.notify_one(); + + for (auto& future : futures) { + auto [status, context] = future.get(); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(context, expected); + } + EXPECT_EQ(loader_calls.load(), 1); +} + +TEST(FileScannerV2Test, FileContextRegistryDoesNotRetainInactiveContexts) { + FileContextRegistry registry; + std::shared_ptr context; + ASSERT_TRUE(registry.get_or_create( + "fs::inactive.parquet::mtime=7::size=11", + [](std::shared_ptr* result) { + *result = std::make_shared(); + return Status::OK(); + }, + &context) + .ok()); + + std::weak_ptr weak_context = context; + context.reset(); + EXPECT_TRUE(weak_context.expired()); +} + TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) { TQueryOptions query_options; query_options.__set_enable_file_scanner_v2(true); @@ -720,6 +1028,69 @@ TEST(FileScannerV2Test, RealtimeCounterDeltasUseReaderBytesAsRemoteWithoutCacheS EXPECT_EQ(60, deltas.scan_bytes_from_remote_storage); } +TEST(FileScannerV2Test, FileReadBytesProfilePublishesOnlyNewScannerDelta) { + int64_t reported = 0; + EXPECT_EQ(FileScannerV2::TEST_cumulative_profile_delta(100, &reported), 100); + EXPECT_EQ(FileScannerV2::TEST_cumulative_profile_delta(150, &reported), 50); + EXPECT_EQ(FileScannerV2::TEST_cumulative_profile_delta(150, &reported), 0); + EXPECT_EQ(reported, 150); +} + +TEST(FileScannerV2Test, FileReaderProfilePublishesOnlyNewScannerDeltas) { + io::FileReaderStats stats; + int64_t reported_bytes = 0; + int64_t reported_calls = 0; + int64_t reported_time = 0; + stats.read_bytes = 100; + stats.read_calls = 10; + stats.read_time_ns = 1000; + auto deltas = FileScannerV2::TEST_collect_file_reader_profile_deltas( + stats, &reported_bytes, &reported_calls, &reported_time); + EXPECT_EQ(deltas.read_bytes, 100); + EXPECT_EQ(deltas.read_calls, 10); + EXPECT_EQ(deltas.read_time_ns, 1000); + + stats.read_bytes = 150; + stats.read_calls = 12; + stats.read_time_ns = 1400; + deltas = FileScannerV2::TEST_collect_file_reader_profile_deltas( + stats, &reported_bytes, &reported_calls, &reported_time); + EXPECT_EQ(deltas.read_bytes, 50); + EXPECT_EQ(deltas.read_calls, 2); + EXPECT_EQ(deltas.read_time_ns, 400); +} + +TEST(FileScannerV2Test, SplitPlanningUsesTheSameIgnoredErrorPolicyAsOpen) { + EXPECT_EQ(FileScannerV2::TEST_classify_ignored_split_status(Status::NotFound("missing"), true, + false), + FileScannerV2::IgnoredSplitStatus::NOT_FOUND); + EXPECT_EQ(FileScannerV2::TEST_classify_ignored_split_status(Status::EndOfFile("empty"), false, + false), + FileScannerV2::IgnoredSplitStatus::EMPTY); + EXPECT_EQ(FileScannerV2::TEST_classify_ignored_split_status(Status::EndOfFile("stopped"), false, + true), + FileScannerV2::IgnoredSplitStatus::NONE); +} + +TEST(FileScannerV2Test, IcebergDeleteFilesDisablePhysicalSplitRefinement) { + TFileRangeDesc range; + EXPECT_TRUE(FileScannerV2::TEST_can_refine_source_split(range)); + + TTableFormatFileDesc table_format; + TIcebergFileDesc iceberg; + iceberg.__set_delete_files({}); + table_format.__set_iceberg_params(iceberg); + range.__set_table_format_params(table_format); + EXPECT_TRUE(FileScannerV2::TEST_can_refine_source_split(range)); + + TIcebergDeleteFileDesc delete_file; + delete_file.__set_path("delete.parquet"); + iceberg.__set_delete_files({delete_file}); + table_format.__set_iceberg_params(iceberg); + range.__set_table_format_params(table_format); + EXPECT_FALSE(FileScannerV2::TEST_can_refine_source_split(range)); +} + TEST(FileScannerV2Test, RealtimeCounterDeltasUseFileCacheDeltasWhenAvailable) { io::FileReaderStats file_reader_stats; io::FileCacheStatistics file_cache_statistics; diff --git a/be/test/format_v2/jni/jni_table_reader_test.cpp b/be/test/format_v2/jni/jni_table_reader_test.cpp index b22d25abea69c0..343ea27f19330c 100644 --- a/be/test/format_v2/jni/jni_table_reader_test.cpp +++ b/be/test/format_v2/jni/jni_table_reader_test.cpp @@ -233,6 +233,8 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { .cache = nullptr, .current_range = {}, .current_split_format = FileFormat::JNI, + .file_context = nullptr, + .format_split_id = -1, .global_rowid_context = std::nullopt, }) .ok()); @@ -253,6 +255,8 @@ TEST(JniTableReaderTest, RefreshedConjunctIsReadyBeforeFilteringOpenScanner) { .cache = nullptr, .current_range = {}, .current_split_format = FileFormat::JNI, + .file_context = nullptr, + .format_split_id = -1, .global_rowid_context = std::nullopt, }) .ok()); @@ -283,6 +287,8 @@ TEST(JniTableReaderTest, CommonLifecycleTimersContainJniLifecycleWork) { .cache = nullptr, .current_range = {}, .current_split_format = FileFormat::JNI, + .file_context = nullptr, + .format_split_id = -1, .global_rowid_context = std::nullopt, }) .ok()); diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 8f463713f9810c..49cc25082abd6a 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -68,6 +68,7 @@ #include "format_v2/column_mapper.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" +#include "format_v2/file_scan_context.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" @@ -1775,7 +1776,10 @@ class NewParquetReaderTest : public testing::Test { std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, bool is_immutable = false, bool enable_mapping_varbinary = false, - std::string fs_name = {}, int64_t mtime = 0) const { + std::string fs_name = {}, int64_t mtime = 0, + FileContextRegistry* file_context_registry = nullptr, + std::shared_ptr file_context = nullptr, + int64_t format_split_id = -1) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); @@ -1788,13 +1792,86 @@ class NewParquetReaderTest : public testing::Test { file_description->mtime = mtime; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); + global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary, + file_context_registry, std::move(file_context), format_split_id); } std::filesystem::path _test_dir; std::string _file_path; }; +TEST_F(NewParquetReaderTest, RowGroupSplitsShareOneRegistryFooterContext) { + write_parquet_file(_file_path, 2); + constexpr int64_t TEST_MTIME = 424242; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + FileContextRegistry registry; + RuntimeProfile parent_profile("row_group_split_parent"); + auto parent = create_reader(0, -1, &parent_profile, false, nullptr, std::nullopt, false, false, + {}, TEST_MTIME, ®istry); + ASSERT_TRUE(parent->init(&state).ok()); + + FileScanSplit source; + source.range.__set_path(_file_path); + source.range.__set_start_offset(0); + source.range.__set_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_file_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_modification_time(TEST_MTIME); + TTableFormatFileDesc table_format; + table_format.__set_table_level_row_count(5); + source.range.__set_table_format_params(table_format); + source.is_source_split = true; + std::vector children; + bool was_split = false; + ASSERT_TRUE(parent->build_physical_splits(source, &children, &was_split).ok()); + ASSERT_TRUE(was_split); + ASSERT_EQ(children.size(), 3); + ASSERT_NE(children[0].file_context, nullptr); + for (size_t index = 0; index < children.size(); ++index) { + EXPECT_EQ(children[index].file_context, children[0].file_context); + EXPECT_EQ(children[index].format_split_id, cast_set(index)); + EXPECT_GT(children[index].size, 0); + EXPECT_FALSE(children[index].is_source_split); + EXPECT_FALSE(children[index] + .materialize_range() + .table_format_params.__isset.table_level_row_count); + } + EXPECT_EQ(parent_profile.get_counter("FileFooterReadCalls")->value(), 1); + + RuntimeProfile sibling_profile("same_file_source_split"); + auto sibling = create_reader(0, -1, &sibling_profile, false, nullptr, std::nullopt, false, + false, {}, TEST_MTIME, ®istry); + ASSERT_TRUE(sibling->init(&state).ok()); + EXPECT_EQ(sibling_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(sibling_profile.get_counter("FileFooterHitCache")->value(), 0); + + RuntimeProfile child_profile("row_group_split_child"); + auto child_range = children[0].materialize_range(); + auto child = create_reader(child_range.start_offset, child_range.size, &child_profile, false, + nullptr, std::nullopt, false, false, {}, TEST_MTIME, ®istry, + children[0].file_context, children[0].format_split_id); + ASSERT_TRUE(child->init(&state).ok()); + EXPECT_EQ(child_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(child_profile.get_counter("FileFooterHitCache")->value(), 0); + + std::vector schema; + ASSERT_TRUE(child->get_schema(&schema).ok()); + auto request = std::make_shared(); + for (size_t index = 0; index < schema.size(); ++index) { + request->non_predicate_columns.push_back(field_projection(cast_set(index))); + } + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(child->open(request).ok()); + size_t total_rows = 0; + bool eof = false; + while (!eof) { + auto block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(child->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, 2); +} + TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -3410,21 +3487,45 @@ TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotReuseMutableUnknownVersion) _file_path = (_test_dir / "mutable_footer_cache.parquet").string(); write_parquet_file(_file_path); RuntimeState state {TQueryOptions(), TQueryGlobals()}; + FileContextRegistry registry; RuntimeProfile first_profile("native_footer_cache_mutable_first"); - auto first = create_reader(0, -1, &first_profile); + auto first = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, false, + {}, 0, ®istry); ASSERT_TRUE(first->init(&state).ok()); EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); write_parquet_file(_file_path); RuntimeProfile second_profile("native_footer_cache_mutable_second"); - auto second = create_reader(0, -1, &second_profile); + auto second = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, false, + {}, 0, ®istry); ASSERT_TRUE(second->init(&state).ok()); EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); } +TEST_F(NewParquetReaderTest, MutableUnknownVersionDeclinesPhysicalSplitRefinement) { + write_parquet_file(_file_path, 2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + FileContextRegistry registry; + auto reader = create_reader(0, -1, nullptr, false, nullptr, std::nullopt, false, false, {}, 0, + ®istry); + ASSERT_TRUE(reader->init(&state).ok()); + + FileScanSplit source; + source.range.__set_path(_file_path); + source.range.__set_start_offset(0); + source.range.__set_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_file_size(static_cast(std::filesystem::file_size(_file_path))); + source.is_source_split = true; + std::vector children; + bool was_split = false; + ASSERT_TRUE(reader->build_physical_splits(source, &children, &was_split).ok()); + EXPECT_FALSE(was_split); + EXPECT_TRUE(children.empty()); +} + TEST_F(NewParquetReaderTest, NativeFooterSizeIsBoundedBeforeMetadataAllocation) { constexpr size_t file_size = 256UL << 20; constexpr size_t metadata_limit = 100UL << 20; diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index ade4905449ec46..91b64be20f8973 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -166,6 +166,26 @@ TEST(ParquetScanMetadataSafetyTest, CheckedChunkRangesDrivePrefetchAndSplitAssig .ok()); } +TEST(ParquetScanMetadataSafetyTest, ExactRowGroupUsesPrecomputedFirstRows) { + tparquet::FileMetaData metadata; + tparquet::RowGroup unrelated_before; + unrelated_before.__set_num_rows(-1); + tparquet::RowGroup selected_group; + selected_group.__set_num_rows(7); + tparquet::RowGroup unrelated_after; + unrelated_after.__set_num_rows(-1); + metadata.__set_row_groups({unrelated_before, selected_group, unrelated_after}); + + const std::vector first_rows {0, 10, 17}; + const format::parquet::ParquetScanRange exact_group { + .start_offset = 0, .size = -1, .file_size = 1, .row_group_id = 1}; + std::vector selected; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, exact_group, first_rows, &selected) + .ok()); + EXPECT_EQ(selected, std::vector({1})); +} + class Int32ZoneMapExpr final : public VExpr { public: enum class Op { GE, GT, LT }; diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 42e84c211ec8ac..30efd7966972ee 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -172,6 +172,8 @@ Status prepare_range(LanceTableReader* reader, TFileRangeDesc range) { .cache = nullptr, .current_range = std::move(range), .current_split_format = FileFormat::LANCE, + .file_context = nullptr, + .format_split_id = -1, .global_rowid_context = std::nullopt}); } diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 9218af0b48ada0..a7f0fb153f40f6 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1200,6 +1200,8 @@ struct FakeFileReaderState { bool stop_during_aggregate = false; bool stop_during_read = false; bool not_found_during_init = false; + int physical_split_count = -1; + int build_physical_splits_count = 0; std::shared_ptr last_request; std::shared_ptr pending_request; std::optional last_aggregate_request; @@ -1351,6 +1353,27 @@ class FakeFileReader final : public FileReader { int64_t get_total_rows() const override { return _state->total_rows; } + Status build_physical_splits(const FileScanSplit& source_split, + std::vector* splits, + bool* was_split) const override { + if (_state->physical_split_count < 0) { + return FileReader::build_physical_splits(source_split, splits, was_split); + } + ++_state->build_physical_splits_count; + splits->clear(); + auto source_range = std::make_shared(source_split.range); + for (int index = 0; index < _state->physical_split_count; ++index) { + FileScanSplit child; + child.source_range = source_range; + child.start_offset = index * 100; + child.size = 100; + child.format_split_id = index; + splits->push_back(std::move(child)); + } + *was_split = true; + return Status::OK(); + } + Status close() override { ++_state->close_count; _request.reset(); @@ -1735,6 +1758,51 @@ TEST(TableReaderTest, CanUseInjectedFileReaderForStandaloneUnitTest) { EXPECT_TRUE(eos); } +TEST(TableReaderTest, SinglePhysicalSplitReusesPlanningReader) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->physical_split_count = 1; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + FileScanSplit source_split; + source_split.range = split_options.current_range; + source_split.is_source_split = true; + std::vector children; + bool was_split = false; + ASSERT_TRUE(reader.build_physical_splits(source_split, &children, &was_split).ok()); + + EXPECT_FALSE(was_split); + EXPECT_TRUE(children.empty()); + EXPECT_EQ(fake_state->init_count, 1); + EXPECT_EQ(fake_state->close_count, 0); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(fake_state->init_count, 1); + EXPECT_EQ(fake_state->open_count, 1); + ASSERT_TRUE(reader.close().ok()); +} + TEST(TableReaderTest, PrepareSplitReplacesInitialConjunctSnapshot) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); From 2b0e338d4e42c354d4a80364f172bafb03177c06 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 16 Aug 2026 18:45:42 +0800 Subject: [PATCH 2/4] [improvement](external) Harden row-group split refinement --- be/src/exec/operator/file_scan_operator.cpp | 52 ++++--- be/src/exec/operator/file_scan_operator.h | 14 ++ be/src/exec/scan/file_scanner_v2.cpp | 7 +- be/src/exec/scan/file_scanner_v2.h | 4 +- be/src/format_v2/file_scan_context.cpp | 19 ++- be/src/format_v2/file_scan_context.h | 15 +- .../parquet/parquet_file_context.cpp | 24 ++- .../format_v2/parquet/parquet_file_context.h | 6 + be/src/format_v2/parquet/parquet_profile.cpp | 10 ++ be/src/format_v2/parquet/parquet_profile.h | 5 + be/src/format_v2/parquet/parquet_reader.cpp | 28 +++- be/test/exec/scan/file_scanner_v2_test.cpp | 53 +++++++ .../format_v2/parquet/parquet_reader_test.cpp | 140 ++++++++++++++++++ 13 files changed, 343 insertions(+), 34 deletions(-) diff --git a/be/src/exec/operator/file_scan_operator.cpp b/be/src/exec/operator/file_scan_operator.cpp index 34deecab13d56b..4c93b718f1afe4 100644 --- a/be/src/exec/operator/file_scan_operator.cpp +++ b/be/src/exec/operator/file_scan_operator.cpp @@ -152,6 +152,34 @@ bool FileScanLocalState::_should_use_file_scanner_v2(const TQueryOptions& query_ !is_transactional_hive; } +bool FileScanLocalState::_can_generate_physical_splits(const TQueryOptions& query_options, + bool is_load, + const TFileScanRangeParams& scan_params, + const TFileRangeDesc& range) { + if (!_should_use_file_scanner_v2(query_options, is_load, scan_params)) { + return false; + } + const auto format = range.__isset.format_type ? range.format_type : scan_params.format_type; + if (format == TFileFormatType::FORMAT_PARQUET) { + // Keep scanner creation aligned with the downstream refinement guard. Otherwise an + // Iceberg delete split creates idle scanners even though it can never publish children. + return FileScannerV2::can_refine_source_split(range); + } + if (format != TFileFormatType::FORMAT_JNI || !range.__isset.table_format_params || + range.table_format_params.table_format_type != "paimon" || + !range.table_format_params.__isset.paimon_params) { + return false; + } + const auto& paimon = range.table_format_params.paimon_params; + return paimon.__isset.file_format && paimon.file_format == "parquet" && + !paimon.__isset.paimon_split; +} + +int FileScanLocalState::_adjust_scanner_count(int requested, int initial_ranges, + bool can_generate_physical_splits) { + return can_generate_physical_splits ? requested : std::min(requested, initial_ranges); +} + Status FileScanLocalState::_init_scanners(std::list* scanners) { if (_split_source->num_scan_ranges() == 0) { _eos = true; @@ -277,25 +305,10 @@ void FileScanLocalState::set_scan_ranges(RuntimeState* state, } const bool is_load = state->desc_tbl().get_tuple_descriptor(params->src_tuple_id) != nullptr; - if (!_should_use_file_scanner_v2(state->query_options(), is_load, *params)) { - continue; - } can_generate_parquet_splits = std::ranges::any_of(file_scan_range.ranges, [&](const auto& range) { - const auto format = - range.__isset.format_type ? range.format_type : params->format_type; - if (format == TFileFormatType::FORMAT_PARQUET) { - return true; - } - if (format != TFileFormatType::FORMAT_JNI || - !range.__isset.table_format_params || - range.table_format_params.table_format_type != "paimon" || - !range.table_format_params.__isset.paimon_params) { - return false; - } - const auto& paimon = range.table_format_params.paimon_params; - return paimon.__isset.file_format && paimon.file_format == "parquet" && - !paimon.__isset.paimon_split; + return _can_generate_physical_splits(state->query_options(), is_load, + *params, range); }); if (can_generate_parquet_splits) { break; @@ -303,9 +316,8 @@ void FileScanLocalState::set_scan_ranges(RuntimeState* state, } // Currently the total number of remote splits cannot be accurately obtained, so batch // mode already skips this cap. - if (!can_generate_parquet_splits) { - _max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges()); - } + _max_scanners = _adjust_scanner_count(_max_scanners, _split_source->num_scan_ranges(), + can_generate_parquet_splits); } if (!scan_ranges.empty() && diff --git a/be/src/exec/operator/file_scan_operator.h b/be/src/exec/operator/file_scan_operator.h index 91407faa50e262..974ae40be297cd 100644 --- a/be/src/exec/operator/file_scan_operator.h +++ b/be/src/exec/operator/file_scan_operator.h @@ -59,6 +59,15 @@ class FileScanLocalState final : public ScanLocalState { #ifdef BE_TEST static bool TEST_should_use_file_scanner_v2(const TQueryOptions& query_options, bool is_load, const TFileScanRangeParams& scan_params); + static bool TEST_can_generate_physical_splits(const TQueryOptions& query_options, bool is_load, + const TFileScanRangeParams& scan_params, + const TFileRangeDesc& range) { + return _can_generate_physical_splits(query_options, is_load, scan_params, range); + } + static int TEST_adjust_scanner_count(int requested, int initial_ranges, + bool can_generate_physical_splits) { + return _adjust_scanner_count(requested, initial_ranges, can_generate_physical_splits); + } #endif private: @@ -73,6 +82,11 @@ class FileScanLocalState final : public ScanLocalState { bool _push_down_topn(const RuntimePredicate& predicate) override; static bool _should_use_file_scanner_v2(const TQueryOptions& query_options, bool is_load, const TFileScanRangeParams& scan_params); + static bool _can_generate_physical_splits(const TQueryOptions& query_options, bool is_load, + const TFileScanRangeParams& scan_params, + const TFileRangeDesc& range); + static int _adjust_scanner_count(int requested, int initial_ranges, + bool can_generate_physical_splits); PushDownType _should_push_down_is_null_predicate(VectorizedFnCall* fn_call) const override { return fn_call->fn().name.function_name == "is_null_pred" || diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index be2843d6474f5e..f312bb5b293d60 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -650,7 +650,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { RETURN_IF_ERROR(_complete_current_split()); continue; } - if (_current_split.is_source_split && _can_refine_source_split(_current_range)) { + if (_current_split.is_source_split && can_refine_source_split(_current_range)) { std::vector generated_splits; bool was_split = false; const auto split_status = _table_reader->build_physical_splits( @@ -806,7 +806,8 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range, .current_split_format = current_split_format, .file_context = _current_split.file_context, .format_split_id = _current_split.format_split_id, - .global_rowid_context = _create_global_rowid_context(range), + .global_rowid_context = + _create_global_rowid_context(_current_split.source_identity_range()), })); return Status::OK(); } @@ -834,7 +835,7 @@ FileScannerV2::IgnoredSplitStatus FileScannerV2::_classify_ignored_split_status( return IgnoredSplitStatus::NONE; } -bool FileScannerV2::_can_refine_source_split(const TFileRangeDesc& range) { +bool FileScannerV2::can_refine_source_split(const TFileRangeDesc& range) { if (!range.__isset.table_format_params || !range.table_format_params.__isset.iceberg_params) { return true; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index a19439edd28472..000dcd5119b450 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -72,6 +72,7 @@ class FileScannerV2 final : public Scanner { enum class UncachedReaderBytesStorage { LOCAL, REMOTE, NONE }; static bool is_supported(const TFileScanRangeParams& params, const TFileRangeDesc& range); + static bool can_refine_source_split(const TFileRangeDesc& range); #ifdef BE_TEST FileScannerV2(RuntimeState* state, RuntimeProfile* profile, std::unique_ptr table_reader); @@ -110,7 +111,7 @@ class FileScannerV2 final : public Scanner { return _classify_ignored_split_status(status, ignore_not_found, stopped); } static bool TEST_can_refine_source_split(const TFileRangeDesc& range) { - return _can_refine_source_split(range); + return can_refine_source_split(range); } static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); static bool TEST_should_skip_empty(const Status& status, bool stopped); @@ -164,7 +165,6 @@ class FileScannerV2 final : public Scanner { static bool _should_skip_empty(const Status& status, bool stopped); static IgnoredSplitStatus _classify_ignored_split_status(const Status& status, bool ignore_not_found, bool stopped); - static bool _can_refine_source_split(const TFileRangeDesc& range); static Status _contextualize_output_filter_status(Status status, TFileFormatType::type format_type); static int64_t _cumulative_profile_delta(int64_t current, int64_t* reported); diff --git a/be/src/format_v2/file_scan_context.cpp b/be/src/format_v2/file_scan_context.cpp index fc6ea94ff17214..11469687239b26 100644 --- a/be/src/format_v2/file_scan_context.cpp +++ b/be/src/format_v2/file_scan_context.cpp @@ -23,9 +23,13 @@ namespace doris { Status FileContextRegistry::get_or_create(const std::string& key, const Loader& loader, - std::shared_ptr* context) { + std::shared_ptr* context, + LookupResult* lookup_result) { DORIS_CHECK(context != nullptr); context->reset(); + if (lookup_result != nullptr) { + *lookup_result = {}; + } while (true) { std::shared_ptr entry; bool load = false; @@ -51,6 +55,9 @@ Status FileContextRegistry::get_or_create(const std::string& key, const Loader& } if (load) { + if (lookup_result != nullptr) { + lookup_result->loaded = true; + } std::shared_ptr loaded_context; Status status; try { @@ -77,12 +84,20 @@ Status FileContextRegistry::get_or_create(const std::string& key, const Loader& } std::unique_lock lock(entry->lock); - entry->ready.wait(lock, [&]() { return !entry->loading; }); + if (entry->loading) { + if (lookup_result != nullptr) { + lookup_result->waited = true; + } + entry->ready.wait(lock, [&]() { return !entry->loading; }); + } if (!entry->status.ok()) { return entry->status; } *context = entry->context.lock(); if (*context != nullptr) { + if (lookup_result != nullptr) { + lookup_result->hit = true; + } return Status::OK(); } // The loading caller may already have released its result. Retry so this caller installs diff --git a/be/src/format_v2/file_scan_context.h b/be/src/format_v2/file_scan_context.h index a3719cb81cc078..685cedc5afa95e 100644 --- a/be/src/format_v2/file_scan_context.h +++ b/be/src/format_v2/file_scan_context.h @@ -43,8 +43,15 @@ class FileContextRegistry { public: using Loader = std::function*)>; + struct LookupResult { + bool loaded = false; + bool waited = false; + bool hit = false; + }; + Status get_or_create(const std::string& key, const Loader& loader, - std::shared_ptr* context); + std::shared_ptr* context, + LookupResult* lookup_result = nullptr); private: struct Entry { @@ -93,6 +100,12 @@ struct FileScanSplit { uint64_t source_split_id = 0; std::shared_ptr source_progress; + // GLOBAL_ROWID second-phase reads batch by the FE source mapping, while start_offset/size above + // identify only this first-phase physical child. + const TFileRangeDesc& source_identity_range() const { + return source_range == nullptr ? range : *source_range; + } + TFileRangeDesc materialize_range() const { auto materialized = source_range == nullptr ? range : *source_range; if (source_range != nullptr) { diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 0e994767b18c00..1d711165e259d1 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -279,6 +279,13 @@ std::string build_page_cache_file_key(const io::FileReader& file_reader, } // namespace +bool ParquetFileContext::can_refine_physical_splits() const { + // InMemoryFileReader lazily owns one whole-file buffer per reader. Publishing children would + // reload and copy that complete HTTP object once per child, so keep its initialized reader. + return native_file != nullptr && + typeid_cast(native_file.get()) == nullptr; +} + Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, const io::FileDescription& file_description, bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary, @@ -354,11 +361,22 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont }; std::shared_ptr resolved_context = std::move(file_context); - if (resolved_context == nullptr) { + if (resolved_context != nullptr) { + ++file_context_registry_bypasses; + } else { if (file_context_registry != nullptr && has_stable_meta_cache_identity) { - RETURN_IF_ERROR(file_context_registry->get_or_create(registry_key, load_context, - &resolved_context)); + ++file_context_registry_requests; + FileContextRegistry::LookupResult lookup_result; + const auto registry_status = file_context_registry->get_or_create( + registry_key, load_context, &resolved_context, &lookup_result); + file_context_registry_loads += lookup_result.loaded; + file_context_registry_hits += lookup_result.hit; + file_context_registry_waits += lookup_result.waited; + RETURN_IF_ERROR(registry_status); } else { + // Keep uncacheable identities and scans without a registry visible separately from + // both physical footer reads and process-wide metadata-cache outcomes. + ++file_context_registry_bypasses; RETURN_IF_ERROR(load_context(&resolved_context)); } } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index a73f2b8b89225f..0049a3f9b1c0f4 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -151,6 +151,11 @@ struct ParquetFileContext { std::shared_ptr shared_file_context; int64_t native_footer_read_calls = 0; int64_t native_footer_cache_hits = 0; + int64_t file_context_registry_requests = 0; + int64_t file_context_registry_loads = 0; + int64_t file_context_registry_hits = 0; + int64_t file_context_registry_waits = 0; + int64_t file_context_registry_bypasses = 0; bool native_page_cache_enabled = false; std::string native_page_cache_file_key; // Set once after the logical file schema is built. Per-request planning uses this guard so @@ -162,6 +167,7 @@ struct ParquetFileContext { bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false, FileContextRegistry* file_context_registry = nullptr, std::shared_ptr file_context = nullptr); + bool can_refine_physical_splits() const; Status load_native_offset_indexes( int row_group_id, const std::unordered_set& leaf_column_ids, std::unordered_map* offset_indexes) const; diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index ce3a973bb02541..48518620add277 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -174,6 +174,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); file_footer_hit_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, parquet_profile, 1); + file_context_registry_requests = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FileContextRegistryRequests", TUnit::UNIT, parquet_profile, 1); + file_context_registry_loads = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileContextRegistryLoads", + TUnit::UNIT, parquet_profile, 1); + file_context_registry_hits = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileContextRegistryHits", + TUnit::UNIT, parquet_profile, 1); + file_context_registry_waits = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileContextRegistryWaits", + TUnit::UNIT, parquet_profile, 1); + file_context_registry_bypasses = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FileContextRegistryBypasses", TUnit::UNIT, parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index ef74f312cd15e4..bd902d5f02603f 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -214,6 +214,11 @@ struct ParquetProfile { RuntimeProfile::Counter* open_file_num = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_context_registry_requests = nullptr; + RuntimeProfile::Counter* file_context_registry_loads = nullptr; + RuntimeProfile::Counter* file_context_registry_hits = nullptr; + RuntimeProfile::Counter* file_context_registry_waits = nullptr; + RuntimeProfile::Counter* file_context_registry_bypasses = nullptr; RuntimeProfile::Counter* row_group_filter_time = nullptr; RuntimeProfile::Counter* page_index_read_calls = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 90422f16d7cd34..b8750a2fe65e7d 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -527,19 +527,33 @@ Status ParquetReader::init(RuntimeState* state) { _state->scheduler.set_batch_size(_batch_size); // Opening the file parses the footer before any row group can be scheduled. Keep this timer // around the whole operation so footer/cache latency cannot disappear from a slow profile. + Status file_context_status; { SCOPED_TIMER(_parquet_profile.parse_footer_time); - RETURN_IF_ERROR(_state->file_context.open( + file_context_status = _state->file_context.open( _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, _enable_mapping_timestamp_tz, _enable_mapping_varbinary, _file_context_registry, - _file_context)); + _file_context); } + // Publish registry and physical-footer outcomes even when opening the file fails; otherwise a + // contended or failing footer path disappears from the profile that must diagnose it. if (_profile != nullptr) { COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, _state->file_context.native_footer_read_calls); COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, _state->file_context.native_footer_cache_hits); - } + COUNTER_UPDATE(_parquet_profile.file_context_registry_requests, + _state->file_context.file_context_registry_requests); + COUNTER_UPDATE(_parquet_profile.file_context_registry_loads, + _state->file_context.file_context_registry_loads); + COUNTER_UPDATE(_parquet_profile.file_context_registry_hits, + _state->file_context.file_context_registry_hits); + COUNTER_UPDATE(_parquet_profile.file_context_registry_waits, + _state->file_context.file_context_registry_waits); + COUNTER_UPDATE(_parquet_profile.file_context_registry_bypasses, + _state->file_context.file_context_registry_bypasses); + } + RETURN_IF_ERROR(file_context_status); // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier { @@ -576,6 +590,9 @@ Status ParquetReader::build_physical_splits(const FileScanSplit& source_split, // reader instead of publishing children whose shared footer could become stale. return Status::OK(); } + if (!_state->file_context.can_refine_physical_splits()) { + return Status::OK(); + } ParquetScanRange scan_range { .start_offset = @@ -596,6 +613,11 @@ Status ParquetReader::build_physical_splits(const FileScanSplit& source_split, splits->reserve(selected_row_groups.size()); for (const int row_group_id : selected_row_groups) { const auto& row_group = metadata.row_groups[row_group_id]; + if (row_group.num_rows == 0) { + // Empty row groups are valid and the ordinary scan planner ignores them. Refinement + // must preserve that behavior before inspecting their potentially empty chunks. + continue; + } size_t group_start = std::numeric_limits::max(); size_t group_end = 0; for (size_t column_id = 0; column_id < row_group.columns.size(); ++column_id) { diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 8e12e6d9211497..6ce65fb289d8ae 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -56,6 +56,7 @@ #include "exprs/vslot_ref.h" #include "format_v2/expr/cast.h" #include "format_v2/file_scan_context.h" +#include "storage/id_manager.h" #include "testutil/mock/mock_runtime_state.h" namespace doris { @@ -730,6 +731,33 @@ TEST(FileScannerV2Test, GeneratedChildrenCompleteSourceProgressOnlyOnce) { EXPECT_TRUE(second.source_progress->complete_one()); } +TEST(FileScannerV2Test, GeneratedChildrenKeepOneGlobalRowIdSourceMapping) { + auto source_range = std::make_shared(); + source_range->__set_path("shared.parquet"); + source_range->__set_start_offset(64); + source_range->__set_size(1024); + + FileScanSplit first; + first.source_range = source_range; + first.start_offset = 128; + first.size = 256; + FileScanSplit second; + second.source_range = source_range; + second.start_offset = 512; + second.size = 256; + + EXPECT_NE(first.materialize_range().start_offset, second.materialize_range().start_offset); + EXPECT_EQ(first.source_identity_range().start_offset, source_range->start_offset); + EXPECT_EQ(second.source_identity_range().start_offset, source_range->start_offset); + + IdFileMap id_file_map(0); + const auto first_id = id_file_map.get_file_mapping_id( + std::make_shared(7, first.source_identity_range(), false)); + const auto second_id = id_file_map.get_file_mapping_id( + std::make_shared(7, second.source_identity_range(), false)); + EXPECT_EQ(first_id, second_id); +} + class TestFileContext final : public FileContext {}; TEST(FileScannerV2Test, FileContextRegistryLoadsEachFileOnceConcurrently) { @@ -819,6 +847,31 @@ TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) { EXPECT_TRUE(FileScannerV2::is_supported(params, legacy_paimon_jni_range_without_reader_type())); } +TEST(FileScannerV2Test, IcebergDeleteSplitKeepsInitialScannerCountCap) { + TQueryOptions query_options; + query_options.__set_enable_file_scanner_v2(true); + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + TFileRangeDesc range; + range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + TTableFormatFileDesc table_format; + table_format.__set_table_format_type("iceberg"); + TIcebergFileDesc iceberg; + TIcebergDeleteFileDesc delete_file; + iceberg.__set_delete_files({delete_file}); + table_format.__set_iceberg_params(iceberg); + range.__set_table_format_params(table_format); + + EXPECT_FALSE(FileScanLocalState::TEST_can_generate_physical_splits(query_options, false, params, + range)); + EXPECT_EQ(FileScanLocalState::TEST_adjust_scanner_count(16, 1, false), 1); + + range.table_format_params.iceberg_params.__set_delete_files({}); + EXPECT_TRUE(FileScanLocalState::TEST_can_generate_physical_splits(query_options, false, params, + range)); + EXPECT_EQ(FileScanLocalState::TEST_adjust_scanner_count(16, 1, true), 16); +} + TEST(FileScannerV2Test, FailedTableReaderCloseCanBeRetriedThroughScanner) { RuntimeState state; RuntimeProfile profile("file_scanner_v2_close_retry"); diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 49cc25082abd6a..5017e9b126c3e6 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -75,6 +75,8 @@ #include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/local_file_system.h" #include "io/io_common.h" #include "runtime/runtime_state.h" #include "storage/index/zone_map/zonemap_eval_context.h" @@ -862,6 +864,58 @@ void write_parquet_file(const std::string& file_path, int64_t row_group_size = R row_group_size, builder.build())); } +void write_parquet_file_with_explicit_row_groups( + const std::string& file_path, const std::vector>& row_groups) { + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + const auto id = ::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::None(), + ::parquet::Type::INT32); + const auto schema_node = + ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, {id}); + const auto schema = std::static_pointer_cast<::parquet::schema::GroupNode>(schema_node); + ::parquet::WriterProperties::Builder properties; + properties.compression(::parquet::Compression::UNCOMPRESSED); + properties.disable_dictionary(); + auto writer = ::parquet::ParquetFileWriter::Open(out, schema, properties.build()); + for (const auto& values : row_groups) { + auto* row_group = writer->AppendRowGroup(); + auto* id_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + if (!values.empty()) { + EXPECT_EQ(id_writer->WriteBatch(values.size(), nullptr, nullptr, values.data()), + values.size()); + } + id_writer->Close(); + row_group->Close(); + } + writer->Close(); +} + +class HttpPathFileReader final : public io::FileReader { +public: + explicit HttpPathFileReader(io::FileReaderSPtr delegate) + : _delegate(std::move(delegate)), + _path("https://example.test/multi-row-group.parquet") {} + + Status close() override { return _delegate->close(); } + const io::Path& path() const override { return _path; } + size_t size() const override { return _delegate->size(); } + bool closed() const override { return _delegate->closed(); } + int64_t mtime() const override { return _delegate->mtime(); } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext* io_ctx) override { + return _delegate->read_at(offset, result, bytes_read, io_ctx); + } + +private: + io::FileReaderSPtr _delegate; + io::Path _path; +}; + void write_mixed_variant_row_groups(const std::string& file_path) { auto n_type = arrow::struct_({arrow::field("value", arrow::binary(), true), arrow::field("typed_value", arrow::int32(), true)}); @@ -1836,6 +1890,11 @@ TEST_F(NewParquetReaderTest, RowGroupSplitsShareOneRegistryFooterContext) { .table_format_params.__isset.table_level_row_count); } EXPECT_EQ(parent_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(parent_profile.get_counter("FileContextRegistryRequests")->value(), 1); + EXPECT_EQ(parent_profile.get_counter("FileContextRegistryLoads")->value(), 1); + EXPECT_EQ(parent_profile.get_counter("FileContextRegistryHits")->value(), 0); + EXPECT_EQ(parent_profile.get_counter("FileContextRegistryWaits")->value(), 0); + EXPECT_EQ(parent_profile.get_counter("FileContextRegistryBypasses")->value(), 0); RuntimeProfile sibling_profile("same_file_source_split"); auto sibling = create_reader(0, -1, &sibling_profile, false, nullptr, std::nullopt, false, @@ -1843,6 +1902,11 @@ TEST_F(NewParquetReaderTest, RowGroupSplitsShareOneRegistryFooterContext) { ASSERT_TRUE(sibling->init(&state).ok()); EXPECT_EQ(sibling_profile.get_counter("FileFooterReadCalls")->value(), 0); EXPECT_EQ(sibling_profile.get_counter("FileFooterHitCache")->value(), 0); + EXPECT_EQ(sibling_profile.get_counter("FileContextRegistryRequests")->value(), 1); + EXPECT_EQ(sibling_profile.get_counter("FileContextRegistryLoads")->value(), 0); + EXPECT_EQ(sibling_profile.get_counter("FileContextRegistryHits")->value(), 1); + EXPECT_EQ(sibling_profile.get_counter("FileContextRegistryWaits")->value(), 0); + EXPECT_EQ(sibling_profile.get_counter("FileContextRegistryBypasses")->value(), 0); RuntimeProfile child_profile("row_group_split_child"); auto child_range = children[0].materialize_range(); @@ -1852,6 +1916,11 @@ TEST_F(NewParquetReaderTest, RowGroupSplitsShareOneRegistryFooterContext) { ASSERT_TRUE(child->init(&state).ok()); EXPECT_EQ(child_profile.get_counter("FileFooterReadCalls")->value(), 0); EXPECT_EQ(child_profile.get_counter("FileFooterHitCache")->value(), 0); + EXPECT_EQ(child_profile.get_counter("FileContextRegistryRequests")->value(), 0); + EXPECT_EQ(child_profile.get_counter("FileContextRegistryLoads")->value(), 0); + EXPECT_EQ(child_profile.get_counter("FileContextRegistryHits")->value(), 0); + EXPECT_EQ(child_profile.get_counter("FileContextRegistryWaits")->value(), 0); + EXPECT_EQ(child_profile.get_counter("FileContextRegistryBypasses")->value(), 1); std::vector schema; ASSERT_TRUE(child->get_schema(&schema).ok()); @@ -1872,6 +1941,77 @@ TEST_F(NewParquetReaderTest, RowGroupSplitsShareOneRegistryFooterContext) { EXPECT_EQ(total_rows, 2); } +TEST_F(NewParquetReaderTest, PhysicalSplitsSkipEmptyRowGroupsBeforeAndBetweenData) { + write_parquet_file_with_explicit_row_groups(_file_path, {{}, {10}, {}, {20}}); + constexpr int64_t TEST_MTIME = 515151; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto reader = create_reader(0, -1, nullptr, false, nullptr, std::nullopt, false, false, {}, + TEST_MTIME); + ASSERT_TRUE(reader->init(&state).ok()); + + FileScanSplit source; + source.range.__set_path(_file_path); + source.range.__set_start_offset(0); + source.range.__set_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_file_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_modification_time(TEST_MTIME); + source.is_source_split = true; + std::vector children; + bool was_split = false; + ASSERT_TRUE(reader->build_physical_splits(source, &children, &was_split).ok()); + ASSERT_TRUE(was_split); + ASSERT_EQ(children.size(), 2); + EXPECT_EQ(children[0].format_split_id, 1); + EXPECT_EQ(children[1].format_split_id, 3); +} + +TEST_F(NewParquetReaderTest, PhysicalSplitsCompleteAnAllEmptyFileWithoutChildren) { + write_parquet_file_with_explicit_row_groups(_file_path, {{}, {}}); + constexpr int64_t TEST_MTIME = 616161; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto reader = create_reader(0, -1, nullptr, false, nullptr, std::nullopt, false, false, {}, + TEST_MTIME); + ASSERT_TRUE(reader->init(&state).ok()); + + FileScanSplit source; + source.range.__set_path(_file_path); + source.range.__set_start_offset(0); + source.range.__set_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_file_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_modification_time(TEST_MTIME); + source.is_source_split = true; + std::vector children; + bool was_split = false; + ASSERT_TRUE(reader->build_physical_splits(source, &children, &was_split).ok()); + EXPECT_TRUE(was_split); + EXPECT_TRUE(children.empty()); +} + +TEST_F(NewParquetReaderTest, InMemoryStagedFilesDeclinePhysicalSplitRefinement) { + write_parquet_file(_file_path, 2); + auto parquet_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_GT(parquet_reader->metadata()->num_row_groups(), 1); + io::FileReaderSPtr local_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(_file_path, &local_reader).ok()); + + format::parquet::ParquetFileContext file_context; + io::FileDescription file_description; + file_description.path = "https://example.test/multi-row-group.parquet"; + file_description.file_size = static_cast(local_reader->size()); + file_description.mtime = 123456; + io::IOContext io_context; + ASSERT_TRUE(file_context + .open(std::make_shared(std::move(local_reader)), + &io_context, false, file_description) + .ok()); + EXPECT_FALSE(file_context.can_refine_physical_splits()); + + io::FileReaderSPtr ordinary_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(_file_path, &ordinary_reader).ok()); + file_context.native_file = std::move(ordinary_reader); + EXPECT_TRUE(file_context.can_refine_physical_splits()); +} + TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; From cb739b37466897fc363a7066edbddbdbb6545284 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 16 Aug 2026 20:42:15 +0800 Subject: [PATCH 3/4] [fix](external) Harden split refresh and lifecycle accounting --- be/src/exec/scan/file_scanner_v2.cpp | 13 +- be/src/exec/scan/file_scanner_v2.h | 5 + be/src/format_v2/jni/jni_table_reader.cpp | 5 +- be/src/format_v2/jni/jni_table_reader.h | 4 +- be/src/format_v2/table/hudi_reader.cpp | 9 +- be/src/format_v2/table/hudi_reader.h | 4 +- be/src/format_v2/table/paimon_reader.cpp | 9 +- be/src/format_v2/table/paimon_reader.h | 4 +- be/src/format_v2/table_reader.cpp | 39 +++- be/src/format_v2/table_reader.h | 4 +- be/test/exec/scan/file_scanner_v2_test.cpp | 15 ++ be/test/format_v2/table/hudi_reader_test.cpp | 6 +- .../format_v2/table/paimon_reader_test.cpp | 6 +- be/test/format_v2/table_reader_test.cpp | 208 +++++++++++++++++- 14 files changed, 307 insertions(+), 24 deletions(-) diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index f312bb5b293d60..c09efe4fc49be3 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -534,7 +534,8 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e if (_table_reader_rf_num != _applied_rf_num) { VExprContextSPtrs refreshed_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&refreshed_conjuncts)); - RETURN_IF_ERROR(_table_reader->refresh_conjuncts(std::move(refreshed_conjuncts))); + RETURN_IF_ERROR(_table_reader->refresh_conjuncts( + std::move(refreshed_conjuncts), _current_condition_cache_digest())); _table_reader_rf_num = _applied_rf_num; } if (_should_run_adaptive_batch_size()) { @@ -650,6 +651,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { RETURN_IF_ERROR(_complete_current_split()); continue; } + _update_file_counter(_file_counter, _current_split); if (_current_split.is_source_split && can_refine_source_split(_current_range)) { std::vector generated_splits; bool was_split = false; @@ -687,7 +689,6 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { // so waiters never depend on a later get_block() turn to observe completion. RETURN_IF_ERROR(_retire_current_source_split()); } - COUNTER_UPDATE(_file_counter, 1); _has_prepared_split = true; _table_reader_rf_num = _applied_rf_num; *eos = false; @@ -845,6 +846,14 @@ bool FileScannerV2::can_refine_source_split(const TFileRangeDesc& range) { return !iceberg.__isset.delete_files || iceberg.delete_files.empty(); } +void FileScannerV2::_update_file_counter(RuntimeProfile::Counter* counter, + const FileScanSplit& split) { + if (split.is_source_split) { + // FileNumber describes FE source ranges, so BE-local row-group children must not change it. + COUNTER_UPDATE(counter, 1); + } +} + bool FileScannerV2::_should_enable_file_meta_cache() const { return ExecEnv::GetInstance()->file_meta_cache()->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 000dcd5119b450..f3ba29d73f3afb 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -124,6 +124,10 @@ class FileScannerV2 final : public Scanner { return _should_run_adaptive_batch_size(predictor_initialized, current_split_uses_metadata_count); } + static void TEST_update_file_counter(RuntimeProfile::Counter* counter, + const FileScanSplit& split) { + _update_file_counter(counter, split); + } #endif FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -168,6 +172,7 @@ class FileScannerV2 final : public Scanner { static Status _contextualize_output_filter_status(Status status, TFileFormatType::type format_type); static int64_t _cumulative_profile_delta(int64_t current, int64_t* reported); + static void _update_file_counter(RuntimeProfile::Counter* counter, const FileScanSplit& split); static FileReaderProfileDeltas _collect_file_reader_profile_deltas( const io::FileReaderStats& stats, int64_t* reported_bytes, int64_t* reported_calls, int64_t* reported_time); diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp index a5fbe61fd35d36..b9228216c6845f 100644 --- a/be/src/format_v2/jni/jni_table_reader.cpp +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -77,7 +77,8 @@ Status JniTableReader::prepare_split(const SplitReadOptions& options) { return _open_jni_scanner(); } -Status JniTableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { +Status JniTableReader::refresh_conjuncts(VExprContextSPtrs conjuncts, + std::optional condition_cache_digest) { if (_scanner_opened) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.refresh_conjuncts_timer); @@ -91,7 +92,7 @@ Status JniTableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { RETURN_IF_ERROR(conjunct->open(_runtime_state)); } } - return TableReader::refresh_conjuncts(std::move(conjuncts)); + return TableReader::refresh_conjuncts(std::move(conjuncts), condition_cache_digest); } Status JniTableReader::get_block(Block* output_block, bool* eos) { diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h index 76e51c1c059644..2b60b3c9691c38 100644 --- a/be/src/format_v2/jni/jni_table_reader.h +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -48,7 +48,9 @@ class JniTableReader : public TableReader { Status init(TableReadOptions&& options) override; Status prepare_split(const SplitReadOptions& options) override; - Status refresh_conjuncts(VExprContextSPtrs conjuncts) override; + Status refresh_conjuncts( + VExprContextSPtrs conjuncts, + std::optional condition_cache_digest = std::nullopt) override; Status get_block(Block* block, bool* eos) override; Status abort_split() override; Status close() override; diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 429591f803369a..bcf727240a3e8e 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -87,8 +87,10 @@ Status HudiHybridReader::build_physical_splits(const FileScanSplit& source_split return _current_split_reader->build_physical_splits(source_split, splits, was_split); } -Status HudiHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { - RETURN_IF_ERROR(format::TableReader::refresh_conjuncts(std::move(conjuncts))); +Status HudiHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts, + std::optional condition_cache_digest) { + RETURN_IF_ERROR( + format::TableReader::refresh_conjuncts(std::move(conjuncts), condition_cache_digest)); if (_current_split_reader == nullptr) { return Status::OK(); } @@ -96,7 +98,8 @@ Status HudiHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { RETURN_IF_ERROR(_clone_conjuncts(&child_conjuncts)); // The hybrid wrapper owns no physical reader; forward a clone so the active child, rather than // only the wrapper snapshot, observes late predicates for the remainder of this split. - return _current_split_reader->refresh_conjuncts(std::move(child_conjuncts)); + return _current_split_reader->refresh_conjuncts(std::move(child_conjuncts), + condition_cache_digest); } Status HudiHybridReader::get_block(Block* block, bool* eos) { diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index f98eb766c65c3f..fe39bcf6f4e154 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -61,7 +61,9 @@ class HudiHybridReader final : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; Status build_physical_splits(const FileScanSplit& source_split, std::vector* splits, bool* was_split) override; - Status refresh_conjuncts(VExprContextSPtrs conjuncts) override; + Status refresh_conjuncts( + VExprContextSPtrs conjuncts, + std::optional condition_cache_digest = std::nullopt) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; bool current_split_uses_metadata_count() const override; diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 6cf4bc1910ea7c..3ddf65076bcd92 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -286,8 +286,10 @@ Status PaimonHybridReader::build_physical_splits(const FileScanSplit& source_spl return _current_split_reader->build_physical_splits(source_split, splits, was_split); } -Status PaimonHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { - RETURN_IF_ERROR(format::TableReader::refresh_conjuncts(std::move(conjuncts))); +Status PaimonHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts, + std::optional condition_cache_digest) { + RETURN_IF_ERROR( + format::TableReader::refresh_conjuncts(std::move(conjuncts), condition_cache_digest)); if (_current_split_reader == nullptr) { return Status::OK(); } @@ -295,7 +297,8 @@ Status PaimonHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { RETURN_IF_ERROR(_clone_conjuncts(&child_conjuncts)); // The hybrid wrapper owns no physical reader; forward a clone so the active child, rather than // only the wrapper snapshot, observes late predicates for the remainder of this split. - return _current_split_reader->refresh_conjuncts(std::move(child_conjuncts)); + return _current_split_reader->refresh_conjuncts(std::move(child_conjuncts), + condition_cache_digest); } Status PaimonHybridReader::get_block(Block* block, bool* eos) { diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index de28e2af39c470..9fdc9beef984b8 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -76,7 +76,9 @@ class PaimonHybridReader final : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; Status build_physical_splits(const FileScanSplit& source_split, std::vector* splits, bool* was_split) override; - Status refresh_conjuncts(VExprContextSPtrs conjuncts) override; + Status refresh_conjuncts( + VExprContextSPtrs conjuncts, + std::optional condition_cache_digest = std::nullopt) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; bool current_split_uses_metadata_count() const override; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 05cdde601b539c..ae6dffcb009564 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -884,10 +884,17 @@ bool same_physical_scan_layout(const FileScanRequest& lhs, const FileScanRequest } // namespace -Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { +Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts, + std::optional condition_cache_digest) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.refresh_conjuncts_timer); _conjuncts = std::move(conjuncts); + if (condition_cache_digest.has_value()) { + // A runtime filter can arrive after a physical child is prepared but before its reader is + // created. Keep that child's cache key tied to the same refreshed predicate snapshot. + _condition_cache_digest = *condition_cache_digest; + _condition_cache_digest_covers_current_split = true; + } if (_data_reader.reader == nullptr) { // The split is prepared but its physical reader has not opened yet. open_reader() will use // this newest snapshot directly, so no pending request is needed. @@ -1256,6 +1263,7 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { Status TableReader::build_physical_splits(const FileScanSplit& source_split, std::vector* splits, bool* was_split) { + SCOPED_TIMER(_profile.total_timer); DORIS_CHECK(splits != nullptr); DORIS_CHECK(was_split != nullptr); splits->clear(); @@ -1264,14 +1272,35 @@ Status TableReader::build_physical_splits(const FileScanSplit& source_split, _current_split_uses_metadata_count || _current_task == nullptr) { return Status::OK(); } + SCOPED_TIMER(_profile.create_reader_timer); std::unique_ptr reader; RETURN_IF_ERROR(create_file_reader(&reader)); DORIS_CHECK(reader != nullptr); - RETURN_IF_ERROR(reader->init(_runtime_state)); - const auto status = reader->build_physical_splits(source_split, splits, was_split); + auto close_planning_reader = [&]() { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_close_timer); + return reader->close(); + }; + Status init_status; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_init_timer); + init_status = reader->init(_runtime_state); + } + if (!init_status.ok()) { + // A failed init may still own partially opened resources. Close it through the same + // lifecycle path while preserving the initialization error returned to the scanner. + static_cast(close_planning_reader()); + return init_status; + } + Status status; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + status = reader->build_physical_splits(source_split, splits, was_split); + } if (!status.ok()) { - static_cast(reader->close()); + static_cast(close_planning_reader()); return status; } if (!*was_split || splits->size() == 1) { @@ -1285,7 +1314,7 @@ Status TableReader::build_physical_splits(const FileScanSplit& source_split, } return open_reader(); } - return reader->close(); + return close_planning_reader(); } Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index d80c5a7928e531..0a569da003eff0 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -235,7 +235,9 @@ class TableReader { // Refresh row-level predicates for an already prepared split. Physical readers that support // this operation decide the safe boundary at which the new immutable request becomes active. - virtual Status refresh_conjuncts(VExprContextSPtrs conjuncts); + // A supplied digest describes this exact conjunct snapshot for condition-cache isolation. + virtual Status refresh_conjuncts(VExprContextSPtrs conjuncts, + std::optional condition_cache_digest = std::nullopt); virtual bool current_split_pruned() const { return _current_split_pruned; } virtual bool current_split_uses_metadata_count() const { diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 6ce65fb289d8ae..23b36f26cf659a 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -731,6 +731,21 @@ TEST(FileScannerV2Test, GeneratedChildrenCompleteSourceProgressOnlyOnce) { EXPECT_TRUE(second.source_progress->complete_one()); } +TEST(FileScannerV2Test, PhysicalChildrenDoNotChangeFileNumber) { + RuntimeProfile profile("source_file_counter"); + auto* file_counter = ADD_COUNTER(&profile, "FileNumber", TUnit::UNIT); + FileScanSplit source; + source.is_source_split = true; + FileScannerV2::TEST_update_file_counter(file_counter, source); + + FileScanSplit child; + child.is_source_split = false; + FileScannerV2::TEST_update_file_counter(file_counter, child); + FileScannerV2::TEST_update_file_counter(file_counter, child); + + EXPECT_EQ(file_counter->value(), 1); +} + TEST(FileScannerV2Test, GeneratedChildrenKeepOneGlobalRowIdSourceMapping) { auto source_range = std::make_shared(); source_range->__set_path("shared.parquet"); diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index e75eee47be39d6..9e3c76431deb16 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -145,9 +145,11 @@ class RefreshTrackingTableReader final : public TableReader { public: Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } - Status refresh_conjuncts(VExprContextSPtrs conjuncts) override { + Status refresh_conjuncts( + VExprContextSPtrs conjuncts, + std::optional condition_cache_digest = std::nullopt) override { ++refresh_count; - return TableReader::refresh_conjuncts(std::move(conjuncts)); + return TableReader::refresh_conjuncts(std::move(conjuncts), condition_cache_digest); } int refresh_count = 0; diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 8a215961c0e6b3..1cc57e8b42a575 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -83,9 +83,11 @@ class RefreshTrackingTableReader final : public TableReader { public: Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } - Status refresh_conjuncts(VExprContextSPtrs conjuncts) override { + Status refresh_conjuncts( + VExprContextSPtrs conjuncts, + std::optional condition_cache_digest = std::nullopt) override { ++refresh_count; - return TableReader::refresh_conjuncts(std::move(conjuncts)); + return TableReader::refresh_conjuncts(std::move(conjuncts), condition_cache_digest); } int refresh_count = 0; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index a7f0fb153f40f6..05c597e14983b2 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -24,11 +24,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include @@ -1200,8 +1202,10 @@ struct FakeFileReaderState { bool stop_during_aggregate = false; bool stop_during_read = false; bool not_found_during_init = false; + bool physical_split_error = false; int physical_split_count = -1; int build_physical_splits_count = 0; + std::chrono::milliseconds lifecycle_delay {0}; std::shared_ptr last_request; std::shared_ptr pending_request; std::optional last_aggregate_request; @@ -1220,6 +1224,9 @@ class FakeFileReader final : public FileReader { Status init(RuntimeState* state) override { (void)state; + if (_state->lifecycle_delay.count() > 0) { + std::this_thread::sleep_for(_state->lifecycle_delay); + } ++_state->init_count; if (_state->not_found_during_init) { return Status::NotFound("fake table reader input is missing"); @@ -1238,6 +1245,9 @@ class FakeFileReader final : public FileReader { } Status open(std::shared_ptr request) override { + if (_state->lifecycle_delay.count() > 0) { + std::this_thread::sleep_for(_state->lifecycle_delay); + } RETURN_IF_ERROR(FileReader::open(std::move(request))); _state->last_request = _request; ++_state->open_count; @@ -1359,7 +1369,13 @@ class FakeFileReader final : public FileReader { if (_state->physical_split_count < 0) { return FileReader::build_physical_splits(source_split, splits, was_split); } + if (_state->lifecycle_delay.count() > 0) { + std::this_thread::sleep_for(_state->lifecycle_delay); + } ++_state->build_physical_splits_count; + if (_state->physical_split_error) { + return Status::InternalError("injected physical split planning failure"); + } splits->clear(); auto source_range = std::make_shared(source_split.range); for (int index = 0; index < _state->physical_split_count; ++index) { @@ -1375,6 +1391,9 @@ class FakeFileReader final : public FileReader { } Status close() override { + if (_state->lifecycle_delay.count() > 0) { + std::this_thread::sleep_for(_state->lifecycle_delay); + } ++_state->close_count; _request.reset(); _eof = true; @@ -1766,8 +1785,10 @@ TEST(TableReaderTest, SinglePhysicalSplitReusesPlanningReader) { set_name_identifiers(&projected_columns); RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("single_physical_split"); auto fake_state = std::make_shared(); fake_state->physical_split_count = 1; + fake_state->lifecycle_delay = std::chrono::milliseconds(2); FakeTableReader reader(file_schema, fake_state); ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, @@ -1776,7 +1797,7 @@ TEST(TableReaderTest, SinglePhysicalSplitReusesPlanningReader) { .scan_params = nullptr, .io_ctx = nullptr, .runtime_state = &state, - .scanner_profile = nullptr, + .scanner_profile = &profile, }) .ok()); @@ -1794,6 +1815,13 @@ TEST(TableReaderTest, SinglePhysicalSplitReusesPlanningReader) { EXPECT_TRUE(children.empty()); EXPECT_EQ(fake_state->init_count, 1); EXPECT_EQ(fake_state->close_count, 0); + for (const auto* counter_name : + {"TableReader", "CreateReaderTime", "FileReader", "FileReaderInitTime", "OpenReaderTime", + "FileReaderOpenTime"}) { + const auto* counter = profile.get_counter(counter_name); + ASSERT_NE(counter, nullptr) << counter_name; + EXPECT_GT(counter->value(), 0) << counter_name; + } Block block = build_table_block(projected_columns); bool eos = false; @@ -1803,6 +1831,100 @@ TEST(TableReaderTest, SinglePhysicalSplitReusesPlanningReader) { ASSERT_TRUE(reader.close().ok()); } +TEST(TableReaderTest, PhysicalSplitPlanningProfilesZeroAndManyChildren) { + for (const int child_count : {0, 2}) { + SCOPED_TRACE(child_count); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("temporary_physical_split_reader"); + auto fake_state = std::make_shared(); + fake_state->physical_split_count = child_count; + fake_state->lifecycle_delay = std::chrono::milliseconds(2); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + FileScanSplit source_split; + source_split.range = split_options.current_range; + source_split.is_source_split = true; + std::vector children; + bool was_split = false; + ASSERT_TRUE(reader.build_physical_splits(source_split, &children, &was_split).ok()); + EXPECT_TRUE(was_split); + EXPECT_EQ(children.size(), child_count); + EXPECT_EQ(fake_state->close_count, 1); + for (const auto* counter_name : {"TableReader", "CreateReaderTime", "FileReader", + "FileReaderInitTime", "FileReaderCloseTime"}) { + const auto* counter = profile.get_counter(counter_name); + ASSERT_NE(counter, nullptr) << counter_name; + EXPECT_GT(counter->value(), 0) << counter_name; + } + } +} + +TEST(TableReaderTest, PhysicalSplitPlanningProfilesAndClosesFailedReaders) { + for (const bool fail_during_init : {true, false}) { + SCOPED_TRACE(fail_during_init ? "init" : "build"); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("failed_physical_split_reader"); + auto fake_state = std::make_shared(); + fake_state->physical_split_count = 2; + fake_state->not_found_during_init = fail_during_init; + fake_state->physical_split_error = !fail_during_init; + fake_state->lifecycle_delay = std::chrono::milliseconds(2); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + FileScanSplit source_split; + source_split.range = split_options.current_range; + source_split.is_source_split = true; + std::vector children; + bool was_split = false; + EXPECT_FALSE(reader.build_physical_splits(source_split, &children, &was_split).ok()); + EXPECT_EQ(fake_state->close_count, 1); + for (const auto* counter_name : {"TableReader", "CreateReaderTime", "FileReader", + "FileReaderInitTime", "FileReaderCloseTime"}) { + const auto* counter = profile.get_counter(counter_name); + ASSERT_NE(counter, nullptr) << counter_name; + EXPECT_GT(counter->value(), 0) << counter_name; + } + } +} + TEST(TableReaderTest, PrepareSplitReplacesInitialConjunctSnapshot) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); @@ -3770,6 +3892,90 @@ TEST(TableReaderTest, ConditionCacheAllowsRuntimeFilterCoveredBySplitDigest) { ASSERT_TRUE(reader.close().ok()); } +TEST(TableReaderTest, ConditionCacheLateRefreshBeforeOpenUsesRefreshedDigest) { + ScopedConditionCacheForTest cache; + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->total_rows = ConditionCacheContext::GRANULE_SIZE; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 0))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("late-runtime-filter-input"); + split_options.condition_cache_digest = 11; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + VExprContextSPtrs refreshed { + prepared_conjunct(&state, table_int32_greater_than_expr(0, 0, 0)), + prepared_conjunct(&state, + runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 1)))}; + ASSERT_TRUE(reader.refresh_conjuncts(std::move(refreshed), 19).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + segment_v2::ConditionCacheHandle handle; + segment_v2::ConditionCache::ExternalCacheKey stale_key( + "late-runtime-filter-input", 0, -1, 11, 0, -1, + segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION); + EXPECT_FALSE(cache.get()->lookup(stale_key, &handle)); + segment_v2::ConditionCache::ExternalCacheKey refreshed_key( + "late-runtime-filter-input", 0, -1, 19, 0, -1, + segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION); + EXPECT_TRUE(cache.get()->lookup(refreshed_key, &handle)); + ASSERT_TRUE(reader.close().ok()); + + auto hit_state = std::make_shared(); + hit_state->total_rows = ConditionCacheContext::GRANULE_SIZE; + FakeTableReader hit_reader(file_schema, hit_state); + ASSERT_TRUE( + hit_reader + .init({ + .projected_columns = projected_columns, + .conjuncts = + { + prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 0)), + prepared_conjunct(&state, + runtime_filter_wrapper_expr( + table_int32_greater_than_expr( + 0, 0, 1))), + }, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + split_options.condition_cache_digest = 19; + ASSERT_TRUE(hit_reader.prepare_split(split_options).ok()); + block = build_table_block(projected_columns); + eos = false; + ASSERT_TRUE(hit_reader.get_block(&block, &eos).ok()); + ASSERT_NE(hit_state->condition_cache_ctx, nullptr); + EXPECT_TRUE(hit_state->condition_cache_ctx->is_hit); + EXPECT_EQ(hit_reader.condition_cache_hit_count(), 1); + ASSERT_TRUE(hit_reader.close().ok()); +} + // Scenario: table-format delete files/deletion vectors are outside the data-file cache key. When // TableReader injects delete conjuncts into the file scan request, condition cache must be disabled // for that split. From 687fff909bca359b09c695143220885f66045932 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 16 Aug 2026 22:59:46 +0800 Subject: [PATCH 4/4] Preserve unsplit parquet fallbacks --- be/src/exec/scan/file_scanner_v2.cpp | 13 ++- be/src/exec/scan/file_scanner_v2.h | 6 ++ be/src/format_v2/parquet/parquet_reader.cpp | 7 ++ be/test/exec/scan/file_scanner_v2_test.cpp | 15 ++++ .../format_v2/parquet/parquet_reader_test.cpp | 82 +++++++++++++++++++ 5 files changed, 121 insertions(+), 2 deletions(-) diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index c09efe4fc49be3..8cbe6e9026ef39 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -402,7 +402,8 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat : Scanner(state, local_state, limit, profile), _split_source(std::move(split_source)), _kv_cache(kv_cache), - _file_context_registry(&local_state->_file_context_registry) { + _file_context_registry(&local_state->_file_context_registry), + _constructed_scanners(local_state->_max_scanners) { (void)colname_to_slot_id; if (state->get_query_ctx() != nullptr && state->get_query_ctx()->file_scan_range_params_map.count(local_state->parent_id()) > 0) { @@ -652,7 +653,8 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { continue; } _update_file_counter(_file_counter, _current_split); - if (_current_split.is_source_split && can_refine_source_split(_current_range)) { + if (_current_split.is_source_split && + _should_refine_source_split(_current_range, _constructed_scanners)) { std::vector generated_splits; bool was_split = false; const auto split_status = _table_reader->build_physical_splits( @@ -846,6 +848,13 @@ bool FileScannerV2::can_refine_source_split(const TFileRangeDesc& range) { return !iceberg.__isset.delete_files || iceberg.delete_files.empty(); } +bool FileScannerV2::_should_refine_source_split(const TFileRangeDesc& range, + int constructed_scanners) { + // Row-group children only reduce wall time when another scanner can consume them. Keeping the + // source reader avoids repeated reader setup in serial and explicitly single-scanner scans. + return constructed_scanners >= 2 && can_refine_source_split(range); +} + void FileScannerV2::_update_file_counter(RuntimeProfile::Counter* counter, const FileScanSplit& split) { if (split.is_source_split) { diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index f3ba29d73f3afb..d567a832f52015 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -113,6 +113,10 @@ class FileScannerV2 final : public Scanner { static bool TEST_can_refine_source_split(const TFileRangeDesc& range) { return can_refine_source_split(range); } + static bool TEST_should_refine_source_split(const TFileRangeDesc& range, + int constructed_scanners) { + return _should_refine_source_split(range, constructed_scanners); + } static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); static bool TEST_should_skip_empty(const Status& status, bool stopped); static Status TEST_contextualize_output_filter_status(Status status, @@ -171,6 +175,7 @@ class FileScannerV2 final : public Scanner { bool ignore_not_found, bool stopped); static Status _contextualize_output_filter_status(Status status, TFileFormatType::type format_type); + static bool _should_refine_source_split(const TFileRangeDesc& range, int constructed_scanners); static int64_t _cumulative_profile_delta(int64_t current, int64_t* reported); static void _update_file_counter(RuntimeProfile::Counter* counter, const FileScanSplit& split); static FileReaderProfileDeltas _collect_file_reader_profile_deltas( @@ -241,6 +246,7 @@ class FileScannerV2 final : public Scanner { std::shared_ptr _io_ctx; ShardedKVCache* _kv_cache = nullptr; FileContextRegistry* _file_context_registry = nullptr; + int _constructed_scanners = 1; RuntimeProfile::Counter* _scanner_total_timer = nullptr; RuntimeProfile::Counter* _init_timer = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index b8750a2fe65e7d..9d33855d898fdd 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -618,6 +618,13 @@ Status ParquetReader::build_physical_splits(const FileScanSplit& source_split, // must preserve that behavior before inspecting their potentially empty chunks. continue; } + if (row_group.columns.empty()) { + // A root-only schema can still carry rows for metadata COUNT(*), but it has no byte + // envelope that can identify a child. Keep the initialized source reader so those + // rows are not turned into a corruption error or discarded after tentative children. + splits->clear(); + return Status::OK(); + } size_t group_start = std::numeric_limits::max(); size_t group_end = 0; for (size_t column_id = 0; column_id < row_group.columns.size(); ++column_id) { diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 23b36f26cf659a..bafbb923b95d9a 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -1159,6 +1159,21 @@ TEST(FileScannerV2Test, IcebergDeleteFilesDisablePhysicalSplitRefinement) { EXPECT_FALSE(FileScannerV2::TEST_can_refine_source_split(range)); } +TEST(FileScannerV2Test, PhysicalSplitRefinementRequiresMultipleConstructedScanners) { + TFileRangeDesc range; + EXPECT_FALSE(FileScannerV2::TEST_should_refine_source_split(range, 1)); + EXPECT_TRUE(FileScannerV2::TEST_should_refine_source_split(range, 2)); + + TIcebergDeleteFileDesc delete_file; + delete_file.__set_path("delete.parquet"); + TIcebergFileDesc iceberg; + iceberg.__set_delete_files({delete_file}); + TTableFormatFileDesc table_format; + table_format.__set_iceberg_params(iceberg); + range.__set_table_format_params(table_format); + EXPECT_FALSE(FileScannerV2::TEST_should_refine_source_split(range, 2)); +} + TEST(FileScannerV2Test, RealtimeCounterDeltasUseFileCacheDeltasWhenAvailable) { io::FileReaderStats file_reader_stats; io::FileCacheStatistics file_cache_statistics; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 5017e9b126c3e6..1cbd46ec0bf9f3 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -136,6 +136,53 @@ void annotate_variant_schema(const std::string& file_path) { DORIS_CHECK(output.good()); } +void rewrite_as_root_only_parquet_file(const std::string& file_path) { + std::ifstream input(file_path, std::ios::binary | std::ios::ate); + DORIS_CHECK(input.good()); + const auto input_size = static_cast(input.tellg()); + DORIS_CHECK(input_size >= static_cast(8)); + std::vector file_bytes(cast_set(input_size)); + input.seekg(0); + input.read(reinterpret_cast(file_bytes.data()), cast_set(input_size)); + DORIS_CHECK(input.good()); + DORIS_CHECK(memcmp(file_bytes.data() + file_bytes.size() - 4, "PAR1", 4) == 0); + + const uint32_t footer_size = decode_fixed32_le(file_bytes.data() + file_bytes.size() - 8); + DORIS_CHECK(footer_size <= file_bytes.size() - 8); + const size_t footer_offset = file_bytes.size() - 8 - footer_size; + uint32_t thrift_size = footer_size; + tparquet::FileMetaData metadata; + DORIS_CHECK( + deserialize_thrift_msg(file_bytes.data() + footer_offset, &thrift_size, true, &metadata) + .ok()); + input.close(); + + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(0); + root.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + metadata.__set_schema({root}); + for (auto& row_group : metadata.row_groups) { + row_group.columns.clear(); + } + + file_bytes.resize(footer_offset); + std::vector footer; + ThriftSerializer serializer(/*compact=*/true, 1024); + DORIS_CHECK(serializer.serialize(&metadata, &footer).ok()); + file_bytes.insert(file_bytes.end(), footer.begin(), footer.end()); + std::array encoded_footer_size {}; + encode_fixed32_le(encoded_footer_size.data(), cast_set(footer.size())); + file_bytes.insert(file_bytes.end(), encoded_footer_size.begin(), encoded_footer_size.end()); + file_bytes.insert(file_bytes.end(), {'P', 'A', 'R', '1'}); + + std::ofstream output(file_path, std::ios::binary | std::ios::trunc); + output.write(reinterpret_cast(file_bytes.data()), + cast_set(file_bytes.size())); + output.close(); + DORIS_CHECK(output.good()); +} + format::LocalColumnIndex field_projection(int32_t column_id) { return format::LocalColumnIndex {.index = column_id}; } @@ -1987,6 +2034,41 @@ TEST_F(NewParquetReaderTest, PhysicalSplitsCompleteAnAllEmptyFileWithoutChildren EXPECT_TRUE(children.empty()); } +TEST_F(NewParquetReaderTest, PositiveRowRootOnlyFileKeepsInitializedReaderForCount) { + write_parquet_file(_file_path, 2); + rewrite_as_root_only_parquet_file(_file_path); + constexpr int64_t TEST_MTIME = 717171; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto reader = create_reader(0, -1, nullptr, false, nullptr, std::nullopt, false, false, {}, + TEST_MTIME); + ASSERT_TRUE(reader->init(&state).ok()); + + FileScanSplit source; + source.range.__set_path(_file_path); + source.range.__set_start_offset(0); + source.range.__set_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_file_size(static_cast(std::filesystem::file_size(_file_path))); + source.range.__set_modification_time(TEST_MTIME); + source.is_source_split = true; + std::vector children; + bool was_split = false; + ASSERT_TRUE(reader->build_physical_splits(source, &children, &was_split).ok()); + EXPECT_FALSE(was_split); + EXPECT_TRUE(children.empty()); + + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block; + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, ROW_COUNT); +} + TEST_F(NewParquetReaderTest, InMemoryStagedFilesDeclinePhysicalSplitRefinement) { write_parquet_file(_file_path, 2); auto parquet_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false);