From e3dbd2ae53e47179609464e358dd9ba5952f1d1d Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 10 Aug 2026 12:24:53 +0800 Subject: [PATCH 1/4] [opt](build) 2/x: Move int128 typedefs, RowsetId and FieldType out of olap_common.h Three foundational entities lived in storage/olap_common.h, which drags gen_cpp/Types_types.h, io_common, uid_util (boost/uuid) and friends into every closure that needs any of them: - int128_t/uint128_t -> core/extended_types.h. core/types.h reaches olap_common through binary_cast.hpp -> packed_int128.h purely for these two aliases, which put the storage layer underneath every TU in the codebase; util/coding.h had the same dependency for uint128_t. - RowsetId -> storage/rowset_id.h (new), a dependency-free header (cstdint/functional/iosfwd/string) safe for core/column/column.h. Method bodies, the hex helpers dependency and MAX_ROWSET_ID/LOW_56_BITS move to storage/rowset_id.cpp; std::hash::operator() is defined out of line so the header does not need hash_util.hpp. - enum FieldType (+ the three field_is_*_type helpers) -> storage/field_type.h (new), so data-type headers can name storage cell types without the rest of olap_common.h. olap_common.h includes the three light headers, so its own users are unaffected. Includer fixes: packed_int128.h / util/coding.h / data_type_number_serde.h now use extended_types.h; column.h / column_nullable.h use rowset_id.h; data_type_decimal.h uses field_type.h; data_type_ipv4.h and ipv4/ipv6 serde headers dropped a dead olap_common include; decimal12.h carries its own power table instead of pulling storage/utils.h; segment/rle_page.h takes field_type.h directly -- it names FieldType as a non-type template parameter but reached the enum only through util/coding.h, and its one live consumer (encoding_info.cpp) masks that by including olap_common.h itself. The closure-sweep natural-closure gate (300 first-party TUs, PCH stripped) surfaced every TU that only compiled through the now-cut transitive paths; fixed at the owning header: metadata_adder.h (rowset_fwd), runtime_profile.h (cast_set), column_reader_cache.h (io_common + olap_common), condition_cache.h (rowset_id), python_client.h (olap_define for DISALLOW_COPY_AND_ASSIGN), format_v2/file_reader.h and tracing_file_reader.h (io_common), four data-type headers (field_type). function_encryption.cpp: the const-args dispatch is now `if constexpr` so the arg_num==4 instantiation no longer indexes col_const[4]/argument_columns[4] out of bounds (-Warray-bounds fired once the include graph changed). Verification: closure-sweep 300/300 clean vs p3-pre baseline; compile_bench full rebuild + doris_be link green at -j6; BE UT Debug build green, 31 tests / 6 suites passed (RowsetId behavior: IdManagerTest, IdFileMapTest, StorageResourceTest, TestRowIdConversion, TxnManagerTest; registration contract: BinaryArithmeticRegistrationTest). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ND7L1ZVTJf91TBpLwYSqct --- be/src/core/column/column.h | 2 +- be/src/core/column/column_nullable.h | 2 +- be/src/core/data_type/data_type_decimal.h | 2 +- .../data_type/data_type_fixed_length_object.h | 1 + be/src/core/data_type/data_type_ipv4.h | 1 - be/src/core/data_type/data_type_nothing.h | 1 + be/src/core/data_type/data_type_varbinary.h | 1 + .../data_type_serde/data_type_ipv4_serde.h | 1 - .../data_type_serde/data_type_ipv6_serde.h | 1 - .../data_type_serde/data_type_number_serde.h | 2 +- .../data_type_serde/data_type_string_serde.h | 1 + be/src/core/decimal12.h | 13 +- be/src/core/extended_types.h | 9 + be/src/core/packed_int128.h | 4 +- be/src/exprs/function/function_encryption.cpp | 36 ++-- be/src/format_v2/file_reader.h | 1 + be/src/io/fs/tracing_file_reader.h | 1 + be/src/runtime/runtime_profile.h | 1 + be/src/storage/field_type.h | 111 ++++++++++ be/src/storage/metadata_adder.h | 1 + be/src/storage/olap_common.h | 203 +----------------- be/src/storage/rowset_id.cpp | 109 ++++++++++ be/src/storage/rowset_id.h | 79 +++++++ be/src/storage/segment/column_reader_cache.h | 2 + be/src/storage/segment/condition_cache.h | 1 + be/src/storage/segment/rle_page.h | 1 + be/src/udf/python/python_client.h | 1 + be/src/util/coding.h | 2 +- 28 files changed, 373 insertions(+), 217 deletions(-) create mode 100644 be/src/storage/field_type.h create mode 100644 be/src/storage/rowset_id.cpp create mode 100644 be/src/storage/rowset_id.h diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index b3545309cbc2eb..513383e2855df9 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -36,7 +36,7 @@ #include "core/string_ref.h" #include "core/typeid_cast.h" #include "core/types.h" -#include "storage/olap_common.h" +#include "storage/rowset_id.h" namespace doris { class SipHash; diff --git a/be/src/core/column/column_nullable.h b/be/src/core/column/column_nullable.h index c4ec70fb69abc2..bc4bc9358a1302 100644 --- a/be/src/core/column/column_nullable.h +++ b/be/src/core/column/column_nullable.h @@ -31,7 +31,7 @@ #include "core/string_ref.h" #include "core/typeid_cast.h" #include "core/types.h" -#include "storage/olap_common.h" +#include "storage/rowset_id.h" class SipHash; diff --git a/be/src/core/data_type/data_type_decimal.h b/be/src/core/data_type/data_type_decimal.h index a863316cbce4d4..7476c4d7bbd789 100644 --- a/be/src/core/data_type/data_type_decimal.h +++ b/be/src/core/data_type/data_type_decimal.h @@ -46,7 +46,7 @@ #include "core/typeid_cast.h" #include "core/types.h" #include "exec/common/arithmetic_overflow.h" -#include "storage/olap_common.h" +#include "storage/field_type.h" namespace doris { class DecimalV2Value; diff --git a/be/src/core/data_type/data_type_fixed_length_object.h b/be/src/core/data_type/data_type_fixed_length_object.h index d8ff996ab62626..896ceee5fa262a 100644 --- a/be/src/core/data_type/data_type_fixed_length_object.h +++ b/be/src/core/data_type/data_type_fixed_length_object.h @@ -31,6 +31,7 @@ #include "core/data_type_serde/data_type_string_serde.h" #include "core/field.h" #include "core/types.h" +#include "storage/field_type.h" namespace doris { diff --git a/be/src/core/data_type/data_type_ipv4.h b/be/src/core/data_type/data_type_ipv4.h index d9658047baaedf..e5b1de0e59deeb 100644 --- a/be/src/core/data_type/data_type_ipv4.h +++ b/be/src/core/data_type/data_type_ipv4.h @@ -30,7 +30,6 @@ #include "core/data_type_serde/data_type_ipv4_serde.h" #include "core/pod_array.h" #include "core/types.h" -#include "storage/olap_common.h" namespace doris { class BufferWritable; diff --git a/be/src/core/data_type/data_type_nothing.h b/be/src/core/data_type/data_type_nothing.h index f73388c954c1df..3b9ce23b49ba6b 100644 --- a/be/src/core/data_type/data_type_nothing.h +++ b/be/src/core/data_type/data_type_nothing.h @@ -35,6 +35,7 @@ #include "core/data_type_serde/data_type_serde.h" #include "core/field.h" #include "core/types.h" +#include "storage/field_type.h" namespace doris { diff --git a/be/src/core/data_type/data_type_varbinary.h b/be/src/core/data_type/data_type_varbinary.h index de50e59c8f8c77..63b4ae4af2a9bd 100644 --- a/be/src/core/data_type/data_type_varbinary.h +++ b/be/src/core/data_type/data_type_varbinary.h @@ -32,6 +32,7 @@ #include "core/data_type_serde/data_type_varbinary_serde.h" #include "core/field.h" #include "core/string_view.h" +#include "storage/field_type.h" namespace doris { class BufferWritable; diff --git a/be/src/core/data_type_serde/data_type_ipv4_serde.h b/be/src/core/data_type_serde/data_type_ipv4_serde.h index e23695dbca504a..8fef89438c6aef 100644 --- a/be/src/core/data_type_serde/data_type_ipv4_serde.h +++ b/be/src/core/data_type_serde/data_type_ipv4_serde.h @@ -31,7 +31,6 @@ #include "core/string_ref.h" #include "core/types.h" #include "core/value/ipv4_value.h" -#include "storage/olap_common.h" namespace doris { diff --git a/be/src/core/data_type_serde/data_type_ipv6_serde.h b/be/src/core/data_type_serde/data_type_ipv6_serde.h index 226e65663f00e5..8da71eb7c954b0 100644 --- a/be/src/core/data_type_serde/data_type_ipv6_serde.h +++ b/be/src/core/data_type_serde/data_type_ipv6_serde.h @@ -31,7 +31,6 @@ #include "core/string_ref.h" #include "core/types.h" #include "core/value/ipv6_value.h" -#include "storage/olap_common.h" namespace doris { diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 2090a5d936a0e7..98396343a98770 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -29,10 +29,10 @@ #include "core/data_type/data_type.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/extended_types.h" #include "core/field.h" #include "core/string_ref.h" #include "core/types.h" -#include "storage/olap_common.h" namespace doris { class JsonbOutStream; diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index 5a95816243a29c..030f642532c018 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -28,6 +28,7 @@ #include "core/column/column_string.h" #include "core/data_type_serde/data_type_serde.h" #include "core/types.h" +#include "storage/field_type.h" namespace doris { class PValues; diff --git a/be/src/core/decimal12.h b/be/src/core/decimal12.h index 85c0dd33d66094..30dae836c4dfdd 100644 --- a/be/src/core/decimal12.h +++ b/be/src/core/decimal12.h @@ -17,11 +17,15 @@ #pragma once +#include #include +#include +#include +#include #include #include -#include "storage/utils.h" +#include "common/status.h" namespace doris { @@ -118,7 +122,7 @@ struct decimal12_t { (nullptr != sepr) ? MAX_FRAC_DIGITS_NUM - static_cast(strlen(sepr + 1)) : MAX_FRAC_DIGITS_NUM; frac_len = frac_len > 0 ? frac_len : 0; - fraction *= g_power_table[frac_len]; + fraction *= k_power_table[frac_len]; } if (sign != nullptr) { @@ -132,6 +136,11 @@ struct decimal12_t { static const int32_t FRAC_RATIO = 1000000000; static const int32_t MAX_INT_DIGITS_NUM = 18; static const int32_t MAX_FRAC_DIGITS_NUM = 9; + // 10^0 .. 10^MAX_FRAC_DIGITS_NUM, used by from_string. A private copy so + // this header does not need storage/utils.h (whose g_power_table drags in + // the whole storage layer). + static constexpr int32_t k_power_table[MAX_FRAC_DIGITS_NUM + 1] = { + 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000}; int64_t integer; int32_t fraction; diff --git a/be/src/core/extended_types.h b/be/src/core/extended_types.h index 811a2003db9618..51ace4006205b6 100644 --- a/be/src/core/extended_types.h +++ b/be/src/core/extended_types.h @@ -103,3 +103,12 @@ struct IsArithmetic { template inline constexpr bool IsArithmeticV = IsArithmetic::value; + +namespace doris { +// Canonical 128-bit integer aliases. They live here (not in a storage header) +// because bottom-of-the-world headers like core/packed_int128.h and +// util/coding.h must be able to name them without dragging in the storage +// layer; storage/olap_common.h re-exports them by including this header. +using int128_t = __int128; +using uint128_t = unsigned __int128; +} // namespace doris diff --git a/be/src/core/packed_int128.h b/be/src/core/packed_int128.h index f8ac745b3af97a..ddd8cef7a8163a 100644 --- a/be/src/core/packed_int128.h +++ b/be/src/core/packed_int128.h @@ -17,7 +17,9 @@ #pragma once -#include "storage/olap_common.h" +#include + +#include "core/extended_types.h" namespace doris { diff --git a/be/src/exprs/function/function_encryption.cpp b/be/src/exprs/function/function_encryption.cpp index 9800b8c9f72e9a..efb388b252494d 100644 --- a/be/src/exprs/function/function_encryption.cpp +++ b/be/src/exprs/function/function_encryption.cpp @@ -315,18 +315,30 @@ struct EncryptionAndDecryptMultiImpl { auto& result_offset = result_column->get_offsets(); result_offset.resize(input_rows_count); - if ((arg_num == 5) && col_const[1] && col_const[2] && col_const[3] && col_const[4]) { - vector_const(assert_cast(argument_columns[0].get()), - argument_columns[1]->get_data_at(0), argument_columns[2]->get_data_at(0), - argument_columns[3]->get_data_at(0), input_rows_count, result_data, - result_offset, result_null_map_column->get_data(), - argument_columns[4]->get_data_at(0)); - } else if ((arg_num == 4) && col_const[1] && col_const[2] && col_const[3]) { - vector_const(assert_cast(argument_columns[0].get()), - argument_columns[1]->get_data_at(0), argument_columns[2]->get_data_at(0), - argument_columns[3]->get_data_at(0), input_rows_count, result_data, - result_offset, result_null_map_column->get_data(), StringRef()); - } else { + // if constexpr: the discarded arg_num instantiation must not index + // col_const[4] / argument_columns[4] out of bounds (-Warray-bounds). + bool all_params_const = false; + if constexpr (arg_num == 5) { + if (col_const[1] && col_const[2] && col_const[3] && col_const[4]) { + vector_const(assert_cast(argument_columns[0].get()), + argument_columns[1]->get_data_at(0), + argument_columns[2]->get_data_at(0), + argument_columns[3]->get_data_at(0), input_rows_count, result_data, + result_offset, result_null_map_column->get_data(), + argument_columns[4]->get_data_at(0)); + all_params_const = true; + } + } else if constexpr (arg_num == 4) { + if (col_const[1] && col_const[2] && col_const[3]) { + vector_const(assert_cast(argument_columns[0].get()), + argument_columns[1]->get_data_at(0), + argument_columns[2]->get_data_at(0), + argument_columns[3]->get_data_at(0), input_rows_count, result_data, + result_offset, result_null_map_column->get_data(), StringRef()); + all_params_const = true; + } + } + if (!all_params_const) { std::vector offsets_list(argument_size); std::vector chars_list(argument_size); for (size_t i = 0; i < argument_size; ++i) { diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 02a722a0fa455e..91a7873d84df42 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -34,6 +34,7 @@ #include "gen_cpp/PlanNodes_types.h" #include "io/file_factory.h" #include "io/fs/file_reader_writer_fwd.h" +#include "io/io_common.h" namespace doris { class Block; diff --git a/be/src/io/fs/tracing_file_reader.h b/be/src/io/fs/tracing_file_reader.h index 48051daff5eff0..66bd003b647e55 100644 --- a/be/src/io/fs/tracing_file_reader.h +++ b/be/src/io/fs/tracing_file_reader.h @@ -18,6 +18,7 @@ #pragma once #include "common/status.h" #include "io/fs/file_reader.h" +#include "io/io_common.h" #include "runtime/runtime_profile.h" namespace doris { diff --git a/be/src/runtime/runtime_profile.h b/be/src/runtime/runtime_profile.h index ea5553764cf32d..1ac0d63c57340d 100644 --- a/be/src/runtime/runtime_profile.h +++ b/be/src/runtime/runtime_profile.h @@ -39,6 +39,7 @@ #include #include +#include "common/cast_set.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" #include "core/binary_cast.hpp" diff --git a/be/src/storage/field_type.h b/be/src/storage/field_type.h new file mode 100644 index 00000000000000..74d04c1db3c976 --- /dev/null +++ b/be/src/storage/field_type.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace doris { + +// Storage-engine cell types, used by TabletColumn / KeyCoder and the +// data_type traits chain. When adding a new value, also extend CppTypeTraits, +// FieldTypeTraits and the field_type_size() switch in storage/types.h. Decide how it maps to +// PrimitiveType and explicitly define its behavior in primitive_type_to_storage_field_type() and +// storage_field_type_to_primitive_type(), either by providing a mapping or by throwing. +enum class FieldType { + OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY + OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2, + OLAP_FIELD_TYPE_SMALLINT = 3, // MYSQL_TYPE_SHORT + OLAP_FIELD_TYPE_UNSIGNED_SMALLINT = 4, + OLAP_FIELD_TYPE_INT = 5, // MYSQL_TYPE_LONG + OLAP_FIELD_TYPE_UNSIGNED_INT = 6, + OLAP_FIELD_TYPE_BIGINT = 7, // MYSQL_TYPE_LONGLONG + OLAP_FIELD_TYPE_UNSIGNED_BIGINT = 8, + OLAP_FIELD_TYPE_LARGEINT = 9, + OLAP_FIELD_TYPE_FLOAT = 10, // MYSQL_TYPE_FLOAT + OLAP_FIELD_TYPE_DOUBLE = 11, // MYSQL_TYPE_DOUBLE + OLAP_FIELD_TYPE_DISCRETE_DOUBLE = 12, + OLAP_FIELD_TYPE_CHAR = 13, // MYSQL_TYPE_STRING + OLAP_FIELD_TYPE_DATE = 14, // MySQL_TYPE_NEWDATE + OLAP_FIELD_TYPE_DATETIME = 15, // MySQL_TYPE_DATETIME + OLAP_FIELD_TYPE_DECIMAL = 16, // DECIMAL, using different store format against MySQL + OLAP_FIELD_TYPE_VARCHAR = 17, + + OLAP_FIELD_TYPE_STRUCT = 18, // Struct + OLAP_FIELD_TYPE_ARRAY = 19, // ARRAY + OLAP_FIELD_TYPE_MAP = 20, // Map + OLAP_FIELD_TYPE_UNKNOWN = 21, // UNKNOW OLAP_FIELD_TYPE_STRING + OLAP_FIELD_TYPE_NONE = 22, + OLAP_FIELD_TYPE_HLL = 23, + OLAP_FIELD_TYPE_BOOL = 24, + OLAP_FIELD_TYPE_BITMAP = 25, + OLAP_FIELD_TYPE_STRING = 26, + OLAP_FIELD_TYPE_QUANTILE_STATE = 27, + OLAP_FIELD_TYPE_DATEV2 = 28, + OLAP_FIELD_TYPE_DATETIMEV2 = 29, + OLAP_FIELD_TYPE_TIMEV2 = 30, + OLAP_FIELD_TYPE_DECIMAL32 = 31, + OLAP_FIELD_TYPE_DECIMAL64 = 32, + OLAP_FIELD_TYPE_DECIMAL128I = 33, + OLAP_FIELD_TYPE_JSONB = 34, + OLAP_FIELD_TYPE_VARIANT = 35, + OLAP_FIELD_TYPE_AGG_STATE = 36, + OLAP_FIELD_TYPE_DECIMAL256 = 37, + OLAP_FIELD_TYPE_IPV4 = 38, + OLAP_FIELD_TYPE_IPV6 = 39, + OLAP_FIELD_TYPE_TIMESTAMPTZ = 40, +}; + +constexpr bool field_is_slice_type(const FieldType& field_type) { + return field_type == FieldType::OLAP_FIELD_TYPE_VARCHAR || + field_type == FieldType::OLAP_FIELD_TYPE_CHAR || + field_type == FieldType::OLAP_FIELD_TYPE_STRING; +} + +constexpr bool field_is_decimal_type(const FieldType& field_type) { + return field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256; +} + +constexpr bool field_is_numeric_type(const FieldType& field_type) { + return field_type == FieldType::OLAP_FIELD_TYPE_INT || + field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT || + field_type == FieldType::OLAP_FIELD_TYPE_BIGINT || + field_type == FieldType::OLAP_FIELD_TYPE_SMALLINT || + field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT || + field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT || + field_type == FieldType::OLAP_FIELD_TYPE_TINYINT || + field_type == FieldType::OLAP_FIELD_TYPE_DOUBLE || + field_type == FieldType::OLAP_FIELD_TYPE_FLOAT || + field_type == FieldType::OLAP_FIELD_TYPE_DATE || + field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 || + field_type == FieldType::OLAP_FIELD_TYPE_DATETIME || + field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 || + field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ || + field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256 || + field_type == FieldType::OLAP_FIELD_TYPE_BOOL || + field_type == FieldType::OLAP_FIELD_TYPE_IPV4 || + field_type == FieldType::OLAP_FIELD_TYPE_IPV6; +} + +} // namespace doris diff --git a/be/src/storage/metadata_adder.h b/be/src/storage/metadata_adder.h index c745bfafa5666f..340b894780fdb5 100644 --- a/be/src/storage/metadata_adder.h +++ b/be/src/storage/metadata_adder.h @@ -23,6 +23,7 @@ #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/runtime_profile.h" +#include "storage/rowset/rowset_fwd.h" namespace doris { diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 26706d3a5da525..cfc9b0d9ed29b9 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -38,22 +38,20 @@ #include "common/cast_set.h" #include "common/config.h" #include "common/exception.h" +#include "core/extended_types.h" #include "io/io_common.h" +#include "storage/field_type.h" #include "storage/index/inverted/inverted_index_stats.h" #include "storage/index/snii/snii_query_stats.h" #include "storage/olap_define.h" #include "storage/rowset/rowset_fwd.h" +#include "storage/rowset_id.h" #include "util/hash_util.hpp" #include "util/time.h" #include "util/uid_util.h" namespace doris { -static constexpr int64_t MAX_ROWSET_ID = 1L << 56; -static constexpr int64_t LOW_56_BITS = 0x00ffffffffffffff; - using SchemaHash = int32_t; -using int128_t = __int128; -using uint128_t = unsigned __int128; using TabletUid = UniqueId; @@ -129,54 +127,9 @@ struct TabletSize { size_t tablet_size; }; -// Storage-engine cell types, used by TabletColumn / KeyCoder and the -// data_type traits chain. When adding a new value, also extend CppTypeTraits, -// FieldTypeTraits and the field_type_size() switch in storage/types.h. Decide how it maps to -// PrimitiveType and explicitly define its behavior in primitive_type_to_storage_field_type() and -// storage_field_type_to_primitive_type(), either by providing a mapping or by throwing. -enum class FieldType { - OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY - OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2, - OLAP_FIELD_TYPE_SMALLINT = 3, // MYSQL_TYPE_SHORT - OLAP_FIELD_TYPE_UNSIGNED_SMALLINT = 4, - OLAP_FIELD_TYPE_INT = 5, // MYSQL_TYPE_LONG - OLAP_FIELD_TYPE_UNSIGNED_INT = 6, - OLAP_FIELD_TYPE_BIGINT = 7, // MYSQL_TYPE_LONGLONG - OLAP_FIELD_TYPE_UNSIGNED_BIGINT = 8, - OLAP_FIELD_TYPE_LARGEINT = 9, - OLAP_FIELD_TYPE_FLOAT = 10, // MYSQL_TYPE_FLOAT - OLAP_FIELD_TYPE_DOUBLE = 11, // MYSQL_TYPE_DOUBLE - OLAP_FIELD_TYPE_DISCRETE_DOUBLE = 12, - OLAP_FIELD_TYPE_CHAR = 13, // MYSQL_TYPE_STRING - OLAP_FIELD_TYPE_DATE = 14, // MySQL_TYPE_NEWDATE - OLAP_FIELD_TYPE_DATETIME = 15, // MySQL_TYPE_DATETIME - OLAP_FIELD_TYPE_DECIMAL = 16, // DECIMAL, using different store format against MySQL - OLAP_FIELD_TYPE_VARCHAR = 17, - - OLAP_FIELD_TYPE_STRUCT = 18, // Struct - OLAP_FIELD_TYPE_ARRAY = 19, // ARRAY - OLAP_FIELD_TYPE_MAP = 20, // Map - OLAP_FIELD_TYPE_UNKNOWN = 21, // UNKNOW OLAP_FIELD_TYPE_STRING - OLAP_FIELD_TYPE_NONE = 22, - OLAP_FIELD_TYPE_HLL = 23, - OLAP_FIELD_TYPE_BOOL = 24, - OLAP_FIELD_TYPE_BITMAP = 25, - OLAP_FIELD_TYPE_STRING = 26, - OLAP_FIELD_TYPE_QUANTILE_STATE = 27, - OLAP_FIELD_TYPE_DATEV2 = 28, - OLAP_FIELD_TYPE_DATETIMEV2 = 29, - OLAP_FIELD_TYPE_TIMEV2 = 30, - OLAP_FIELD_TYPE_DECIMAL32 = 31, - OLAP_FIELD_TYPE_DECIMAL64 = 32, - OLAP_FIELD_TYPE_DECIMAL128I = 33, - OLAP_FIELD_TYPE_JSONB = 34, - OLAP_FIELD_TYPE_VARIANT = 35, - OLAP_FIELD_TYPE_AGG_STATE = 36, - OLAP_FIELD_TYPE_DECIMAL256 = 37, - OLAP_FIELD_TYPE_IPV4 = 38, - OLAP_FIELD_TYPE_IPV6 = 39, - OLAP_FIELD_TYPE_TIMESTAMPTZ = 40, -}; +// FieldType moved to storage/field_type.h (included above) so that +// data-type headers can name storage cell types without pulling in the +// whole of olap_common.h. // Define all aggregation methods supported by TabletColumn // Note that in practice, not all types can use all the following aggregation methods @@ -204,46 +157,6 @@ enum class PushType { PUSH_NORMAL_V2 = 4, // for spark load }; -constexpr bool field_is_slice_type(const FieldType& field_type) { - return field_type == FieldType::OLAP_FIELD_TYPE_VARCHAR || - field_type == FieldType::OLAP_FIELD_TYPE_CHAR || - field_type == FieldType::OLAP_FIELD_TYPE_STRING; -} - -constexpr bool field_is_decimal_type(const FieldType& field_type) { - return field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256; -} - -constexpr bool field_is_numeric_type(const FieldType& field_type) { - return field_type == FieldType::OLAP_FIELD_TYPE_INT || - field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT || - field_type == FieldType::OLAP_FIELD_TYPE_BIGINT || - field_type == FieldType::OLAP_FIELD_TYPE_SMALLINT || - field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT || - field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT || - field_type == FieldType::OLAP_FIELD_TYPE_TINYINT || - field_type == FieldType::OLAP_FIELD_TYPE_DOUBLE || - field_type == FieldType::OLAP_FIELD_TYPE_FLOAT || - field_type == FieldType::OLAP_FIELD_TYPE_DATE || - field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 || - field_type == FieldType::OLAP_FIELD_TYPE_DATETIME || - field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 || - field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ || - field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256 || - field_type == FieldType::OLAP_FIELD_TYPE_BOOL || - field_type == FieldType::OLAP_FIELD_TYPE_IPV4 || - field_type == FieldType::OLAP_FIELD_TYPE_IPV6; -} - // , such as <100, 110> //using Version = std::pair; @@ -510,92 +423,9 @@ using UniqueIdSet = std::set; // Column unique Id -> column id map using UniqueIdToColumnIdMap = std::map; -// 8 bit rowset id version -// 56 bit, inc number from 1 -// 128 bit backend uid, it is a uuid bit, id version -struct RowsetId { - int8_t version = 0; - int64_t hi = 0; - int64_t mi = 0; - int64_t lo = 0; - - void init(std::string_view rowset_id_str) { - // for new rowsetid its a 48 hex string - // if the len < 48, then it is an old format rowset id - if (rowset_id_str.length() < 48) [[unlikely]] { - int64_t high; - auto [_, ec] = std::from_chars(rowset_id_str.data(), - rowset_id_str.data() + rowset_id_str.length(), high); - if (ec != std::errc {}) [[unlikely]] { - if (config::force_regenerate_rowsetid_on_start_error) { - LOG(WARNING) << "failed to init rowset id: " << rowset_id_str; - high = MAX_ROWSET_ID - 1; - } else { - throw Exception( - Status::FatalError("failed to init rowset id: {}", rowset_id_str)); - } - } - init(1, high, 0, 0); - } else { - int64_t high = 0; - int64_t middle = 0; - int64_t low = 0; - from_hex(&high, rowset_id_str.substr(0, 16)); - from_hex(&middle, rowset_id_str.substr(16, 16)); - from_hex(&low, rowset_id_str.substr(32, 16)); - init(high >> 56, high & LOW_56_BITS, middle, low); - } - } - - // to compatible with old version - void init(int64_t rowset_id) { init(1, rowset_id, 0, 0); } - - void init(int64_t id_version, int64_t high, int64_t middle, int64_t low) { - version = cast_set(id_version); - if (UNLIKELY(high >= MAX_ROWSET_ID)) { - throw Exception(Status::FatalError("inc rowsetid is too large:{}", high)); - } - hi = (id_version << 56) + (high & LOW_56_BITS); - mi = middle; - lo = low; - } - - std::string to_string() const { - if (version < 2) { - return std::to_string(hi & LOW_56_BITS); - } else { - char buf[48]; - to_hex(hi, buf); - to_hex(mi, buf + 16); - to_hex(lo, buf + 32); - return {buf, 48}; - } - } - - // std::unordered_map need this api - bool operator==(const RowsetId& rhs) const { - return hi == rhs.hi && mi == rhs.mi && lo == rhs.lo; - } - - bool operator!=(const RowsetId& rhs) const { - return hi != rhs.hi || mi != rhs.mi || lo != rhs.lo; - } - - bool operator<(const RowsetId& rhs) const { - if (hi != rhs.hi) { - return hi < rhs.hi; - } else if (mi != rhs.mi) { - return mi < rhs.mi; - } else { - return lo < rhs.lo; - } - } - - friend std::ostream& operator<<(std::ostream& out, const RowsetId& rowset_id) { - out << rowset_id.to_string(); - return out; - } -}; +// RowsetId moved to storage/rowset_id.h (included above): core/column/column.h +// needs the complete type, and this way it gets it without the rest of +// olap_common.h. using RowsetIdUnorderedSet = std::unordered_set; @@ -659,18 +489,3 @@ struct VersionWithTime { } }; } // namespace doris - -// This intended to be a "good" hash function. It may change from time to time. -template <> -struct std::hash { - size_t operator()(const doris::RowsetId& rowset_id) const { - size_t seed = 0; - seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.hi, sizeof(rowset_id.hi), - seed); - seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.mi, sizeof(rowset_id.mi), - seed); - seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.lo, sizeof(rowset_id.lo), - seed); - return seed; - } -}; diff --git a/be/src/storage/rowset_id.cpp b/be/src/storage/rowset_id.cpp new file mode 100644 index 00000000000000..2040ba41ba52ff --- /dev/null +++ b/be/src/storage/rowset_id.cpp @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/rowset_id.h" + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/compiler_util.h" +#include "common/config.h" +#include "common/exception.h" +#include "common/logging.h" +#include "common/status.h" +#include "util/hash_util.hpp" +#include "util/uid_util.h" + +namespace doris { + +namespace { +constexpr int64_t MAX_ROWSET_ID = 1L << 56; +constexpr int64_t LOW_56_BITS = 0x00ffffffffffffff; +} // namespace + +void RowsetId::init(std::string_view rowset_id_str) { + // for new rowsetid its a 48 hex string + // if the len < 48, then it is an old format rowset id + if (rowset_id_str.length() < 48) [[unlikely]] { + int64_t high; + auto [_, ec] = std::from_chars(rowset_id_str.data(), + rowset_id_str.data() + rowset_id_str.length(), high); + if (ec != std::errc {}) [[unlikely]] { + if (config::force_regenerate_rowsetid_on_start_error) { + LOG(WARNING) << "failed to init rowset id: " << rowset_id_str; + high = MAX_ROWSET_ID - 1; + } else { + throw Exception(Status::FatalError("failed to init rowset id: {}", rowset_id_str)); + } + } + init(1, high, 0, 0); + } else { + int64_t high = 0; + int64_t middle = 0; + int64_t low = 0; + from_hex(&high, rowset_id_str.substr(0, 16)); + from_hex(&middle, rowset_id_str.substr(16, 16)); + from_hex(&low, rowset_id_str.substr(32, 16)); + init(high >> 56, high & LOW_56_BITS, middle, low); + } +} + +void RowsetId::init(int64_t rowset_id) { + init(1, rowset_id, 0, 0); +} + +void RowsetId::init(int64_t id_version, int64_t high, int64_t middle, int64_t low) { + version = cast_set(id_version); + if (UNLIKELY(high >= MAX_ROWSET_ID)) { + throw Exception(Status::FatalError("inc rowsetid is too large:{}", high)); + } + hi = (id_version << 56) + (high & LOW_56_BITS); + mi = middle; + lo = low; +} + +std::string RowsetId::to_string() const { + if (version < 2) { + return std::to_string(hi & LOW_56_BITS); + } else { + char buf[48]; + to_hex(hi, buf); + to_hex(mi, buf + 16); + to_hex(lo, buf + 32); + return {buf, 48}; + } +} + +std::ostream& operator<<(std::ostream& out, const RowsetId& rowset_id) { + out << rowset_id.to_string(); + return out; +} + +} // namespace doris + +size_t std::hash::operator()(const doris::RowsetId& rowset_id) const { + size_t seed = 0; + seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.hi, sizeof(rowset_id.hi), + seed); + seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.mi, sizeof(rowset_id.mi), + seed); + seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.lo, sizeof(rowset_id.lo), + seed); + return seed; +} diff --git a/be/src/storage/rowset_id.h b/be/src/storage/rowset_id.h new file mode 100644 index 00000000000000..e89960cc164e9d --- /dev/null +++ b/be/src/storage/rowset_id.h @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +// Deliberately kept dependency-free (no config/exception/logging/hex utils): +// this header is included by core/column/column.h and therefore by nearly +// every TU. The method bodies that need those facilities live in +// storage/rowset_id.cpp. + +namespace doris { + +// 8 bit rowset id version +// 56 bit, inc number from 1 +// 128 bit backend uid, it is a uuid bit, id version +struct RowsetId { + int8_t version = 0; + int64_t hi = 0; + int64_t mi = 0; + int64_t lo = 0; + + void init(std::string_view rowset_id_str); + + // to compatible with old version + void init(int64_t rowset_id); + + void init(int64_t id_version, int64_t high, int64_t middle, int64_t low); + + std::string to_string() const; + + // std::unordered_map need this api + bool operator==(const RowsetId& rhs) const { + return hi == rhs.hi && mi == rhs.mi && lo == rhs.lo; + } + + bool operator!=(const RowsetId& rhs) const { + return hi != rhs.hi || mi != rhs.mi || lo != rhs.lo; + } + + bool operator<(const RowsetId& rhs) const { + if (hi != rhs.hi) { + return hi < rhs.hi; + } else if (mi != rhs.mi) { + return mi < rhs.mi; + } else { + return lo < rhs.lo; + } + } +}; + +std::ostream& operator<<(std::ostream& out, const RowsetId& rowset_id); + +} // namespace doris + +// This intended to be a "good" hash function. It may change from time to time. +template <> +struct std::hash { + size_t operator()(const doris::RowsetId& rowset_id) const; +}; diff --git a/be/src/storage/segment/column_reader_cache.h b/be/src/storage/segment/column_reader_cache.h index 2282b56c32ffff..2bd41c83d52549 100644 --- a/be/src/storage/segment/column_reader_cache.h +++ b/be/src/storage/segment/column_reader_cache.h @@ -21,6 +21,8 @@ #include "agent/be_exec_version_manager.h" #include "core/field.h" #include "io/fs/file_reader.h" +#include "io/io_common.h" +#include "storage/olap_common.h" #include "storage/segment/stream_reader.h" #include "storage/tablet/tablet_fwd.h" #include "util/json/path_in_data.h" diff --git a/be/src/storage/segment/condition_cache.h b/be/src/storage/segment/condition_cache.h index 0ef5534a622826..41e0c0b1f73c6f 100644 --- a/be/src/storage/segment/condition_cache.h +++ b/be/src/storage/segment/condition_cache.h @@ -35,6 +35,7 @@ #include "runtime/exec_env.h" #include "runtime/memory/lru_cache_policy.h" #include "runtime/memory/mem_tracker.h" +#include "storage/rowset_id.h" #include "util/lru_cache.h" #include "util/slice.h" #include "util/time.h" diff --git a/be/src/storage/segment/rle_page.h b/be/src/storage/segment/rle_page.h index 422202be03ee4b..61a0065c65187e 100644 --- a/be/src/storage/segment/rle_page.h +++ b/be/src/storage/segment/rle_page.h @@ -18,6 +18,7 @@ #pragma once #include "common/cast_set.h" +#include "storage/field_type.h" // for FieldType #include "storage/segment/options.h" // for PageBuilderOptions/PageDecoderOptions #include "storage/segment/page_builder.h" // for PageBuilder #include "storage/segment/page_decoder.h" // for PageDecoder diff --git a/be/src/udf/python/python_client.h b/be/src/udf/python/python_client.h index 110fc2080b7443..62d576774e3c84 100644 --- a/be/src/udf/python/python_client.h +++ b/be/src/udf/python/python_client.h @@ -22,6 +22,7 @@ #include "arrow/flight/client.h" #include "common/status.h" #include "format/arrow/arrow_utils.h" +#include "storage/olap_define.h" #include "udf/python/python_udf_meta.h" #include "udf/python/python_udf_runtime.h" diff --git a/be/src/util/coding.h b/be/src/util/coding.h index 26d6a2f827c95f..b91fc14d4b40c7 100644 --- a/be/src/util/coding.h +++ b/be/src/util/coding.h @@ -16,8 +16,8 @@ #include #include +#include "core/extended_types.h" #include "exec/common/endian.h" -#include "storage/olap_common.h" #include "util/slice.h" namespace doris { From 1e34c363b76a435e769df272ccbc640fe29f4bcd Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 10 Aug 2026 13:22:20 +0800 Subject: [PATCH 2/4] [opt](build) 3/x: Cut function.h's three storage-domain include edges exprs/function/function.h is included by every scalar-function TU, and three of its includes pulled the storage/index domain (zone-map -> thrift + S3 SDK; inverted iterator -> reader -> tablet_schema + CLucene; inverted parser) into all of them. All heavy types appear only as pointers/references in the interface, so forward declarations suffice: DictionaryEvalContext/BloomFilterEvalContext are declared with their real doris::expr_zonemap:: home plus the same doris-level aliases expr_zonemap_filter.h defines; segment_v2::IndexIterator and InvertedIndexResultBitmap as classes. zonemap_filter_result.h (456 preprocessed lines) stays: ZoneMapFilterResult is returned by value. multiply.cpp natural closure: 432,112 -> 308,140 lines (-28.7%) together with the previous commit; the three edges are redundant with each other, so cutting them individually moves almost nothing -- they only pay off as a set. doris_be also shrinks 328KB from fewer transitively visible inline weak symbols in function TUs. Dependents that really use the cut headers now include them directly (closure-sweep verified, 300/300 clean): functions_comparison.h and function_ip.h materialize InvertedIndexParam (inverted_index_iterator.h); is_null.h/is_not_null.h call IndexIterator methods (index_iterator.h); cast_to_string.h/cast_to_timestamptz.h dereference RuntimeState (runtime/runtime_state.h). index_file_reader.h wraps its CLucene includes in the codebase's established -Wconversion suppression: whether CLucene's first expansion lands inside someone's suppressed region depends on include order, and this TU became the first expansion point once the function.h edge disappeared. Two be/test TUs added upstream after that sweep leaned on the same edge and get direct includes here; both are past the point where BE UT aborted on the previous push, so neither showed up as a second failure: - snii/bench/bkd_native_vs_clucene_bench_test.cpp names segment_v2::DirectoryDeleter, reached through key_coder.h -> types.h -> cast_to_timestamptz.h -> function.h -> inverted_index_iterator.h -> index_iterator.h -> ann_index_reader.h -> inverted_index_common.h. - snii/snii_index_reader_count_fallback_test.cpp calls IndexIterator methods, reached through function_multi_match.h -> function.h -> inverted_index_iterator.h -> index_iterator.h. Note the macOS BE UT job cannot catch either: it configures -DMAKE_TEST=OFF and never compiles be/test. Verification: closure-sweep 300/300 clean vs p3-pre; compile_bench rebuild (200 targets) + doris_be link green at -j6; BE UT green -- 44 tests passed (ExprZonemapFilterTest, SegmentIteratorExprZonemapTest, SegmentIteratorApplyIndexExprTest, BinaryArithmeticRegistrationTest). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ND7L1ZVTJf91TBpLwYSqct --- be/src/exprs/function/cast/cast_to_string.h | 1 + be/src/exprs/function/cast/cast_to_timestamptz.h | 1 + be/src/exprs/function/function.h | 13 ++++++++++--- be/src/exprs/function/function_ip.h | 1 + be/src/exprs/function/functions_comparison.h | 1 + be/src/exprs/function/is_not_null.h | 1 + be/src/exprs/function/is_null.h | 1 + be/src/storage/index/index_file_reader.h | 12 ++++++++++++ .../snii/bench/bkd_native_vs_clucene_bench_test.cpp | 1 + .../snii/snii_index_reader_count_fallback_test.cpp | 1 + 10 files changed, 30 insertions(+), 3 deletions(-) diff --git a/be/src/exprs/function/cast/cast_to_string.h b/be/src/exprs/function/cast/cast_to_string.h index 66287f5c988a68..b4898acc450173 100644 --- a/be/src/exprs/function/cast/cast_to_string.h +++ b/be/src/exprs/function/cast/cast_to_string.h @@ -21,6 +21,7 @@ #include "core/types.h" #include "core/value/time_value.h" #include "exprs/function/cast/cast_base.h" +#include "runtime/runtime_state.h" #include "util/mysql_global.h" #include "util/to_string.h" namespace doris { diff --git a/be/src/exprs/function/cast/cast_to_timestamptz.h b/be/src/exprs/function/cast/cast_to_timestamptz.h index 5e7dcdb551e81f..65e04947bb22ce 100644 --- a/be/src/exprs/function/cast/cast_to_timestamptz.h +++ b/be/src/exprs/function/cast/cast_to_timestamptz.h @@ -27,6 +27,7 @@ #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" #include "exprs/function/cast/cast_to_timestamptz_impl.hpp" +#include "runtime/runtime_state.h" namespace doris { diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 8307a6a1c7c1b0..879f279e28ec88 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -42,15 +42,22 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" #include "core/types.h" -#include "exprs/expr_zonemap_filter.h" #include "exprs/function_context.h" #include "exprs/vexpr_fwd.h" -#include "storage/index/inverted/inverted_index_iterator.h" // IWYU pragma: keep -#include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/zone_map/zonemap_filter_result.h" namespace doris { struct InvertedIndexAnalyzerCtx; +namespace expr_zonemap { +struct DictionaryEvalContext; +struct BloomFilterEvalContext; +} // namespace expr_zonemap +using DictionaryEvalContext = expr_zonemap::DictionaryEvalContext; +using BloomFilterEvalContext = expr_zonemap::BloomFilterEvalContext; +namespace segment_v2 { +class IndexIterator; +class InvertedIndexResultBitmap; +} // namespace segment_v2 } // namespace doris namespace doris { diff --git a/be/src/exprs/function/function_ip.h b/be/src/exprs/function/function_ip.h index 2600041547bee1..f09825a0033d38 100644 --- a/be/src/exprs/function/function_ip.h +++ b/be/src/exprs/function/function_ip.h @@ -49,6 +49,7 @@ #include "exprs/function/function.h" #include "exprs/function/function_helpers.h" #include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/inverted_index_iterator.h" namespace doris { diff --git a/be/src/exprs/function/functions_comparison.h b/be/src/exprs/function/functions_comparison.h index 9876ecc9eac26d..1d94c66190e0d1 100644 --- a/be/src/exprs/function/functions_comparison.h +++ b/be/src/exprs/function/functions_comparison.h @@ -48,6 +48,7 @@ #include "exprs/function/functions_logical.h" #include "exprs/vexpr.h" #include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/inverted_index_iterator.h" namespace doris { /** Comparison functions: ==, !=, <, >, <=, >=. diff --git a/be/src/exprs/function/is_not_null.h b/be/src/exprs/function/is_not_null.h index 73313005f83ce4..069e13eed67bee 100644 --- a/be/src/exprs/function/is_not_null.h +++ b/be/src/exprs/function/is_not_null.h @@ -40,6 +40,7 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/function/function.h" #include "exprs/vslot_ref.h" +#include "storage/index/index_iterator.h" namespace doris { class FunctionContext; diff --git a/be/src/exprs/function/is_null.h b/be/src/exprs/function/is_null.h index eeceadedd4d322..4121ac9509a9f0 100644 --- a/be/src/exprs/function/is_null.h +++ b/be/src/exprs/function/is_null.h @@ -37,6 +37,7 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/function/function.h" #include "exprs/vslot_ref.h" +#include "storage/index/index_iterator.h" namespace doris { class FunctionContext; diff --git a/be/src/storage/index/index_file_reader.h b/be/src/storage/index/index_file_reader.h index c04a8d6ec207c4..729baf03bcd6f2 100644 --- a/be/src/storage/index/index_file_reader.h +++ b/be/src/storage/index/index_file_reader.h @@ -17,8 +17,20 @@ #pragma once +// CLucene is third-party code and is not clean under -Wconversion (which +// -Wshorten-64-to-32 belongs to). Whether its first expansion lands inside +// someone else's suppressed region depends on include order, so suppress it +// deliberately here (same pattern as inverted_index_common_impl.h). +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wconversion" +#endif #include // IWYU pragma: keep #include +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + #include #include diff --git a/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp b/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp index 6f7d06cba1211f..79d2cef83d76be 100644 --- a/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp +++ b/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp @@ -71,6 +71,7 @@ #include "common/check.h" #include "common/config.h" +#include "storage/index/inverted/inverted_index_common.h" // for DirectoryDeleter #include "storage/index/inverted/inverted_index_fs_directory.h" #include "storage/index/snii/bkd/bkd_builder.h" #include "storage/index/snii/bkd/bkd_index_block.h" diff --git a/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp b/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp index 7704c19f708959..455073febd672c 100644 --- a/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp +++ b/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp @@ -47,6 +47,7 @@ #include "runtime/runtime_state.h" #include "runtime/thread_context.h" #include "storage/index/index_file_reader.h" +#include "storage/index/index_iterator.h" // for IndexIterator #include "storage/index/inverted/analyzer/custom_analyzer.h" #include "storage/index/inverted/common_grams/common_grams_key_codec.h" #include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" From 15a04b75ea90f68def8586eb3def3894f78ba9e8 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 10 Aug 2026 14:42:46 +0800 Subject: [PATCH 3/4] [opt](build) 4/x: Cut four side-door include edges under the column/type base Four small usages were each dragging a heavy header into the closure of nearly every TU: - core/data_type/primitive_type.h included util/json/path_in_data.h (-> gen_cpp/segment_v2.pb.h, ~11.6k preprocessed lines) for one `using VariantMap = std::map` alias. core/field.h used to carry the same alias and include until #65561 moved both into core/value/variant/variant_field.h, which field.h now includes -- the same edge, one hop longer. A forward declaration is enough for the alias in both headers. VariantField's std::unique_ptr member needs every special member out-of-line for that; the only inline one, the class-body defaulted default constructor, would instantiate the VariantMap destructor through the _legacy deleter, so it moves to the .cpp (declared noexcept, defaulted there). TUs that instantiate the map include the real header. - common/logging.h included util/uid_util.h (-> Types_types.h + boost/uuid -> boost/tti -> boost/function_types) so that TaggableLogger::tag could name TUniqueId/PUniqueId in an `if constexpr` and call print_id. std::is_same_v works on incomplete types, so declaring the two classes plus the two print_id overloads suffices; callers that actually log an id already have the definitions. - util/pretty_printer.h included boost/algorithm/string.hpp (~60k lines through boost/function) for one boost::algorithm::join and two boost::enable_if_c. Replaced by direct streaming (which also drops the intermediate vector materialization) and std::enable_if_t. runtime_profile.h includes pretty_printer.h, so every TU with a profile was paying for it. Dependents that really use the cut headers now include them directly (closure-sweep verified, 300/300 clean): segment_iterator.h, field.cpp, variant_field.cpp, variant_field_test.cpp and column_variant_v2_test.cpp (path_in_data.h), vmatch_predicate.cpp and schema_scan_operator.cpp (boost/algorithm/string.hpp for split/iequals). ColumnPredicateInfo::debug_string also used boost::join and now streams its std::set directly. multiply.cpp natural closure with the whole Phase 3 series so far: 432,112 -> 242,764 lines (-43.8%); aws, CLucene and segment_v2.pb.h are gone from it entirely. Verification: closure-sweep 300/300 clean vs p3-pre; compile_bench rebuild (318 targets, PCH-wide) + doris_be link green at -j6; BE UT build green, 43 tests passed across the affected suites plus 462 in the variant/profile sweep. The two SegmentFlusherFormatTest "...KeepTheirSegmentBytes" cases fail both with and without this change (verified by rebuilding the UT binary at the parent commit), so they are pre-existing and unrelated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ND7L1ZVTJf91TBpLwYSqct --- be/src/common/logging.h | 9 +++++++- be/src/core/data_type/primitive_type.h | 1 + be/src/core/field.cpp | 1 + be/src/core/value/variant/variant_field.cpp | 3 +++ be/src/core/value/variant/variant_field.h | 6 ++++-- be/src/exec/operator/schema_scan_operator.cpp | 1 + be/src/exprs/vmatch_predicate.cpp | 1 + be/src/storage/segment/segment_iterator.h | 12 +++++++++-- be/src/util/pretty_printer.h | 21 +++++++++---------- .../core/column/column_variant_v2_test.cpp | 1 + be/test/util/variant/variant_field_test.cpp | 1 + 11 files changed, 41 insertions(+), 16 deletions(-) diff --git a/be/src/common/logging.h b/be/src/common/logging.h index 724df413dd1f46..d002de7ab2fba6 100644 --- a/be/src/common/logging.h +++ b/be/src/common/logging.h @@ -60,10 +60,17 @@ #include -#include "util/uid_util.h" +#include namespace doris { +// TaggableLogger::tag only needs these declarations; TUs that actually log a +// TUniqueId/PUniqueId have the full types from their own includes. +class TUniqueId; +class PUniqueId; +std::string print_id(const TUniqueId& id); +std::string print_id(const PUniqueId& id); + // glog doesn't allow multiple invocations of InitGoogleLogging. This method conditionally // calls InitGoogleLogging only if it hasn't been called before. bool init_glog(const char* basename); diff --git a/be/src/core/data_type/primitive_type.h b/be/src/core/data_type/primitive_type.h index c236c2111d1f8b..4be148370e0a61 100644 --- a/be/src/core/data_type/primitive_type.h +++ b/be/src/core/data_type/primitive_type.h @@ -36,6 +36,7 @@ #include "exec/common/template_helpers.hpp" namespace doris { +class PathInData; template class ColumnStr; class IColumnDummy; diff --git a/be/src/core/field.cpp b/be/src/core/field.cpp index f63aaf8227a7c6..4e86c9dce65791 100644 --- a/be/src/core/field.cpp +++ b/be/src/core/field.cpp @@ -31,6 +31,7 @@ #include "core/value/timestamptz_value.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_string.h" +#include "util/json/path_in_data.h" #include "util/var_int.h" namespace doris { diff --git a/be/src/core/value/variant/variant_field.cpp b/be/src/core/value/variant/variant_field.cpp index 12126ab5c7dc46..a3efc10e1897f2 100644 --- a/be/src/core/value/variant/variant_field.cpp +++ b/be/src/core/value/variant/variant_field.cpp @@ -27,6 +27,7 @@ #include "core/field.h" #include "core/value/variant/variant_parquet_encoding.h" #include "core/value/variant/variant_scalar.h" +#include "util/json/path_in_data.h" #include "util/utf8_check.h" namespace doris { @@ -285,6 +286,8 @@ void validate_variant_payload(VariantRef value) { VariantField::VariantField(std::unique_ptr data, size_t size) noexcept : _data(std::move(data)), _size(size) {} +VariantField::VariantField() noexcept = default; + VariantField::~VariantField() = default; VariantField::VariantField(VariantMap legacy) diff --git a/be/src/core/value/variant/variant_field.h b/be/src/core/value/variant/variant_field.h index 6b739d8e00cdad..b7275e3165e7c0 100644 --- a/be/src/core/value/variant/variant_field.h +++ b/be/src/core/value/variant/variant_field.h @@ -23,11 +23,11 @@ #include "core/string_ref.h" #include "core/value/variant/variant_value.h" -#include "util/json/path_in_data.h" namespace doris { struct FieldWithDataType; +class PathInData; class VariantScalarRef; using VariantMap = std::map; @@ -43,7 +43,9 @@ void validate_variant_payload(VariantRef value); // [u32 little-endian metadata_size][metadata][exactly one value]. class VariantField { public: - VariantField() noexcept = default; + // Every special member lives in the .cpp: with PathInData forward-declared, any inline + // definition would instantiate the VariantMap destructor through the _legacy deleter. + VariantField() noexcept; ~VariantField(); VariantField(const VariantField& other); diff --git a/be/src/exec/operator/schema_scan_operator.cpp b/be/src/exec/operator/schema_scan_operator.cpp index 8fab158e320988..dbafd35e129b41 100644 --- a/be/src/exec/operator/schema_scan_operator.cpp +++ b/be/src/exec/operator/schema_scan_operator.cpp @@ -19,6 +19,7 @@ #include +#include #include #include "core/column/column_nullable.h" diff --git a/be/src/exprs/vmatch_predicate.cpp b/be/src/exprs/vmatch_predicate.cpp index d7a593f2ba4832..397189d5f8dd6a 100644 --- a/be/src/exprs/vmatch_predicate.cpp +++ b/be/src/exprs/vmatch_predicate.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include #include diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 61ed0afccfd428..6cc38ef45cf59c 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -55,6 +55,7 @@ #include "storage/schema.h" #include "storage/segment/common.h" #include "storage/segment/segment.h" +#include "util/json/path_in_data.h" #include "util/slice.h" namespace doris { @@ -78,8 +79,15 @@ struct ColumnPredicateInfo { std::string debug_string() const { std::stringstream ss; - ss << "column_name=" << column_name << ", query_op=" << query_op - << ", query_value=" << boost::join(query_values, ","); + ss << "column_name=" << column_name << ", query_op=" << query_op << ", query_value="; + bool first = true; + for (const auto& query_value : query_values) { + if (!first) { + ss << ","; + } + first = false; + ss << query_value; + } return ss.str(); } diff --git a/be/src/util/pretty_printer.h b/be/src/util/pretty_printer.h index 48a90ee260e613..ab5aa7e7b4ba39 100644 --- a/be/src/util/pretty_printer.h +++ b/be/src/util/pretty_printer.h @@ -22,10 +22,10 @@ #include -#include #include #include #include +#include #include "core/binary_cast.hpp" #include "util/cpu_info.h" @@ -178,14 +178,14 @@ class PrettyPrinter { /// Utility method to print an iterable type to a stringstream like [v1, v2, v3] template static void print_stringList(const I& iterable, TUnit::type unit, std::stringstream* out) { - std::vector strings; + (*out) << "["; for (typename I::const_iterator it = iterable.begin(); it != iterable.end(); ++it) { - std::stringstream ss; - ss << PrettyPrinter::print(*it, unit); - strings.push_back(ss.str()); + if (it != iterable.begin()) { + (*out) << ", "; + } + (*out) << PrettyPrinter::print(*it, unit); } - - (*out) << "[" << boost::algorithm::join(strings, ", ") << "]"; + (*out) << "]"; } /// Convenience method @@ -248,14 +248,13 @@ class PrettyPrinter { /// Utility to perform integer modulo if T is integral, otherwise to use fmod(). template - static typename boost::enable_if_c::value, int64_t>::type mod( - const T& value, const int modulus) { + static std::enable_if_t::value, int64_t> mod(const T& value, + const int modulus) { return value % modulus; } template - static typename boost::enable_if_c::value, double>::type mod( - const T& value, int modulus) { + static std::enable_if_t::value, double> mod(const T& value, int modulus) { return fmod(value, 1. * modulus); } diff --git a/be/test/core/column/column_variant_v2_test.cpp b/be/test/core/column/column_variant_v2_test.cpp index 67020bdf049ee9..9df569b4facab9 100644 --- a/be/test/core/column/column_variant_v2_test.cpp +++ b/be/test/core/column/column_variant_v2_test.cpp @@ -68,6 +68,7 @@ #include "exprs/function/parse/variant_string_parse.h" #include "runtime/memory/mem_tracker.h" #include "runtime/thread_context.h" +#include "util/json/path_in_data.h" #include "util/jsonb_writer.h" #include "util/variant/variant_test_utils.h" diff --git a/be/test/util/variant/variant_field_test.cpp b/be/test/util/variant/variant_field_test.cpp index 4c900dfbdd39ac..07019be9aef6f1 100644 --- a/be/test/util/variant/variant_field_test.cpp +++ b/be/test/util/variant/variant_field_test.cpp @@ -32,6 +32,7 @@ #include "core/value/variant/variant_parquet_encoding.h" #include "exprs/function/parse/variant_string_parse.h" #include "util/json/json_parser.h" +#include "util/json/path_in_data.h" #include "util/json/simd_json_parser.h" namespace doris { From 1b1db25f27764b9885784386542d5990b0167057 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 10 Aug 2026 14:53:58 +0800 Subject: [PATCH 4/4] [opt](build) 5/x: Drop storage/olap_common.h from pch.h pch.h is a dependency of every first-party object, so each doris header it pulls in becomes a tripwire: touching that header invalidates the PCH and rebuilds the whole BE, however few TUs actually name a symbol from it. storage/olap_common.h dragged in 22 more doris headers this way (io_common, uid_util, hash_util, time, olap_define, rowset_fwd, inverted_index_stats, ...). With the preceding commits every TU compiles against its natural include closure (closure-sweep --no-pch is clean for all 300 first-party TUs), so the entry can go. Third-party headers stay: absorbing those is what a PCH is for. One TU grew a bare dependency upstream since that sweep: data_type_array_serde.cpp now names FieldType (#66413 series) and only compiled through the PCH's olap_common.h; it gets the direct storage/field_type.h include. Effect on the incremental rebuild radius (ninja -t deps, first-party objects): storage/olap_common.h 318 (all, via PCH) -> 182 real dependents util/uid_util.h 318 (all, via PCH) -> 196 real dependents and the world-rebuild tripwire set shrinks from 31 doris headers to 9 (config/status/check/compiler_util/expected/version_internal/stack_util and pch.h itself). multiply.cpp preprocessed size, PCH-inclusive (the size the front end actually sees in a real build): 806k -> 717,285 lines. Verification: compile_bench full rebuild (318 targets) + doris_be link green at -j6. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ND7L1ZVTJf91TBpLwYSqct --- be/src/core/data_type_serde/data_type_array_serde.cpp | 1 + be/src/pch/pch.h | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index 6c5a0927e6aa97..32f3e044f5d1b5 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -35,6 +35,7 @@ #include "core/data_type_serde/orc_serde_utils.h" #include "core/string_ref.h" #include "exprs/function/function_helpers.h" +#include "storage/field_type.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" diff --git a/be/src/pch/pch.h b/be/src/pch/pch.h index 7acf6124cff4df..56d3c735fc005d 100644 --- a/be/src/pch/pch.h +++ b/be/src/pch/pch.h @@ -527,4 +527,3 @@ #include "common/config.h" #include "common/status.h" #include "common/version_internal.h" -#include "storage/olap_common.h"