diff --git a/be/src/common/logging.h b/be/src/common/logging.h index 724df413dd1f46..d002de7ab2fba6 100644 --- a/be/src/common/logging.h +++ b/be/src/common/logging.h @@ -60,10 +60,17 @@ #include -#include "util/uid_util.h" +#include namespace doris { +// TaggableLogger::tag only needs these declarations; TUs that actually log a +// TUniqueId/PUniqueId have the full types from their own includes. +class TUniqueId; +class PUniqueId; +std::string print_id(const TUniqueId& id); +std::string print_id(const PUniqueId& id); + // glog doesn't allow multiple invocations of InitGoogleLogging. This method conditionally // calls InitGoogleLogging only if it hasn't been called before. bool init_glog(const char* basename); diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index b3545309cbc2eb..513383e2855df9 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -36,7 +36,7 @@ #include "core/string_ref.h" #include "core/typeid_cast.h" #include "core/types.h" -#include "storage/olap_common.h" +#include "storage/rowset_id.h" namespace doris { class SipHash; diff --git a/be/src/core/column/column_nullable.h b/be/src/core/column/column_nullable.h index c4ec70fb69abc2..bc4bc9358a1302 100644 --- a/be/src/core/column/column_nullable.h +++ b/be/src/core/column/column_nullable.h @@ -31,7 +31,7 @@ #include "core/string_ref.h" #include "core/typeid_cast.h" #include "core/types.h" -#include "storage/olap_common.h" +#include "storage/rowset_id.h" class SipHash; diff --git a/be/src/core/data_type/data_type_decimal.h b/be/src/core/data_type/data_type_decimal.h index a863316cbce4d4..7476c4d7bbd789 100644 --- a/be/src/core/data_type/data_type_decimal.h +++ b/be/src/core/data_type/data_type_decimal.h @@ -46,7 +46,7 @@ #include "core/typeid_cast.h" #include "core/types.h" #include "exec/common/arithmetic_overflow.h" -#include "storage/olap_common.h" +#include "storage/field_type.h" namespace doris { class DecimalV2Value; diff --git a/be/src/core/data_type/data_type_fixed_length_object.h b/be/src/core/data_type/data_type_fixed_length_object.h index d8ff996ab62626..896ceee5fa262a 100644 --- a/be/src/core/data_type/data_type_fixed_length_object.h +++ b/be/src/core/data_type/data_type_fixed_length_object.h @@ -31,6 +31,7 @@ #include "core/data_type_serde/data_type_string_serde.h" #include "core/field.h" #include "core/types.h" +#include "storage/field_type.h" namespace doris { diff --git a/be/src/core/data_type/data_type_ipv4.h b/be/src/core/data_type/data_type_ipv4.h index d9658047baaedf..e5b1de0e59deeb 100644 --- a/be/src/core/data_type/data_type_ipv4.h +++ b/be/src/core/data_type/data_type_ipv4.h @@ -30,7 +30,6 @@ #include "core/data_type_serde/data_type_ipv4_serde.h" #include "core/pod_array.h" #include "core/types.h" -#include "storage/olap_common.h" namespace doris { class BufferWritable; diff --git a/be/src/core/data_type/data_type_nothing.h b/be/src/core/data_type/data_type_nothing.h index f73388c954c1df..3b9ce23b49ba6b 100644 --- a/be/src/core/data_type/data_type_nothing.h +++ b/be/src/core/data_type/data_type_nothing.h @@ -35,6 +35,7 @@ #include "core/data_type_serde/data_type_serde.h" #include "core/field.h" #include "core/types.h" +#include "storage/field_type.h" namespace doris { diff --git a/be/src/core/data_type/data_type_varbinary.h b/be/src/core/data_type/data_type_varbinary.h index de50e59c8f8c77..63b4ae4af2a9bd 100644 --- a/be/src/core/data_type/data_type_varbinary.h +++ b/be/src/core/data_type/data_type_varbinary.h @@ -32,6 +32,7 @@ #include "core/data_type_serde/data_type_varbinary_serde.h" #include "core/field.h" #include "core/string_view.h" +#include "storage/field_type.h" namespace doris { class BufferWritable; diff --git a/be/src/core/data_type/primitive_type.h b/be/src/core/data_type/primitive_type.h index c236c2111d1f8b..4be148370e0a61 100644 --- a/be/src/core/data_type/primitive_type.h +++ b/be/src/core/data_type/primitive_type.h @@ -36,6 +36,7 @@ #include "exec/common/template_helpers.hpp" namespace doris { +class PathInData; template class ColumnStr; class IColumnDummy; diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index 6c5a0927e6aa97..32f3e044f5d1b5 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -35,6 +35,7 @@ #include "core/data_type_serde/orc_serde_utils.h" #include "core/string_ref.h" #include "exprs/function/function_helpers.h" +#include "storage/field_type.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" diff --git a/be/src/core/data_type_serde/data_type_ipv4_serde.h b/be/src/core/data_type_serde/data_type_ipv4_serde.h index e23695dbca504a..8fef89438c6aef 100644 --- a/be/src/core/data_type_serde/data_type_ipv4_serde.h +++ b/be/src/core/data_type_serde/data_type_ipv4_serde.h @@ -31,7 +31,6 @@ #include "core/string_ref.h" #include "core/types.h" #include "core/value/ipv4_value.h" -#include "storage/olap_common.h" namespace doris { diff --git a/be/src/core/data_type_serde/data_type_ipv6_serde.h b/be/src/core/data_type_serde/data_type_ipv6_serde.h index 226e65663f00e5..8da71eb7c954b0 100644 --- a/be/src/core/data_type_serde/data_type_ipv6_serde.h +++ b/be/src/core/data_type_serde/data_type_ipv6_serde.h @@ -31,7 +31,6 @@ #include "core/string_ref.h" #include "core/types.h" #include "core/value/ipv6_value.h" -#include "storage/olap_common.h" namespace doris { diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 2090a5d936a0e7..98396343a98770 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -29,10 +29,10 @@ #include "core/data_type/data_type.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/extended_types.h" #include "core/field.h" #include "core/string_ref.h" #include "core/types.h" -#include "storage/olap_common.h" namespace doris { class JsonbOutStream; diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index 5a95816243a29c..030f642532c018 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -28,6 +28,7 @@ #include "core/column/column_string.h" #include "core/data_type_serde/data_type_serde.h" #include "core/types.h" +#include "storage/field_type.h" namespace doris { class PValues; diff --git a/be/src/core/decimal12.h b/be/src/core/decimal12.h index 85c0dd33d66094..30dae836c4dfdd 100644 --- a/be/src/core/decimal12.h +++ b/be/src/core/decimal12.h @@ -17,11 +17,15 @@ #pragma once +#include #include +#include +#include +#include #include #include -#include "storage/utils.h" +#include "common/status.h" namespace doris { @@ -118,7 +122,7 @@ struct decimal12_t { (nullptr != sepr) ? MAX_FRAC_DIGITS_NUM - static_cast(strlen(sepr + 1)) : MAX_FRAC_DIGITS_NUM; frac_len = frac_len > 0 ? frac_len : 0; - fraction *= g_power_table[frac_len]; + fraction *= k_power_table[frac_len]; } if (sign != nullptr) { @@ -132,6 +136,11 @@ struct decimal12_t { static const int32_t FRAC_RATIO = 1000000000; static const int32_t MAX_INT_DIGITS_NUM = 18; static const int32_t MAX_FRAC_DIGITS_NUM = 9; + // 10^0 .. 10^MAX_FRAC_DIGITS_NUM, used by from_string. A private copy so + // this header does not need storage/utils.h (whose g_power_table drags in + // the whole storage layer). + static constexpr int32_t k_power_table[MAX_FRAC_DIGITS_NUM + 1] = { + 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000}; int64_t integer; int32_t fraction; diff --git a/be/src/core/extended_types.h b/be/src/core/extended_types.h index 811a2003db9618..51ace4006205b6 100644 --- a/be/src/core/extended_types.h +++ b/be/src/core/extended_types.h @@ -103,3 +103,12 @@ struct IsArithmetic { template inline constexpr bool IsArithmeticV = IsArithmetic::value; + +namespace doris { +// Canonical 128-bit integer aliases. They live here (not in a storage header) +// because bottom-of-the-world headers like core/packed_int128.h and +// util/coding.h must be able to name them without dragging in the storage +// layer; storage/olap_common.h re-exports them by including this header. +using int128_t = __int128; +using uint128_t = unsigned __int128; +} // namespace doris diff --git a/be/src/core/field.cpp b/be/src/core/field.cpp index f63aaf8227a7c6..4e86c9dce65791 100644 --- a/be/src/core/field.cpp +++ b/be/src/core/field.cpp @@ -31,6 +31,7 @@ #include "core/value/timestamptz_value.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_string.h" +#include "util/json/path_in_data.h" #include "util/var_int.h" namespace doris { diff --git a/be/src/core/packed_int128.h b/be/src/core/packed_int128.h index f8ac745b3af97a..ddd8cef7a8163a 100644 --- a/be/src/core/packed_int128.h +++ b/be/src/core/packed_int128.h @@ -17,7 +17,9 @@ #pragma once -#include "storage/olap_common.h" +#include + +#include "core/extended_types.h" namespace doris { diff --git a/be/src/core/value/variant/variant_field.cpp b/be/src/core/value/variant/variant_field.cpp index 12126ab5c7dc46..a3efc10e1897f2 100644 --- a/be/src/core/value/variant/variant_field.cpp +++ b/be/src/core/value/variant/variant_field.cpp @@ -27,6 +27,7 @@ #include "core/field.h" #include "core/value/variant/variant_parquet_encoding.h" #include "core/value/variant/variant_scalar.h" +#include "util/json/path_in_data.h" #include "util/utf8_check.h" namespace doris { @@ -285,6 +286,8 @@ void validate_variant_payload(VariantRef value) { VariantField::VariantField(std::unique_ptr data, size_t size) noexcept : _data(std::move(data)), _size(size) {} +VariantField::VariantField() noexcept = default; + VariantField::~VariantField() = default; VariantField::VariantField(VariantMap legacy) diff --git a/be/src/core/value/variant/variant_field.h b/be/src/core/value/variant/variant_field.h index 6b739d8e00cdad..b7275e3165e7c0 100644 --- a/be/src/core/value/variant/variant_field.h +++ b/be/src/core/value/variant/variant_field.h @@ -23,11 +23,11 @@ #include "core/string_ref.h" #include "core/value/variant/variant_value.h" -#include "util/json/path_in_data.h" namespace doris { struct FieldWithDataType; +class PathInData; class VariantScalarRef; using VariantMap = std::map; @@ -43,7 +43,9 @@ void validate_variant_payload(VariantRef value); // [u32 little-endian metadata_size][metadata][exactly one value]. class VariantField { public: - VariantField() noexcept = default; + // Every special member lives in the .cpp: with PathInData forward-declared, any inline + // definition would instantiate the VariantMap destructor through the _legacy deleter. + VariantField() noexcept; ~VariantField(); VariantField(const VariantField& other); diff --git a/be/src/exec/operator/schema_scan_operator.cpp b/be/src/exec/operator/schema_scan_operator.cpp index 8fab158e320988..dbafd35e129b41 100644 --- a/be/src/exec/operator/schema_scan_operator.cpp +++ b/be/src/exec/operator/schema_scan_operator.cpp @@ -19,6 +19,7 @@ #include +#include #include #include "core/column/column_nullable.h" diff --git a/be/src/exprs/function/cast/cast_to_string.h b/be/src/exprs/function/cast/cast_to_string.h index 66287f5c988a68..b4898acc450173 100644 --- a/be/src/exprs/function/cast/cast_to_string.h +++ b/be/src/exprs/function/cast/cast_to_string.h @@ -21,6 +21,7 @@ #include "core/types.h" #include "core/value/time_value.h" #include "exprs/function/cast/cast_base.h" +#include "runtime/runtime_state.h" #include "util/mysql_global.h" #include "util/to_string.h" namespace doris { diff --git a/be/src/exprs/function/cast/cast_to_timestamptz.h b/be/src/exprs/function/cast/cast_to_timestamptz.h index 5e7dcdb551e81f..65e04947bb22ce 100644 --- a/be/src/exprs/function/cast/cast_to_timestamptz.h +++ b/be/src/exprs/function/cast/cast_to_timestamptz.h @@ -27,6 +27,7 @@ #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" #include "exprs/function/cast/cast_to_timestamptz_impl.hpp" +#include "runtime/runtime_state.h" namespace doris { diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 8307a6a1c7c1b0..879f279e28ec88 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -42,15 +42,22 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" #include "core/types.h" -#include "exprs/expr_zonemap_filter.h" #include "exprs/function_context.h" #include "exprs/vexpr_fwd.h" -#include "storage/index/inverted/inverted_index_iterator.h" // IWYU pragma: keep -#include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/zone_map/zonemap_filter_result.h" namespace doris { struct InvertedIndexAnalyzerCtx; +namespace expr_zonemap { +struct DictionaryEvalContext; +struct BloomFilterEvalContext; +} // namespace expr_zonemap +using DictionaryEvalContext = expr_zonemap::DictionaryEvalContext; +using BloomFilterEvalContext = expr_zonemap::BloomFilterEvalContext; +namespace segment_v2 { +class IndexIterator; +class InvertedIndexResultBitmap; +} // namespace segment_v2 } // namespace doris namespace doris { diff --git a/be/src/exprs/function/function_encryption.cpp b/be/src/exprs/function/function_encryption.cpp index 9800b8c9f72e9a..efb388b252494d 100644 --- a/be/src/exprs/function/function_encryption.cpp +++ b/be/src/exprs/function/function_encryption.cpp @@ -315,18 +315,30 @@ struct EncryptionAndDecryptMultiImpl { auto& result_offset = result_column->get_offsets(); result_offset.resize(input_rows_count); - if ((arg_num == 5) && col_const[1] && col_const[2] && col_const[3] && col_const[4]) { - vector_const(assert_cast(argument_columns[0].get()), - argument_columns[1]->get_data_at(0), argument_columns[2]->get_data_at(0), - argument_columns[3]->get_data_at(0), input_rows_count, result_data, - result_offset, result_null_map_column->get_data(), - argument_columns[4]->get_data_at(0)); - } else if ((arg_num == 4) && col_const[1] && col_const[2] && col_const[3]) { - vector_const(assert_cast(argument_columns[0].get()), - argument_columns[1]->get_data_at(0), argument_columns[2]->get_data_at(0), - argument_columns[3]->get_data_at(0), input_rows_count, result_data, - result_offset, result_null_map_column->get_data(), StringRef()); - } else { + // if constexpr: the discarded arg_num instantiation must not index + // col_const[4] / argument_columns[4] out of bounds (-Warray-bounds). + bool all_params_const = false; + if constexpr (arg_num == 5) { + if (col_const[1] && col_const[2] && col_const[3] && col_const[4]) { + vector_const(assert_cast(argument_columns[0].get()), + argument_columns[1]->get_data_at(0), + argument_columns[2]->get_data_at(0), + argument_columns[3]->get_data_at(0), input_rows_count, result_data, + result_offset, result_null_map_column->get_data(), + argument_columns[4]->get_data_at(0)); + all_params_const = true; + } + } else if constexpr (arg_num == 4) { + if (col_const[1] && col_const[2] && col_const[3]) { + vector_const(assert_cast(argument_columns[0].get()), + argument_columns[1]->get_data_at(0), + argument_columns[2]->get_data_at(0), + argument_columns[3]->get_data_at(0), input_rows_count, result_data, + result_offset, result_null_map_column->get_data(), StringRef()); + all_params_const = true; + } + } + if (!all_params_const) { std::vector offsets_list(argument_size); std::vector chars_list(argument_size); for (size_t i = 0; i < argument_size; ++i) { diff --git a/be/src/exprs/function/function_ip.h b/be/src/exprs/function/function_ip.h index 2600041547bee1..f09825a0033d38 100644 --- a/be/src/exprs/function/function_ip.h +++ b/be/src/exprs/function/function_ip.h @@ -49,6 +49,7 @@ #include "exprs/function/function.h" #include "exprs/function/function_helpers.h" #include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/inverted_index_iterator.h" namespace doris { diff --git a/be/src/exprs/function/functions_comparison.h b/be/src/exprs/function/functions_comparison.h index 9876ecc9eac26d..1d94c66190e0d1 100644 --- a/be/src/exprs/function/functions_comparison.h +++ b/be/src/exprs/function/functions_comparison.h @@ -48,6 +48,7 @@ #include "exprs/function/functions_logical.h" #include "exprs/vexpr.h" #include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/inverted_index_iterator.h" namespace doris { /** Comparison functions: ==, !=, <, >, <=, >=. diff --git a/be/src/exprs/function/is_not_null.h b/be/src/exprs/function/is_not_null.h index 73313005f83ce4..069e13eed67bee 100644 --- a/be/src/exprs/function/is_not_null.h +++ b/be/src/exprs/function/is_not_null.h @@ -40,6 +40,7 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/function/function.h" #include "exprs/vslot_ref.h" +#include "storage/index/index_iterator.h" namespace doris { class FunctionContext; diff --git a/be/src/exprs/function/is_null.h b/be/src/exprs/function/is_null.h index eeceadedd4d322..4121ac9509a9f0 100644 --- a/be/src/exprs/function/is_null.h +++ b/be/src/exprs/function/is_null.h @@ -37,6 +37,7 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/function/function.h" #include "exprs/vslot_ref.h" +#include "storage/index/index_iterator.h" namespace doris { class FunctionContext; diff --git a/be/src/exprs/vmatch_predicate.cpp b/be/src/exprs/vmatch_predicate.cpp index d7a593f2ba4832..397189d5f8dd6a 100644 --- a/be/src/exprs/vmatch_predicate.cpp +++ b/be/src/exprs/vmatch_predicate.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include #include diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 02a722a0fa455e..91a7873d84df42 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -34,6 +34,7 @@ #include "gen_cpp/PlanNodes_types.h" #include "io/file_factory.h" #include "io/fs/file_reader_writer_fwd.h" +#include "io/io_common.h" namespace doris { class Block; diff --git a/be/src/io/fs/tracing_file_reader.h b/be/src/io/fs/tracing_file_reader.h index 48051daff5eff0..66bd003b647e55 100644 --- a/be/src/io/fs/tracing_file_reader.h +++ b/be/src/io/fs/tracing_file_reader.h @@ -18,6 +18,7 @@ #pragma once #include "common/status.h" #include "io/fs/file_reader.h" +#include "io/io_common.h" #include "runtime/runtime_profile.h" namespace doris { diff --git a/be/src/pch/pch.h b/be/src/pch/pch.h index 7acf6124cff4df..56d3c735fc005d 100644 --- a/be/src/pch/pch.h +++ b/be/src/pch/pch.h @@ -527,4 +527,3 @@ #include "common/config.h" #include "common/status.h" #include "common/version_internal.h" -#include "storage/olap_common.h" diff --git a/be/src/runtime/runtime_profile.h b/be/src/runtime/runtime_profile.h index ea5553764cf32d..1ac0d63c57340d 100644 --- a/be/src/runtime/runtime_profile.h +++ b/be/src/runtime/runtime_profile.h @@ -39,6 +39,7 @@ #include #include +#include "common/cast_set.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" #include "core/binary_cast.hpp" diff --git a/be/src/storage/field_type.h b/be/src/storage/field_type.h new file mode 100644 index 00000000000000..74d04c1db3c976 --- /dev/null +++ b/be/src/storage/field_type.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace doris { + +// Storage-engine cell types, used by TabletColumn / KeyCoder and the +// data_type traits chain. When adding a new value, also extend CppTypeTraits, +// FieldTypeTraits and the field_type_size() switch in storage/types.h. Decide how it maps to +// PrimitiveType and explicitly define its behavior in primitive_type_to_storage_field_type() and +// storage_field_type_to_primitive_type(), either by providing a mapping or by throwing. +enum class FieldType { + OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY + OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2, + OLAP_FIELD_TYPE_SMALLINT = 3, // MYSQL_TYPE_SHORT + OLAP_FIELD_TYPE_UNSIGNED_SMALLINT = 4, + OLAP_FIELD_TYPE_INT = 5, // MYSQL_TYPE_LONG + OLAP_FIELD_TYPE_UNSIGNED_INT = 6, + OLAP_FIELD_TYPE_BIGINT = 7, // MYSQL_TYPE_LONGLONG + OLAP_FIELD_TYPE_UNSIGNED_BIGINT = 8, + OLAP_FIELD_TYPE_LARGEINT = 9, + OLAP_FIELD_TYPE_FLOAT = 10, // MYSQL_TYPE_FLOAT + OLAP_FIELD_TYPE_DOUBLE = 11, // MYSQL_TYPE_DOUBLE + OLAP_FIELD_TYPE_DISCRETE_DOUBLE = 12, + OLAP_FIELD_TYPE_CHAR = 13, // MYSQL_TYPE_STRING + OLAP_FIELD_TYPE_DATE = 14, // MySQL_TYPE_NEWDATE + OLAP_FIELD_TYPE_DATETIME = 15, // MySQL_TYPE_DATETIME + OLAP_FIELD_TYPE_DECIMAL = 16, // DECIMAL, using different store format against MySQL + OLAP_FIELD_TYPE_VARCHAR = 17, + + OLAP_FIELD_TYPE_STRUCT = 18, // Struct + OLAP_FIELD_TYPE_ARRAY = 19, // ARRAY + OLAP_FIELD_TYPE_MAP = 20, // Map + OLAP_FIELD_TYPE_UNKNOWN = 21, // UNKNOW OLAP_FIELD_TYPE_STRING + OLAP_FIELD_TYPE_NONE = 22, + OLAP_FIELD_TYPE_HLL = 23, + OLAP_FIELD_TYPE_BOOL = 24, + OLAP_FIELD_TYPE_BITMAP = 25, + OLAP_FIELD_TYPE_STRING = 26, + OLAP_FIELD_TYPE_QUANTILE_STATE = 27, + OLAP_FIELD_TYPE_DATEV2 = 28, + OLAP_FIELD_TYPE_DATETIMEV2 = 29, + OLAP_FIELD_TYPE_TIMEV2 = 30, + OLAP_FIELD_TYPE_DECIMAL32 = 31, + OLAP_FIELD_TYPE_DECIMAL64 = 32, + OLAP_FIELD_TYPE_DECIMAL128I = 33, + OLAP_FIELD_TYPE_JSONB = 34, + OLAP_FIELD_TYPE_VARIANT = 35, + OLAP_FIELD_TYPE_AGG_STATE = 36, + OLAP_FIELD_TYPE_DECIMAL256 = 37, + OLAP_FIELD_TYPE_IPV4 = 38, + OLAP_FIELD_TYPE_IPV6 = 39, + OLAP_FIELD_TYPE_TIMESTAMPTZ = 40, +}; + +constexpr bool field_is_slice_type(const FieldType& field_type) { + return field_type == FieldType::OLAP_FIELD_TYPE_VARCHAR || + field_type == FieldType::OLAP_FIELD_TYPE_CHAR || + field_type == FieldType::OLAP_FIELD_TYPE_STRING; +} + +constexpr bool field_is_decimal_type(const FieldType& field_type) { + return field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256; +} + +constexpr bool field_is_numeric_type(const FieldType& field_type) { + return field_type == FieldType::OLAP_FIELD_TYPE_INT || + field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT || + field_type == FieldType::OLAP_FIELD_TYPE_BIGINT || + field_type == FieldType::OLAP_FIELD_TYPE_SMALLINT || + field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT || + field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT || + field_type == FieldType::OLAP_FIELD_TYPE_TINYINT || + field_type == FieldType::OLAP_FIELD_TYPE_DOUBLE || + field_type == FieldType::OLAP_FIELD_TYPE_FLOAT || + field_type == FieldType::OLAP_FIELD_TYPE_DATE || + field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 || + field_type == FieldType::OLAP_FIELD_TYPE_DATETIME || + field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 || + field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ || + field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I || + field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256 || + field_type == FieldType::OLAP_FIELD_TYPE_BOOL || + field_type == FieldType::OLAP_FIELD_TYPE_IPV4 || + field_type == FieldType::OLAP_FIELD_TYPE_IPV6; +} + +} // namespace doris diff --git a/be/src/storage/index/index_file_reader.h b/be/src/storage/index/index_file_reader.h index c04a8d6ec207c4..729baf03bcd6f2 100644 --- a/be/src/storage/index/index_file_reader.h +++ b/be/src/storage/index/index_file_reader.h @@ -17,8 +17,20 @@ #pragma once +// CLucene is third-party code and is not clean under -Wconversion (which +// -Wshorten-64-to-32 belongs to). Whether its first expansion lands inside +// someone else's suppressed region depends on include order, so suppress it +// deliberately here (same pattern as inverted_index_common_impl.h). +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wconversion" +#endif #include // IWYU pragma: keep #include +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + #include #include diff --git a/be/src/storage/metadata_adder.h b/be/src/storage/metadata_adder.h index c745bfafa5666f..340b894780fdb5 100644 --- a/be/src/storage/metadata_adder.h +++ b/be/src/storage/metadata_adder.h @@ -23,6 +23,7 @@ #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/runtime_profile.h" +#include "storage/rowset/rowset_fwd.h" namespace doris { diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 26706d3a5da525..cfc9b0d9ed29b9 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -38,22 +38,20 @@ #include "common/cast_set.h" #include "common/config.h" #include "common/exception.h" +#include "core/extended_types.h" #include "io/io_common.h" +#include "storage/field_type.h" #include "storage/index/inverted/inverted_index_stats.h" #include "storage/index/snii/snii_query_stats.h" #include "storage/olap_define.h" #include "storage/rowset/rowset_fwd.h" +#include "storage/rowset_id.h" #include "util/hash_util.hpp" #include "util/time.h" #include "util/uid_util.h" namespace doris { -static constexpr int64_t MAX_ROWSET_ID = 1L << 56; -static constexpr int64_t LOW_56_BITS = 0x00ffffffffffffff; - using SchemaHash = int32_t; -using int128_t = __int128; -using uint128_t = unsigned __int128; using TabletUid = UniqueId; @@ -129,54 +127,9 @@ struct TabletSize { size_t tablet_size; }; -// Storage-engine cell types, used by TabletColumn / KeyCoder and the -// data_type traits chain. When adding a new value, also extend CppTypeTraits, -// FieldTypeTraits and the field_type_size() switch in storage/types.h. Decide how it maps to -// PrimitiveType and explicitly define its behavior in primitive_type_to_storage_field_type() and -// storage_field_type_to_primitive_type(), either by providing a mapping or by throwing. -enum class FieldType { - OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY - OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2, - OLAP_FIELD_TYPE_SMALLINT = 3, // MYSQL_TYPE_SHORT - OLAP_FIELD_TYPE_UNSIGNED_SMALLINT = 4, - OLAP_FIELD_TYPE_INT = 5, // MYSQL_TYPE_LONG - OLAP_FIELD_TYPE_UNSIGNED_INT = 6, - OLAP_FIELD_TYPE_BIGINT = 7, // MYSQL_TYPE_LONGLONG - OLAP_FIELD_TYPE_UNSIGNED_BIGINT = 8, - OLAP_FIELD_TYPE_LARGEINT = 9, - OLAP_FIELD_TYPE_FLOAT = 10, // MYSQL_TYPE_FLOAT - OLAP_FIELD_TYPE_DOUBLE = 11, // MYSQL_TYPE_DOUBLE - OLAP_FIELD_TYPE_DISCRETE_DOUBLE = 12, - OLAP_FIELD_TYPE_CHAR = 13, // MYSQL_TYPE_STRING - OLAP_FIELD_TYPE_DATE = 14, // MySQL_TYPE_NEWDATE - OLAP_FIELD_TYPE_DATETIME = 15, // MySQL_TYPE_DATETIME - OLAP_FIELD_TYPE_DECIMAL = 16, // DECIMAL, using different store format against MySQL - OLAP_FIELD_TYPE_VARCHAR = 17, - - OLAP_FIELD_TYPE_STRUCT = 18, // Struct - OLAP_FIELD_TYPE_ARRAY = 19, // ARRAY - OLAP_FIELD_TYPE_MAP = 20, // Map - OLAP_FIELD_TYPE_UNKNOWN = 21, // UNKNOW OLAP_FIELD_TYPE_STRING - OLAP_FIELD_TYPE_NONE = 22, - OLAP_FIELD_TYPE_HLL = 23, - OLAP_FIELD_TYPE_BOOL = 24, - OLAP_FIELD_TYPE_BITMAP = 25, - OLAP_FIELD_TYPE_STRING = 26, - OLAP_FIELD_TYPE_QUANTILE_STATE = 27, - OLAP_FIELD_TYPE_DATEV2 = 28, - OLAP_FIELD_TYPE_DATETIMEV2 = 29, - OLAP_FIELD_TYPE_TIMEV2 = 30, - OLAP_FIELD_TYPE_DECIMAL32 = 31, - OLAP_FIELD_TYPE_DECIMAL64 = 32, - OLAP_FIELD_TYPE_DECIMAL128I = 33, - OLAP_FIELD_TYPE_JSONB = 34, - OLAP_FIELD_TYPE_VARIANT = 35, - OLAP_FIELD_TYPE_AGG_STATE = 36, - OLAP_FIELD_TYPE_DECIMAL256 = 37, - OLAP_FIELD_TYPE_IPV4 = 38, - OLAP_FIELD_TYPE_IPV6 = 39, - OLAP_FIELD_TYPE_TIMESTAMPTZ = 40, -}; +// FieldType moved to storage/field_type.h (included above) so that +// data-type headers can name storage cell types without pulling in the +// whole of olap_common.h. // Define all aggregation methods supported by TabletColumn // Note that in practice, not all types can use all the following aggregation methods @@ -204,46 +157,6 @@ enum class PushType { PUSH_NORMAL_V2 = 4, // for spark load }; -constexpr bool field_is_slice_type(const FieldType& field_type) { - return field_type == FieldType::OLAP_FIELD_TYPE_VARCHAR || - field_type == FieldType::OLAP_FIELD_TYPE_CHAR || - field_type == FieldType::OLAP_FIELD_TYPE_STRING; -} - -constexpr bool field_is_decimal_type(const FieldType& field_type) { - return field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256; -} - -constexpr bool field_is_numeric_type(const FieldType& field_type) { - return field_type == FieldType::OLAP_FIELD_TYPE_INT || - field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT || - field_type == FieldType::OLAP_FIELD_TYPE_BIGINT || - field_type == FieldType::OLAP_FIELD_TYPE_SMALLINT || - field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT || - field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT || - field_type == FieldType::OLAP_FIELD_TYPE_TINYINT || - field_type == FieldType::OLAP_FIELD_TYPE_DOUBLE || - field_type == FieldType::OLAP_FIELD_TYPE_FLOAT || - field_type == FieldType::OLAP_FIELD_TYPE_DATE || - field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 || - field_type == FieldType::OLAP_FIELD_TYPE_DATETIME || - field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 || - field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ || - field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I || - field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256 || - field_type == FieldType::OLAP_FIELD_TYPE_BOOL || - field_type == FieldType::OLAP_FIELD_TYPE_IPV4 || - field_type == FieldType::OLAP_FIELD_TYPE_IPV6; -} - // , such as <100, 110> //using Version = std::pair; @@ -510,92 +423,9 @@ using UniqueIdSet = std::set; // Column unique Id -> column id map using UniqueIdToColumnIdMap = std::map; -// 8 bit rowset id version -// 56 bit, inc number from 1 -// 128 bit backend uid, it is a uuid bit, id version -struct RowsetId { - int8_t version = 0; - int64_t hi = 0; - int64_t mi = 0; - int64_t lo = 0; - - void init(std::string_view rowset_id_str) { - // for new rowsetid its a 48 hex string - // if the len < 48, then it is an old format rowset id - if (rowset_id_str.length() < 48) [[unlikely]] { - int64_t high; - auto [_, ec] = std::from_chars(rowset_id_str.data(), - rowset_id_str.data() + rowset_id_str.length(), high); - if (ec != std::errc {}) [[unlikely]] { - if (config::force_regenerate_rowsetid_on_start_error) { - LOG(WARNING) << "failed to init rowset id: " << rowset_id_str; - high = MAX_ROWSET_ID - 1; - } else { - throw Exception( - Status::FatalError("failed to init rowset id: {}", rowset_id_str)); - } - } - init(1, high, 0, 0); - } else { - int64_t high = 0; - int64_t middle = 0; - int64_t low = 0; - from_hex(&high, rowset_id_str.substr(0, 16)); - from_hex(&middle, rowset_id_str.substr(16, 16)); - from_hex(&low, rowset_id_str.substr(32, 16)); - init(high >> 56, high & LOW_56_BITS, middle, low); - } - } - - // to compatible with old version - void init(int64_t rowset_id) { init(1, rowset_id, 0, 0); } - - void init(int64_t id_version, int64_t high, int64_t middle, int64_t low) { - version = cast_set(id_version); - if (UNLIKELY(high >= MAX_ROWSET_ID)) { - throw Exception(Status::FatalError("inc rowsetid is too large:{}", high)); - } - hi = (id_version << 56) + (high & LOW_56_BITS); - mi = middle; - lo = low; - } - - std::string to_string() const { - if (version < 2) { - return std::to_string(hi & LOW_56_BITS); - } else { - char buf[48]; - to_hex(hi, buf); - to_hex(mi, buf + 16); - to_hex(lo, buf + 32); - return {buf, 48}; - } - } - - // std::unordered_map need this api - bool operator==(const RowsetId& rhs) const { - return hi == rhs.hi && mi == rhs.mi && lo == rhs.lo; - } - - bool operator!=(const RowsetId& rhs) const { - return hi != rhs.hi || mi != rhs.mi || lo != rhs.lo; - } - - bool operator<(const RowsetId& rhs) const { - if (hi != rhs.hi) { - return hi < rhs.hi; - } else if (mi != rhs.mi) { - return mi < rhs.mi; - } else { - return lo < rhs.lo; - } - } - - friend std::ostream& operator<<(std::ostream& out, const RowsetId& rowset_id) { - out << rowset_id.to_string(); - return out; - } -}; +// RowsetId moved to storage/rowset_id.h (included above): core/column/column.h +// needs the complete type, and this way it gets it without the rest of +// olap_common.h. using RowsetIdUnorderedSet = std::unordered_set; @@ -659,18 +489,3 @@ struct VersionWithTime { } }; } // namespace doris - -// This intended to be a "good" hash function. It may change from time to time. -template <> -struct std::hash { - size_t operator()(const doris::RowsetId& rowset_id) const { - size_t seed = 0; - seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.hi, sizeof(rowset_id.hi), - seed); - seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.mi, sizeof(rowset_id.mi), - seed); - seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.lo, sizeof(rowset_id.lo), - seed); - return seed; - } -}; diff --git a/be/src/storage/rowset_id.cpp b/be/src/storage/rowset_id.cpp new file mode 100644 index 00000000000000..2040ba41ba52ff --- /dev/null +++ b/be/src/storage/rowset_id.cpp @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/rowset_id.h" + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/compiler_util.h" +#include "common/config.h" +#include "common/exception.h" +#include "common/logging.h" +#include "common/status.h" +#include "util/hash_util.hpp" +#include "util/uid_util.h" + +namespace doris { + +namespace { +constexpr int64_t MAX_ROWSET_ID = 1L << 56; +constexpr int64_t LOW_56_BITS = 0x00ffffffffffffff; +} // namespace + +void RowsetId::init(std::string_view rowset_id_str) { + // for new rowsetid its a 48 hex string + // if the len < 48, then it is an old format rowset id + if (rowset_id_str.length() < 48) [[unlikely]] { + int64_t high; + auto [_, ec] = std::from_chars(rowset_id_str.data(), + rowset_id_str.data() + rowset_id_str.length(), high); + if (ec != std::errc {}) [[unlikely]] { + if (config::force_regenerate_rowsetid_on_start_error) { + LOG(WARNING) << "failed to init rowset id: " << rowset_id_str; + high = MAX_ROWSET_ID - 1; + } else { + throw Exception(Status::FatalError("failed to init rowset id: {}", rowset_id_str)); + } + } + init(1, high, 0, 0); + } else { + int64_t high = 0; + int64_t middle = 0; + int64_t low = 0; + from_hex(&high, rowset_id_str.substr(0, 16)); + from_hex(&middle, rowset_id_str.substr(16, 16)); + from_hex(&low, rowset_id_str.substr(32, 16)); + init(high >> 56, high & LOW_56_BITS, middle, low); + } +} + +void RowsetId::init(int64_t rowset_id) { + init(1, rowset_id, 0, 0); +} + +void RowsetId::init(int64_t id_version, int64_t high, int64_t middle, int64_t low) { + version = cast_set(id_version); + if (UNLIKELY(high >= MAX_ROWSET_ID)) { + throw Exception(Status::FatalError("inc rowsetid is too large:{}", high)); + } + hi = (id_version << 56) + (high & LOW_56_BITS); + mi = middle; + lo = low; +} + +std::string RowsetId::to_string() const { + if (version < 2) { + return std::to_string(hi & LOW_56_BITS); + } else { + char buf[48]; + to_hex(hi, buf); + to_hex(mi, buf + 16); + to_hex(lo, buf + 32); + return {buf, 48}; + } +} + +std::ostream& operator<<(std::ostream& out, const RowsetId& rowset_id) { + out << rowset_id.to_string(); + return out; +} + +} // namespace doris + +size_t std::hash::operator()(const doris::RowsetId& rowset_id) const { + size_t seed = 0; + seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.hi, sizeof(rowset_id.hi), + seed); + seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.mi, sizeof(rowset_id.mi), + seed); + seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.lo, sizeof(rowset_id.lo), + seed); + return seed; +} diff --git a/be/src/storage/rowset_id.h b/be/src/storage/rowset_id.h new file mode 100644 index 00000000000000..e89960cc164e9d --- /dev/null +++ b/be/src/storage/rowset_id.h @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +// Deliberately kept dependency-free (no config/exception/logging/hex utils): +// this header is included by core/column/column.h and therefore by nearly +// every TU. The method bodies that need those facilities live in +// storage/rowset_id.cpp. + +namespace doris { + +// 8 bit rowset id version +// 56 bit, inc number from 1 +// 128 bit backend uid, it is a uuid bit, id version +struct RowsetId { + int8_t version = 0; + int64_t hi = 0; + int64_t mi = 0; + int64_t lo = 0; + + void init(std::string_view rowset_id_str); + + // to compatible with old version + void init(int64_t rowset_id); + + void init(int64_t id_version, int64_t high, int64_t middle, int64_t low); + + std::string to_string() const; + + // std::unordered_map need this api + bool operator==(const RowsetId& rhs) const { + return hi == rhs.hi && mi == rhs.mi && lo == rhs.lo; + } + + bool operator!=(const RowsetId& rhs) const { + return hi != rhs.hi || mi != rhs.mi || lo != rhs.lo; + } + + bool operator<(const RowsetId& rhs) const { + if (hi != rhs.hi) { + return hi < rhs.hi; + } else if (mi != rhs.mi) { + return mi < rhs.mi; + } else { + return lo < rhs.lo; + } + } +}; + +std::ostream& operator<<(std::ostream& out, const RowsetId& rowset_id); + +} // namespace doris + +// This intended to be a "good" hash function. It may change from time to time. +template <> +struct std::hash { + size_t operator()(const doris::RowsetId& rowset_id) const; +}; diff --git a/be/src/storage/segment/column_reader_cache.h b/be/src/storage/segment/column_reader_cache.h index 2282b56c32ffff..2bd41c83d52549 100644 --- a/be/src/storage/segment/column_reader_cache.h +++ b/be/src/storage/segment/column_reader_cache.h @@ -21,6 +21,8 @@ #include "agent/be_exec_version_manager.h" #include "core/field.h" #include "io/fs/file_reader.h" +#include "io/io_common.h" +#include "storage/olap_common.h" #include "storage/segment/stream_reader.h" #include "storage/tablet/tablet_fwd.h" #include "util/json/path_in_data.h" diff --git a/be/src/storage/segment/condition_cache.h b/be/src/storage/segment/condition_cache.h index 0ef5534a622826..41e0c0b1f73c6f 100644 --- a/be/src/storage/segment/condition_cache.h +++ b/be/src/storage/segment/condition_cache.h @@ -35,6 +35,7 @@ #include "runtime/exec_env.h" #include "runtime/memory/lru_cache_policy.h" #include "runtime/memory/mem_tracker.h" +#include "storage/rowset_id.h" #include "util/lru_cache.h" #include "util/slice.h" #include "util/time.h" diff --git a/be/src/storage/segment/rle_page.h b/be/src/storage/segment/rle_page.h index 422202be03ee4b..61a0065c65187e 100644 --- a/be/src/storage/segment/rle_page.h +++ b/be/src/storage/segment/rle_page.h @@ -18,6 +18,7 @@ #pragma once #include "common/cast_set.h" +#include "storage/field_type.h" // for FieldType #include "storage/segment/options.h" // for PageBuilderOptions/PageDecoderOptions #include "storage/segment/page_builder.h" // for PageBuilder #include "storage/segment/page_decoder.h" // for PageDecoder diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 61ed0afccfd428..6cc38ef45cf59c 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -55,6 +55,7 @@ #include "storage/schema.h" #include "storage/segment/common.h" #include "storage/segment/segment.h" +#include "util/json/path_in_data.h" #include "util/slice.h" namespace doris { @@ -78,8 +79,15 @@ struct ColumnPredicateInfo { std::string debug_string() const { std::stringstream ss; - ss << "column_name=" << column_name << ", query_op=" << query_op - << ", query_value=" << boost::join(query_values, ","); + ss << "column_name=" << column_name << ", query_op=" << query_op << ", query_value="; + bool first = true; + for (const auto& query_value : query_values) { + if (!first) { + ss << ","; + } + first = false; + ss << query_value; + } return ss.str(); } diff --git a/be/src/udf/python/python_client.h b/be/src/udf/python/python_client.h index 110fc2080b7443..62d576774e3c84 100644 --- a/be/src/udf/python/python_client.h +++ b/be/src/udf/python/python_client.h @@ -22,6 +22,7 @@ #include "arrow/flight/client.h" #include "common/status.h" #include "format/arrow/arrow_utils.h" +#include "storage/olap_define.h" #include "udf/python/python_udf_meta.h" #include "udf/python/python_udf_runtime.h" diff --git a/be/src/util/coding.h b/be/src/util/coding.h index 26d6a2f827c95f..b91fc14d4b40c7 100644 --- a/be/src/util/coding.h +++ b/be/src/util/coding.h @@ -16,8 +16,8 @@ #include #include +#include "core/extended_types.h" #include "exec/common/endian.h" -#include "storage/olap_common.h" #include "util/slice.h" namespace doris { diff --git a/be/src/util/pretty_printer.h b/be/src/util/pretty_printer.h index 48a90ee260e613..ab5aa7e7b4ba39 100644 --- a/be/src/util/pretty_printer.h +++ b/be/src/util/pretty_printer.h @@ -22,10 +22,10 @@ #include -#include #include #include #include +#include #include "core/binary_cast.hpp" #include "util/cpu_info.h" @@ -178,14 +178,14 @@ class PrettyPrinter { /// Utility method to print an iterable type to a stringstream like [v1, v2, v3] template static void print_stringList(const I& iterable, TUnit::type unit, std::stringstream* out) { - std::vector strings; + (*out) << "["; for (typename I::const_iterator it = iterable.begin(); it != iterable.end(); ++it) { - std::stringstream ss; - ss << PrettyPrinter::print(*it, unit); - strings.push_back(ss.str()); + if (it != iterable.begin()) { + (*out) << ", "; + } + (*out) << PrettyPrinter::print(*it, unit); } - - (*out) << "[" << boost::algorithm::join(strings, ", ") << "]"; + (*out) << "]"; } /// Convenience method @@ -248,14 +248,13 @@ class PrettyPrinter { /// Utility to perform integer modulo if T is integral, otherwise to use fmod(). template - static typename boost::enable_if_c::value, int64_t>::type mod( - const T& value, const int modulus) { + static std::enable_if_t::value, int64_t> mod(const T& value, + const int modulus) { return value % modulus; } template - static typename boost::enable_if_c::value, double>::type mod( - const T& value, int modulus) { + static std::enable_if_t::value, double> mod(const T& value, int modulus) { return fmod(value, 1. * modulus); } diff --git a/be/test/core/column/column_variant_v2_test.cpp b/be/test/core/column/column_variant_v2_test.cpp index 67020bdf049ee9..9df569b4facab9 100644 --- a/be/test/core/column/column_variant_v2_test.cpp +++ b/be/test/core/column/column_variant_v2_test.cpp @@ -68,6 +68,7 @@ #include "exprs/function/parse/variant_string_parse.h" #include "runtime/memory/mem_tracker.h" #include "runtime/thread_context.h" +#include "util/json/path_in_data.h" #include "util/jsonb_writer.h" #include "util/variant/variant_test_utils.h" diff --git a/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp b/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp index 6f7d06cba1211f..79d2cef83d76be 100644 --- a/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp +++ b/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp @@ -71,6 +71,7 @@ #include "common/check.h" #include "common/config.h" +#include "storage/index/inverted/inverted_index_common.h" // for DirectoryDeleter #include "storage/index/inverted/inverted_index_fs_directory.h" #include "storage/index/snii/bkd/bkd_builder.h" #include "storage/index/snii/bkd/bkd_index_block.h" diff --git a/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp b/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp index 7704c19f708959..455073febd672c 100644 --- a/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp +++ b/be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp @@ -47,6 +47,7 @@ #include "runtime/runtime_state.h" #include "runtime/thread_context.h" #include "storage/index/index_file_reader.h" +#include "storage/index/index_iterator.h" // for IndexIterator #include "storage/index/inverted/analyzer/custom_analyzer.h" #include "storage/index/inverted/common_grams/common_grams_key_codec.h" #include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" diff --git a/be/test/util/variant/variant_field_test.cpp b/be/test/util/variant/variant_field_test.cpp index 4c900dfbdd39ac..07019be9aef6f1 100644 --- a/be/test/util/variant/variant_field_test.cpp +++ b/be/test/util/variant/variant_field_test.cpp @@ -32,6 +32,7 @@ #include "core/value/variant/variant_parquet_encoding.h" #include "exprs/function/parse/variant_string_parse.h" #include "util/json/json_parser.h" +#include "util/json/path_in_data.h" #include "util/json/simd_json_parser.h" namespace doris {