From 3658d94679a2fb929a4bd9952918c584678f9fac Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Mon, 12 Oct 2026 20:14:29 +0900 Subject: [PATCH 1/5] Port the extended `uniqApacheHLL` from the upstream pull request Brings the implementation of https://github.com/ClickHouse/ClickHouse/pull/124667 to this branch: - the third parameter `estimator` (`'DEFAULT'` or `'COMPOSITE'`), the second computes the estimate from the registers of the sketch only, so it does not depend on the order of the inserts and of the merges; - a state that is read keeps the sketch as it was written, and a mixed state is merged in the order in which it is folded; - states of any parameters have one normalized state type; - the bytes of a `UUID` do not depend on the byte order of the host; - the function is marked as order dependent and is not registered on big-endian platforms; - the documentation tells how to hash `UInt8`, `UInt16` and `UInt32` like the C++ library. Related: https://github.com/ClickHouse/ClickHouse/pull/124667 Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: UnamedRus --- .../AggregateFunctionUniqApacheHLL.cpp | 57 +++++++++++----- .../AggregateFunctionUniqApacheHLL.h | 68 +++++++++++++------ .../registerAggregateFunctions.cpp | 6 +- .../0_stateless/04327_uniq_apache_hll.sql | 4 +- .../05136_uniq_apache_hll_corrupted_state.sh | 2 +- .../05137_uniq_apache_hll_hash_borders.sql | 2 +- .../05138_uniq_apache_hll_null_and_empty.sql | 2 +- ...39_uniq_apache_hll_nullable_state_type.sql | 2 +- ...q_apache_hll_import_keeps_sketch.reference | 10 +++ ...95_uniq_apache_hll_import_keeps_sketch.sql | 30 ++++++++ ...niq_apache_hll_state_types_unify.reference | 12 ++++ ...5296_uniq_apache_hll_state_types_unify.sql | 36 ++++++++++ .../05297_uniq_apache_hll_estimator.reference | 13 ++++ .../05297_uniq_apache_hll_estimator.sql | 52 ++++++++++++++ ..._uniq_apache_hll_order_dependent.reference | 6 ++ .../05301_uniq_apache_hll_order_dependent.sql | 14 ++++ ...02_uniq_apache_hll_unsigned_cast.reference | 17 +++++ .../05302_uniq_apache_hll_unsigned_cast.sql | 28 ++++++++ 18 files changed, 317 insertions(+), 44 deletions(-) create mode 100644 tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.reference create mode 100644 tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.sql create mode 100644 tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.reference create mode 100644 tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.sql create mode 100644 tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference create mode 100644 tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql create mode 100644 tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.reference create mode 100644 tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.sql create mode 100644 tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.reference create mode 100644 tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.sql diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp index 95144e70c22a..404991b18f24 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp @@ -21,10 +21,11 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( { uint8_t lg_config_k = 12; datasketches::target_hll_type target_type = datasketches::HLL_4; + bool use_composite_estimate = false; - if (params.size() > 2) + if (params.size() > 3) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Aggregate function {} accepts at most two parameters (lg_k, type).", name); + "Aggregate function {} accepts at most three parameters (lg_k, type, estimator).", name); if (!params.empty()) { @@ -35,7 +36,7 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( lg_config_k = static_cast(lg_k_param); } - if (params.size() == 2) + if (params.size() >= 2) { if (params[1].getType() != Field::Types::String) throw Exception(ErrorCodes::BAD_ARGUMENTS, @@ -53,6 +54,22 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( "Parameter type for aggregate function {} must be one of 'HLL_4', 'HLL_6', 'HLL_8'.", name); } + if (params.size() == 3) + { + if (params[2].getType() != Field::Types::String) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Parameter estimator for aggregate function {} must be a string.", name); + + const String estimator_param = params[2].safeGet(); + if (estimator_param == "DEFAULT") + use_composite_estimate = false; + else if (estimator_param == "COMPOSITE") + use_composite_estimate = true; + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Parameter estimator for aggregate function {} must be one of 'DEFAULT', 'COMPOSITE'.", name); + } + assertUnary(name, argument_types); const IDataType & argument_type = *argument_types[0]; @@ -60,35 +77,35 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( /// Unlike other decimals, `DateTime64` has an interoperable representation as epoch ticks. if (which.isDateTime64()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); /// Exclude wide integers: DataSketches has no portable representation for them. if (!which.isInt128() && !which.isInt256() && !which.isUInt128() && !which.isUInt256()) { AggregateFunctionPtr res(createWithNumericType( - argument_type, lg_config_k, target_type, argument_types, params)); + argument_type, lg_config_k, target_type, use_composite_estimate, argument_types, params)); if (res) return res; } if (which.isDate()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isDate32()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isDateTime()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isStringOrFixedString()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isUUID()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isIPv4()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isIPv6()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); /// For `Nullable(Nothing)` the `Null` combinator replaces this function with `nothing`, but it must be created first. if (argument_type.onlyNull()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument for aggregate function {}. Use uniq, uniqCombined or uniqHLL12 for unsupported types.", @@ -105,21 +122,27 @@ The `-State` and `-Merge` combinators exchange Apache DataSketches HLL sketches For interoperability, integers of at most 64 bits are hashed as 8-byte integers, floating-point values as IEEE-754 doubles, strings as raw bytes, `UUID` values as canonical 16 bytes, and `IPv6` addresses in network order. `Date`, `Date32`, `DateTime` and `DateTime64` use their underlying integer values. External producers must use the same unit; for example, `DateTime64(3)` uses epoch milliseconds. +An unsigned `UInt8`, `UInt16` or `UInt32` value is hashed as the same number as a 64-bit integer, as Java and Spark hash a number that they hold as a `long`. +The C++ library hashes `uint8_t`, `uint16_t` and `uint32_t` through the signed type of the same width, so for the values from 128, 32768 and 2147483648 respectively (and for `Date` from 32768, `DateTime` from 2147483648, and `IPv4` from 128.0.0.0) its sketch is another one. +To get the sketch of the C++ library, cast the argument to the signed type of the same width with a cast that wraps: `toInt8(x)`, `toInt16(x)` or `toInt32(x)`; for `Date` use `toInt16(toUInt16(x))`, for `DateTime` use `toInt32(x)`, and for `IPv4` use `toInt32(toUInt32(x))`. +Do not use `accurateCast`, which rejects these values. Unsupported types and multiple arguments are rejected. Use `uniq`, `uniqCombined` or `uniqHLL12` for those inputs. `NULL` values and empty strings are ignored, as in the Java, Python and C++ implementations and in Spark; unlike `uniq`, an empty string is not counted as a value. A `Nullable` argument gives the same state as a non-`Nullable` one, a bare DataSketches sketch, and the same state type, `AggregateFunction(uniqApacheHLL, T)`. -Merging can switch from the HIP estimator to the less accurate composite estimator, so results can depend on partitioning across threads, parts and shards. +Merging can switch from the HIP estimator to the less accurate composite estimator, and the HIP estimate of a union of small sketches depends on the order of the merges, so with the default estimator results can depend on partitioning across threads, parts and shards. +The `estimator` parameter `'COMPOSITE'` makes the result depend only on the registers of the sketch and not on the order of the inserts and of the merges, at the cost of a lower accuracy for sketches of a few hundred distinct values. It does not change the state. Merging a lower-resolution sketch permanently lowers the result's resolution, regardless of the declared `lg_k`. )"; - FunctionDocumentation::Syntax syntax = "uniqApacheHLL([lg_k, [type]])(x)"; + FunctionDocumentation::Syntax syntax = "uniqApacheHLL([lg_k, [type, [estimator]]])(x)"; FunctionDocumentation::Arguments arguments = { {"x", "Column to compute the number of distinct values of.", {"(U)Int8/16/32/64", "Enum", "BFloat16", "Float32", "Float64", "String", "FixedString", "UUID", "IPv4", "IPv6", "Date", "Date32", "DateTime", "DateTime64"}}, }; FunctionDocumentation::Parameters parameters = { {"lg_k", "Optional. Log-base-2 of the number of buckets, in range [4, 21]. Higher means better accuracy and more memory. Default: 12.", {"UInt8"}}, {"type", "Optional. Storage format of the sketch: 'HLL_4', 'HLL_6', or 'HLL_8'. Default: 'HLL_4'.", {"String"}}, + {"estimator", "Optional. The estimator used for the result: 'DEFAULT' (the estimator of the library, which can depend on the order of the inserts and of the merges) or 'COMPOSITE' (depends only on the registers of the sketch). Default: 'DEFAULT'.", {"String"}}, }; FunctionDocumentation::ReturnedValue returned_value = {"Returns the approximate number of distinct values.", {"UInt64"}}; FunctionDocumentation::Examples examples = { @@ -130,7 +153,9 @@ Merging a lower-resolution sketch permanently lowers the result's resolution, re FunctionDocumentation::Category category = FunctionDocumentation::Category::AggregateFunction; FunctionDocumentation documentation = {description, syntax, arguments, parameters, returned_value, examples, introduced_in, category}; - AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = false }; + /// The HIP estimate and the bytes of the sketch depend on the order of the updates and of the merges, so the planner must not + /// remove an `ORDER BY` below this function. + AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = true }; factory.registerFunction("uniqApacheHLL", {createAggregateFunctionUniqApacheHLL, documentation, properties}); } diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h index 76a0cd41de68..fc84f57013d8 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h @@ -4,14 +4,17 @@ #if USE_DATASKETCHES +#include #include #include #include #include #include #include +#include #include #include +#include #include #include #include @@ -72,7 +75,7 @@ class HllSketchData if (sk_union && sk_update) { sk_union->update(*sk_update); - sk_update.reset(); + sk_update = nullptr; } } @@ -88,38 +91,45 @@ class HllSketchData getSkUpdate(lg_config_k, tgt_type)->update(static_cast(data), size); } - UInt64 size() const + /// With `use_composite_estimate` the estimate is computed from the registers of the sketch only, so it does not depend on the order + /// of the inserts and of the merges. The estimate of the library is the HIP estimate while the sketch is in order, which is more + /// accurate for a small sketch, but it depends on that order. + UInt64 size(bool use_composite_estimate) const { foldUpdateIntoUnionIfNeeded(); /// Rounding preserves exact cardinalities despite floating-point error. if (sk_union) - return static_cast(std::llround(sk_union->get_estimate())); + return static_cast(std::llround(use_composite_estimate ? sk_union->get_composite_estimate() : sk_union->get_estimate())); if (sk_update) - return static_cast(std::llround(sk_update->get_estimate())); + return static_cast(std::llround(use_composite_estimate ? sk_update->get_composite_estimate() : sk_update->get_estimate())); return 0; } void merge(const HllSketchData & rhs, uint8_t lg_config_k) { - if (!rhs.sk_update && !rhs.sk_union) + const bool rhs_has_update = rhs.sk_update && !rhs.sk_update->is_empty(); + if (!rhs_has_update && !rhs.sk_union) return; datasketches::hll_union * u = getSkUnion(lg_config_k); - /// `rhs` may hold both sketches; take both without modifying it. - if (rhs.sk_update) - u->update(*rhs.sk_update); - + /// `rhs` may hold both sketches; take both without modifying it, in the order of `foldUpdateIntoUnionIfNeeded`: the union, then the + /// updates. The registers of the result do not depend on the order, but the HIP estimate and the bytes of the sketch do. /// The union accepts a sketch of any type, and `HLL_8` is its own type, so there is no re-encoding. if (rhs.sk_union) u->update(rhs.sk_union->get_result(datasketches::HLL_8)); + if (rhs_has_update) + u->update(*rhs.sk_update); + foldUpdateIntoUnionIfNeeded(); } - /// You can only call this for an empty object. - void read(ReadBuffer & in, uint8_t lg_config_k) + /// You can only call this for an empty object. The sketch is kept as it was written, with its own resolution and + /// storage type: reading a state must not lose what writing it back would then persist. It is folded into a union, + /// at the declared resolution, only when it is merged. + void read(ReadBuffer & in) { datasketches::hll_sketch::vector_bytes bytes; readVectorBinary(bytes, in); @@ -129,7 +139,7 @@ class HllSketchData try { auto sk = datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()); - getSkUnion(lg_config_k)->update(std::move(sk)); + sk_update = std::make_unique(std::move(sk)); } catch (const DB::Exception &) { @@ -169,16 +179,19 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper uint8_t lg_config_k; datasketches::target_hll_type target_type; + bool use_composite_estimate; public: AggregateFunctionUniqApacheHLL( uint8_t lg_config_k_, datasketches::target_hll_type target_type_, + bool use_composite_estimate_, const DataTypes & argument_types_, const Array & params_) : Base(argument_types_, params_, std::make_shared()) , lg_config_k(lg_config_k_) , target_type(target_type_) + , use_composite_estimate(use_composite_estimate_) { } @@ -199,7 +212,7 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper /// Other DataSketches implementations ignore empty strings, and the sketch of a mixed /// input must be the same as theirs. - if (value.size() == 0) + if (value.empty()) return; data.insertData(value.data(), value.size(), lg_config_k, target_type); @@ -210,11 +223,11 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper if constexpr (std::is_same_v) { - /// Convert the two host-order halves of a `UUID` to canonical bytes. - const UInt64 halves[2] = { - std::byteswap(UUIDHelpers::getHighBytes(value)), - std::byteswap(UUIDHelpers::getLowBytes(value)), - }; + /// The two halves of a `UUID` are numbers in host order; the canonical bytes are their big-endian + /// representation on every platform. + UInt64 halves[2] = {UUIDHelpers::getHighBytes(value), UUIDHelpers::getLowBytes(value)}; + transformEndianness(halves[0]); + transformEndianness(halves[1]); data.insertData(reinterpret_cast(halves), sizeof(halves), lg_config_k, target_type); } else if constexpr (std::is_same_v) @@ -249,6 +262,21 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper [](const auto & lhs, const auto & rhs_type) { return removeNullable(lhs)->equals(*removeNullable(rhs_type)); }); } + /// The parameters do not change what a state means, so every spelling of them is the same type wherever a common type is + /// needed (`UNION ALL`, arrays, `if`) and wherever a state is accepted by the name of its type. The default parameters + /// stand for all of them. + DataTypePtr getNormalizedStateType() const override + { + DataTypes normalized_argument_types; + normalized_argument_types.reserve(this->argument_types.size()); + for (const auto & type : this->argument_types) + normalized_argument_types.emplace_back(type->getNormalizedType()); + + AggregateFunctionProperties properties; + auto function = AggregateFunctionFactory::instance().get("uniqApacheHLL", NullsAction::EMPTY, normalized_argument_types, {}, properties); + return std::make_shared(function, normalized_argument_types, Array{}); + } + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override { this->data(place).merge(this->data(rhs), lg_config_k); @@ -261,12 +289,12 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override { - this->data(place).read(buf, lg_config_k); + this->data(place).read(buf); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - assert_cast(to).getData().push_back(this->data(place).size()); + assert_cast(to).getData().push_back(this->data(place).size(use_composite_estimate)); } }; diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index 22291d140905..d5e10987d3a9 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -63,7 +63,9 @@ void registerAggregateFunctionSumMap(AggregateFunctionFactory &); void registerAggregateFunctionsUniq(AggregateFunctionFactory &); void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &); void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &); -#if USE_DATASKETCHES +/// Apache DataSketches hashes values and serializes sketches in the byte order of the host, so `uniqApacheHLL` is registered only on +/// little-endian platforms, as if DataSketches were not built. +#if USE_DATASKETCHES && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ void registerAggregateFunctionUniqApacheHLL(AggregateFunctionFactory &); #endif void registerAggregateFunctionTopK(AggregateFunctionFactory &); @@ -179,7 +181,7 @@ void registerAggregateFunctions() registerAggregateFunctionsUniq(factory); registerAggregateFunctionUniqCombined(factory); registerAggregateFunctionUniqUpTo(factory); -#if USE_DATASKETCHES +#if USE_DATASKETCHES && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ registerAggregateFunctionUniqApacheHLL(factory); #endif registerAggregateFunctionTopK(factory); diff --git a/tests/queries/0_stateless/04327_uniq_apache_hll.sql b/tests/queries/0_stateless/04327_uniq_apache_hll.sql index cf1111734501..7d2fa6f1c145 100644 --- a/tests/queries/0_stateless/04327_uniq_apache_hll.sql +++ b/tests/queries/0_stateless/04327_uniq_apache_hll.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-cpu-s390x -- ^ DataSketches is not built in fast-test builds. SELECT 'accuracy'; @@ -37,7 +37,7 @@ SELECT uniqApacheHLL(3)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_O SELECT uniqApacheHLL(22)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } SELECT uniqApacheHLL(-1)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } SELECT uniqApacheHLL(12, 'HLL_9')(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } -SELECT uniqApacheHLL(12, 'HLL_4', 1)(number) FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT uniqApacheHLL(12, 'HLL_4', 'DEFAULT', 1)(number) FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT 'argument types'; -- Small counts stay in coupon mode, where the estimate is exact. diff --git a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh index 80c16c7e17f6..650dcae83ec2 100755 --- a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh +++ b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-cpu-s390x # no-fasttest -- compiled w/o datasketches # Malformed states must raise `CORRUPTED_DATA`, not a logical exception. diff --git a/tests/queries/0_stateless/05137_uniq_apache_hll_hash_borders.sql b/tests/queries/0_stateless/05137_uniq_apache_hll_hash_borders.sql index 589ec1b16a7d..3fa830a84749 100644 --- a/tests/queries/0_stateless/05137_uniq_apache_hll_hash_borders.sql +++ b/tests/queries/0_stateless/05137_uniq_apache_hll_hash_borders.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-cpu-s390x -- ^ DataSketches is not built in fast-test builds. -- Pins how every supported argument type is hashed, on the border values of its range. diff --git a/tests/queries/0_stateless/05138_uniq_apache_hll_null_and_empty.sql b/tests/queries/0_stateless/05138_uniq_apache_hll_null_and_empty.sql index 94282a2c346b..557c6c393606 100644 --- a/tests/queries/0_stateless/05138_uniq_apache_hll_null_and_empty.sql +++ b/tests/queries/0_stateless/05138_uniq_apache_hll_null_and_empty.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-cpu-s390x -- ^ DataSketches is not built in fast-test builds. -- `NULL` values and empty strings are ignored by the Apache DataSketches implementations in Java, Python diff --git a/tests/queries/0_stateless/05139_uniq_apache_hll_nullable_state_type.sql b/tests/queries/0_stateless/05139_uniq_apache_hll_nullable_state_type.sql index c75d1cf0153f..1c76cd9d4170 100644 --- a/tests/queries/0_stateless/05139_uniq_apache_hll_nullable_state_type.sql +++ b/tests/queries/0_stateless/05139_uniq_apache_hll_nullable_state_type.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-cpu-s390x -- ^ DataSketches is not built in fast-test builds. -- The state of a `Nullable` argument is the same bare DataSketches sketch as the state of a plain one, diff --git a/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.reference b/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.reference new file mode 100644 index 000000000000..b06e8260028f --- /dev/null +++ b/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.reference @@ -0,0 +1,10 @@ +a state read as a type with a lower lg_k and another storage type is written back unchanged +1 +the same for the default parameters +1 +the estimate of the imported state is the estimate of the sketch +1 +merging brings it down to the declared resolution +1 +an empty imported state merges to nothing +3 diff --git a/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.sql b/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.sql new file mode 100644 index 000000000000..291ca2ed2f18 --- /dev/null +++ b/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.sql @@ -0,0 +1,30 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- A sketch that is read as a state of another type, for example one with a lower `lg_k`, is kept as it was written. +-- Reading must not lose anything that writing the state back would then persist. The sketch is brought down to the +-- declared resolution only when it is merged. + +SELECT 'a state read as a type with a lower lg_k and another storage type is written back unchanged'; +WITH (SELECT hex(toString(uniqApacheHLLState(14, 'HLL_6')(number))) FROM numbers(100000)) AS h +SELECT hex(toString(CAST(unhex(h), 'AggregateFunction(uniqApacheHLL(8, \'HLL_4\'), UInt64)'))) = h; + +SELECT 'the same for the default parameters'; +WITH (SELECT hex(toString(uniqApacheHLLState(14, 'HLL_8')(number))) FROM numbers(100000)) AS h +SELECT hex(toString(CAST(unhex(h), 'AggregateFunction(uniqApacheHLL, UInt64)'))) = h; + +SELECT 'the estimate of the imported state is the estimate of the sketch'; +WITH (SELECT uniqApacheHLLState(14)(number) FROM numbers(100000)) AS s +SELECT finalizeAggregation(CAST(unhex(hex(toString(s))), 'AggregateFunction(uniqApacheHLL(8), UInt64)')) = finalizeAggregation(s); + +SELECT 'merging brings it down to the declared resolution'; +WITH (SELECT hex(toString(uniqApacheHLLState(14)(number))) FROM numbers(100000)) AS h +SELECT length(toString(uniqApacheHLLMergeState(8)(CAST(unhex(h), 'AggregateFunction(uniqApacheHLL(8), UInt64)')))) < length(unhex(h)) / 10; + +SELECT 'an empty imported state merges to nothing'; +SELECT uniqApacheHLLMerge(s) FROM +( + SELECT CAST(unhex('080201070C030C0000'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s + UNION ALL + SELECT uniqApacheHLLState(number) AS s FROM numbers(3) +); diff --git a/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.reference b/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.reference new file mode 100644 index 000000000000..b8b4ec1525f5 --- /dev/null +++ b/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.reference @@ -0,0 +1,12 @@ +UNION ALL of states with different parameters +9 +an array of such states has one element type +Array(AggregateFunction(uniqApacheHLL(8), UInt64)) +if between such states +AggregateFunction(uniqApacheHLL(8), UInt64) +a state with a Nullable argument is the same type as well +Array(AggregateFunction(uniqApacheHLL(8), UInt64)) +a different argument type is a different type +Array(Variant(AggregateFunction(uniqApacheHLL(8), UInt64), AggregateFunction(uniqApacheHLL, String))) +changing the parameters of a column is applied +AggregateFunction(uniqApacheHLL(8), UInt64) 5 diff --git a/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.sql b/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.sql new file mode 100644 index 000000000000..c84cdae87935 --- /dev/null +++ b/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.sql @@ -0,0 +1,36 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- The parameters of `uniqApacheHLL` (`lg_k` and the storage type) do not change what a state means: every sketch carries +-- its own configuration, so states of any parameters can be merged. The types of such states are therefore the same type +-- for the places that need a common type, and not only convertible with `CAST`. + +SELECT 'UNION ALL of states with different parameters'; +SELECT uniqApacheHLLMerge(s) FROM +( + SELECT uniqApacheHLLState(8)(toUInt64(number)) AS s FROM numbers(3) + UNION ALL + SELECT uniqApacheHLLState(14, 'HLL_6')(toUInt64(number + 10)) AS s FROM numbers(3) + UNION ALL + SELECT uniqApacheHLLState(toUInt64(number + 20)) AS s FROM numbers(3) +); + +SELECT 'an array of such states has one element type'; +SELECT toTypeName([uniqApacheHLLState(8)(toUInt64(1)), uniqApacheHLLState(14, 'HLL_6')(toUInt64(2)), uniqApacheHLLState(toUInt64(3))]) FROM numbers(1); + +SELECT 'if between such states'; +SELECT toTypeName(if(1, uniqApacheHLLState(8)(number), uniqApacheHLLState(14, 'HLL_6')(number))) FROM numbers(1); + +SELECT 'a state with a Nullable argument is the same type as well'; +SELECT toTypeName([uniqApacheHLLState(8)(toNullable(toUInt64(1))), uniqApacheHLLState(toUInt64(2))]) FROM numbers(1); + +SELECT 'a different argument type is a different type'; +SELECT toTypeName([uniqApacheHLLState(8)(toUInt64(1)), uniqApacheHLLState(toString(2))]) FROM numbers(1); + +SELECT 'changing the parameters of a column is applied'; +DROP TABLE IF EXISTS hll_state_types; +CREATE TABLE hll_state_types (k UInt8, s AggregateFunction(uniqApacheHLL, UInt64)) ENGINE = AggregatingMergeTree ORDER BY k; +INSERT INTO hll_state_types SELECT 1, uniqApacheHLLState(number) FROM numbers(5); +ALTER TABLE hll_state_types MODIFY COLUMN s AggregateFunction(uniqApacheHLL(8), UInt64); +SELECT toTypeName(s), uniqApacheHLLMerge(8)(s) FROM hll_state_types GROUP BY toTypeName(s); +DROP TABLE hll_state_types; diff --git a/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference new file mode 100644 index 000000000000..6b1729aa5df0 --- /dev/null +++ b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference @@ -0,0 +1,13 @@ +the composite estimate of a merged state does not depend on the order of the merges +1 +the estimator does not change the state, and states of any estimator have one type where a common type is needed +1 +2 +a small sketch has the same estimate +100 100 +the composite estimate of a sketch that is built by inserts does not depend on the order of the inserts +1 +the same for a state that is finalized, and for a state that is read +1 +1 +wrong parameters diff --git a/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql new file mode 100644 index 000000000000..f71c9ab3c93b --- /dev/null +++ b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql @@ -0,0 +1,52 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- The HIP estimate of the library depends on the order of the inserts and of the merges. The third parameter, `estimator`, chooses +-- the estimate: `DEFAULT` is the one of the library, `COMPOSITE` is computed from the registers of the sketch only, so it does not +-- depend on the order. Only the second is checked here, because the first one differs between orders only sometimes. + +SET max_threads = 1; + +-- 8 sketches of 100 values each, neighbours overlap by half, 450 distinct values in total, merged in three orders + +SELECT 'the composite estimate of a merged state does not depend on the order of the merges'; +SELECT count() = 3 AND uniqExact(r) = 1 AND any(r) BETWEEN 430 AND 470 FROM +( + SELECT uniqApacheHLLMerge(11, 'HLL_8', 'COMPOSITE')(s) AS r FROM (SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(0) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(1) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(2) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(3) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(4) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(5) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(6) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(7) * 50 + number) AS s FROM numbers(100)) UNION ALL + SELECT uniqApacheHLLMerge(11, 'HLL_8', 'COMPOSITE')(s) AS r FROM (SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(7) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(6) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(5) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(4) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(3) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(2) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(1) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(0) * 50 + number) AS s FROM numbers(100)) UNION ALL + SELECT uniqApacheHLLMerge(11, 'HLL_8', 'COMPOSITE')(s) AS r FROM (SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(3) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(6) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(0) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(5) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(1) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(7) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(2) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(4) * 50 + number) AS s FROM numbers(100)) +); + +SELECT 'the estimator does not change the state, and states of any estimator have one type where a common type is needed'; +SELECT hex(toString(uniqApacheHLLState(11, 'HLL_8', 'COMPOSITE')(number))) = hex(toString(uniqApacheHLLState(11, 'HLL_8', 'DEFAULT')(number))) FROM numbers(1000); +SELECT length([uniqApacheHLLState(8, 'HLL_6', 'COMPOSITE')(toUInt64(1)), uniqApacheHLLState(toUInt64(2))]); + +SELECT 'a small sketch has the same estimate'; +SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(number), uniqApacheHLL(11, 'HLL_8', 'DEFAULT')(number) FROM numbers(100); + +SELECT 'the composite estimate of a sketch that is built by inserts does not depend on the order of the inserts'; +SELECT count() = 3 AND uniqExact(r) = 1 AND any(r) BETWEEN 2800 AND 3200 FROM +( + SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(x) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY x) UNION ALL + SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(x) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY -x) UNION ALL + SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(x) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY cityHash64(x)) +); + +SELECT 'the same for a state that is finalized, and for a state that is read'; +SELECT count() = 3 AND uniqExact(r) = 1 FROM +( + SELECT finalizeAggregation(uniqApacheHLLState(11, 'HLL_8', 'COMPOSITE')(x)) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY x) UNION ALL + SELECT finalizeAggregation(uniqApacheHLLState(11, 'HLL_8', 'COMPOSITE')(x)) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY -x) UNION ALL + SELECT finalizeAggregation(uniqApacheHLLState(11, 'HLL_8', 'COMPOSITE')(x)) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY cityHash64(x)) +); +SELECT count() = 3 AND uniqExact(r) = 1 FROM +( + SELECT finalizeAggregation(CAST(toString(uniqApacheHLLState(11, 'HLL_8')(x)), 'AggregateFunction(uniqApacheHLL(11, \'HLL_8\', \'COMPOSITE\'), UInt64)')) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY x) UNION ALL + SELECT finalizeAggregation(CAST(toString(uniqApacheHLLState(11, 'HLL_8')(x)), 'AggregateFunction(uniqApacheHLL(11, \'HLL_8\', \'COMPOSITE\'), UInt64)')) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY -x) UNION ALL + SELECT finalizeAggregation(CAST(toString(uniqApacheHLLState(11, 'HLL_8')(x)), 'AggregateFunction(uniqApacheHLL(11, \'HLL_8\', \'COMPOSITE\'), UInt64)')) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY cityHash64(x)) +); + +SELECT 'wrong parameters'; +SELECT uniqApacheHLL(11, 'HLL_8', 'HIP')(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } +SELECT uniqApacheHLL(11, 'HLL_8', 1)(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } +SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE', 1)(number) FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } diff --git a/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.reference b/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.reference new file mode 100644 index 000000000000..c21307d22faa --- /dev/null +++ b/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.reference @@ -0,0 +1,6 @@ +uniq 0 +uniqApacheHLL 1 +uniqApacheHLL params 1 +uniqApacheHLLState 1 +uniqApacheHLLMerge 1 +uniqApacheHLLIf 1 diff --git a/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.sql b/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.sql new file mode 100644 index 000000000000..b16d71d9ad40 --- /dev/null +++ b/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.sql @@ -0,0 +1,14 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- The HIP estimate and the bytes of a `uniqApacheHLL` sketch depend on the order of the updates and of the merges, so the planner must keep an `ORDER BY` below it, +-- also below its `-State`, `-Merge` and `-If` forms. `uniq` is the control: its result does not depend on the order, so the sorting below it is removed. + +SET query_plan_remove_redundant_sorting = 1; +SET optimize_aggregators_of_group_by_keys = 0; +SELECT 'uniq', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniq(x) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); +SELECT 'uniqApacheHLL', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLL(x) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); +SELECT 'uniqApacheHLL params', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLL(14, 'HLL_8')(x) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); +SELECT 'uniqApacheHLLState', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLLState(x) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); +SELECT 'uniqApacheHLLMerge', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLLMerge(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(10) GROUP BY number % 3 ORDER BY number % 3 DESC)); +SELECT 'uniqApacheHLLIf', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLLIf(x, x > 2) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); diff --git a/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.reference b/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.reference new file mode 100644 index 000000000000..6664c4a9c9d2 --- /dev/null +++ b/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.reference @@ -0,0 +1,17 @@ +by value, as Java and Spark hash a long +1 +1 +1 +a cast to the signed type of the same width gives the sketch of the C++ library +1 +1 +1 +1 +1 +1 +the two are different sketches for these values +1 +the values below the signed range are not affected +1 +1 +a cast that checks the range rejects these values diff --git a/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.sql b/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.sql new file mode 100644 index 000000000000..d114cdbf49be --- /dev/null +++ b/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.sql @@ -0,0 +1,28 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- An unsigned `UInt8`, `UInt16` or `UInt32` value is hashed as the same number as a 64-bit integer. The C++ library hashes such a value +-- through the signed type of the same width, and its sketch is what the documented casts to that type give. + +SELECT 'by value, as Java and Spark hash a long'; +SELECT hex(toString(uniqApacheHLLState(toUInt32(3000000000)))) = hex(toString(uniqApacheHLLState(toInt64(3000000000)))); +SELECT hex(toString(uniqApacheHLLState(toUInt16(50000)))) = hex(toString(uniqApacheHLLState(toInt64(50000)))); +SELECT hex(toString(uniqApacheHLLState(toUInt8(200)))) = hex(toString(uniqApacheHLLState(toInt64(200)))); + +SELECT 'a cast to the signed type of the same width gives the sketch of the C++ library'; +SELECT hex(toString(uniqApacheHLLState(toInt32(toUInt32(3000000000))))) = hex(toString(uniqApacheHLLState(toInt64(-1294967296)))); +SELECT hex(toString(uniqApacheHLLState(toInt16(toUInt16(50000))))) = hex(toString(uniqApacheHLLState(toInt64(-15536)))); +SELECT hex(toString(uniqApacheHLLState(toInt8(toUInt8(200))))) = hex(toString(uniqApacheHLLState(toInt64(-56)))); +SELECT hex(toString(uniqApacheHLLState(toInt32(toUInt32(toIPv4('192.168.0.1')))))) = hex(toString(uniqApacheHLLState(toInt64(-1062731775)))); +SELECT hex(toString(uniqApacheHLLState(toInt32(toDateTime(2147483648, 'UTC'))))) = hex(toString(uniqApacheHLLState(toInt64(-2147483648)))); +SELECT hex(toString(uniqApacheHLLState(toInt16(toUInt16(toDate(32768)))))) = hex(toString(uniqApacheHLLState(toInt64(-32768)))); + +SELECT 'the two are different sketches for these values'; +SELECT hex(toString(uniqApacheHLLState(toUInt32(3000000000)))) != hex(toString(uniqApacheHLLState(toInt32(toUInt32(3000000000))))); + +SELECT 'the values below the signed range are not affected'; +SELECT hex(toString(uniqApacheHLLState(toUInt32(2147483647)))) = hex(toString(uniqApacheHLLState(toInt32(2147483647)))); +SELECT hex(toString(uniqApacheHLLState(toIPv4('10.0.0.1')))) = hex(toString(uniqApacheHLLState(toInt32(toUInt32(toIPv4('10.0.0.1')))))); + +SELECT 'a cast that checks the range rejects these values'; +SELECT accurateCast(toUInt32(3000000000), 'Int32'); -- { serverError CANNOT_CONVERT_TYPE } From ab14f316bd3e9f76948e7fc951514e0e1742a47f Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Mon, 12 Oct 2026 20:14:29 +0900 Subject: [PATCH 2/5] Add `addVarUIntLengthPrefix`, `removeVarUIntLengthPrefix` and `toVarUInt` A sketch that another system writes, for example Spark into Parquet, has no length in front of it, while the state of `uniqApacheHLL` stores its sketch as a `VarUInt` length and the bytes. `CAST` of such a sketch to `AggregateFunction(uniqApacheHLL, ...)` fails with `CORRUPTED_DATA`, and `toString` of a state gives bytes that the other system cannot read. `addVarUIntLengthPrefix` puts the length of a string in front of it, `removeVarUIntLengthPrefix` takes it off and throws an exception when the length is not complete or is not the number of the bytes that follow, and `toVarUInt` encodes one unsigned number. The length is read as the `RowBinary` reader reads it. Related: https://github.com/ClickHouse/ClickHouse/pull/124667 Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: UnamedRus --- src/Functions/varUIntLengthPrefix.cpp | 264 ++++++++++++++++++ .../05298_var_uint_length_prefix.reference | 20 ++ .../05298_var_uint_length_prefix.sql | 37 +++ ...nt_length_prefix_uniq_apache_hll.reference | 11 + ...var_uint_length_prefix_uniq_apache_hll.sql | 26 ++ ...int_length_prefix_without_length.reference | 2 + ...0_var_uint_length_prefix_without_length.sh | 20 ++ 7 files changed, 380 insertions(+) create mode 100644 src/Functions/varUIntLengthPrefix.cpp create mode 100644 tests/queries/0_stateless/05298_var_uint_length_prefix.reference create mode 100644 tests/queries/0_stateless/05298_var_uint_length_prefix.sql create mode 100644 tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.reference create mode 100644 tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.sql create mode 100644 tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.reference create mode 100755 tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.sh diff --git a/src/Functions/varUIntLengthPrefix.cpp b/src/Functions/varUIntLengthPrefix.cpp new file mode 100644 index 000000000000..593a97840d98 --- /dev/null +++ b/src/Functions/varUIntLengthPrefix.cpp @@ -0,0 +1,264 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; +} + +namespace +{ + +/// A string that is stored with its length in front of it, as `RowBinary` and `Native` store a `String` and as the state of +/// some aggregate functions is stored, has the length as a `VarUInt`: an unsigned LEB128 number, 7 bits per byte, the lowest +/// bits first, the high bit of every byte but the last one is set. The functions below convert between such a string and its +/// contents, and encode one number. + +void checkIsStringOrFixedString(const String & function_name, const DataTypePtr & type) +{ + if (!isStringOrFixedString(type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}, expected String or FixedString", type->getName(), function_name); +} + +class FunctionAddVarUIntLengthPrefix final : public IFunction +{ +public: + static constexpr auto name = "addVarUIntLengthPrefix"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + checkIsStringOrFixedString(getName(), arguments[0]); + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const IColumn & column = *arguments[0].column; + + auto result = ColumnString::create(); + auto & chars = result->getChars(); + auto & offsets = result->getOffsets(); + offsets.resize(input_rows_count); + chars.reserve(column.byteSize() + input_rows_count * 2); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const std::string_view value = column.getDataAt(row); + const size_t prefix_size = getLengthOfVarUInt(value.size()); + + const size_t old_size = chars.size(); + chars.resize(old_size + prefix_size + value.size()); + + char * pos = reinterpret_cast(chars.data() + old_size); + pos = writeVarUInt(value.size(), pos); + if (!value.empty()) + memcpy(pos, value.data(), value.size()); + + offsets[row] = chars.size(); + } + + return result; + } +}; + +class FunctionRemoveVarUIntLengthPrefix final : public IFunction +{ +public: + static constexpr auto name = "removeVarUIntLengthPrefix"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + checkIsStringOrFixedString(getName(), arguments[0]); + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const IColumn & column = *arguments[0].column; + + auto result = ColumnString::create(); + result->reserve(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const std::string_view value = column.getDataAt(row); + + UInt64 length = 0; + size_t prefix_size = 0; + bool complete = false; + while (prefix_size < value.size() && prefix_size < VAR_UINT_MAX_SIZE) + { + const UInt64 byte = static_cast(value[prefix_size]); + + /// The tenth byte can only carry the highest bit of a `UInt64`. + if (prefix_size == VAR_UINT_MAX_SIZE - 1 && byte > 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "The length prefix in the argument of function {} does not fit UInt64", getName()); + + length |= (byte & 0x7F) << (7 * prefix_size); + ++prefix_size; + + if (!(byte & 0x80)) + { + complete = true; + break; + } + } + + if (!complete) + throw Exception(ErrorCodes::INCORRECT_DATA, "The argument of function {} does not start with a complete VarUInt length", getName()); + + const size_t rest = value.size() - prefix_size; + if (length != rest) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "The length prefix in the argument of function {} is {}, but {} bytes follow it", + getName(), + length, + rest); + + result->insertData(value.data() + prefix_size, rest); + } + + return result; + } +}; + +class FunctionToVarUInt final : public IFunction +{ +public: + static constexpr auto name = "toVarUInt"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!WhichDataType(arguments[0]).isNativeUInt()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument of function {}, expected UInt8, UInt16, UInt32 or UInt64", + arguments[0]->getName(), + getName()); + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const IColumn & column = *arguments[0].column; + + auto result = ColumnString::create(); + auto & chars = result->getChars(); + auto & offsets = result->getOffsets(); + offsets.resize(input_rows_count); + chars.reserve(input_rows_count * 2); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const UInt64 value = column.getUInt(row); + + const size_t old_size = chars.size(); + chars.resize(old_size + getLengthOfVarUInt(value)); + writeVarUInt(value, reinterpret_cast(chars.data() + old_size)); + + offsets[row] = chars.size(); + } + + return result; + } +}; + +} + +REGISTER_FUNCTION(VarUIntLengthPrefix) +{ + { + FunctionDocumentation::Description description = R"( +Puts the length of a string in front of it, encoded as a `VarUInt` (an unsigned LEB128 number). +This is how `RowBinary` and `Native` store a `String`, and how the state of some aggregate functions (for example `uniqApacheHLL` and `uniqTheta`) stores an external sketch. +It is the inverse of [`removeVarUIntLengthPrefix`](#removeVarUIntLengthPrefix). + +A sketch that was written by another system, without the length, can be read as a state of such a function after adding the prefix: `CAST(addVarUIntLengthPrefix(sketch) AS AggregateFunction(...))`. +)"; + FunctionDocumentation::Syntax syntax = "addVarUIntLengthPrefix(s)"; + FunctionDocumentation::Arguments arguments = {{"s", "The string.", {"String", "FixedString"}}}; + FunctionDocumentation::ReturnedValue returned_value = {"The string with its length in front of it.", {"String"}}; + FunctionDocumentation::Examples examples = { + {"Usage example", "SELECT hex(addVarUIntLengthPrefix('abc'))", "03616263"}, + }; + FunctionDocumentation::IntroducedIn introduced_in = {26, 6}; + FunctionDocumentation::Category category = FunctionDocumentation::Category::String; + FunctionDocumentation documentation = {description, syntax, arguments, {}, returned_value, examples, introduced_in, category}; + + factory.registerFunction(documentation); + } + + { + FunctionDocumentation::Description description = R"( +Removes the length that is stored in front of a string as a `VarUInt` (an unsigned LEB128 number), and returns the string. +It is the inverse of [`addVarUIntLengthPrefix`](#addVarUIntLengthPrefix). + +The length must be complete and equal to the number of bytes that follow it, otherwise the function throws an exception. +The encoding of the length is read as the `RowBinary` reader reads it, so a length that is padded with zero groups is accepted. + +It gives another system the bytes of a sketch that the state of an aggregate function stores, for example `removeVarUIntLengthPrefix(toString(state))`. +Apply it only to a state that is stored as a length followed by bytes, such as the state of `uniqApacheHLL`. +)"; + FunctionDocumentation::Syntax syntax = "removeVarUIntLengthPrefix(s)"; + FunctionDocumentation::Arguments arguments = {{"s", "The string that starts with its length.", {"String", "FixedString"}}}; + FunctionDocumentation::ReturnedValue returned_value = {"The string without its length.", {"String"}}; + FunctionDocumentation::Examples examples = { + {"Usage example", "SELECT removeVarUIntLengthPrefix(unhex('03616263'))", "abc"}, + }; + FunctionDocumentation::IntroducedIn introduced_in = {26, 6}; + FunctionDocumentation::Category category = FunctionDocumentation::Category::String; + FunctionDocumentation documentation = {description, syntax, arguments, {}, returned_value, examples, introduced_in, category}; + + factory.registerFunction(documentation); + } + + { + FunctionDocumentation::Description description = R"( +Encodes an unsigned integer as a `VarUInt` (an unsigned LEB128 number): 7 bits per byte, the lowest bits first, the high bit of every byte but the last one is set. +It takes from one to ten bytes. +)"; + FunctionDocumentation::Syntax syntax = "toVarUInt(x)"; + FunctionDocumentation::Arguments arguments = {{"x", "The number.", {"UInt8", "UInt16", "UInt32", "UInt64"}}}; + FunctionDocumentation::ReturnedValue returned_value = {"The bytes of the encoded number.", {"String"}}; + FunctionDocumentation::Examples examples = { + {"Usage example", "SELECT hex(toVarUInt(300))", "AC02"}, + }; + FunctionDocumentation::IntroducedIn introduced_in = {26, 6}; + FunctionDocumentation::Category category = FunctionDocumentation::Category::String; + FunctionDocumentation documentation = {description, syntax, arguments, {}, returned_value, examples, introduced_in, category}; + + factory.registerFunction(documentation); + } +} + +} diff --git a/tests/queries/0_stateless/05298_var_uint_length_prefix.reference b/tests/queries/0_stateless/05298_var_uint_length_prefix.reference new file mode 100644 index 000000000000..d5b6150623a6 --- /dev/null +++ b/tests/queries/0_stateless/05298_var_uint_length_prefix.reference @@ -0,0 +1,20 @@ +toVarUInt +00 7F 8001 AC02 FF7F 808001 FFFFFFFFFFFFFFFFFF01 +C801 E0D403 80D0ACF30E +00 +01 +02 +addVarUIntLengthPrefix +03616263 00 0461620000 +7F61 800161 AC0261 +128 130 16387 +removeVarUIntLengthPrefix +abc a +a padded length is read as RowBinary reads it +a a +a round trip gives the same string +200 200 +1 +1 +wrong lengths +wrong argument types diff --git a/tests/queries/0_stateless/05298_var_uint_length_prefix.sql b/tests/queries/0_stateless/05298_var_uint_length_prefix.sql new file mode 100644 index 000000000000..e0cfdb9b0016 --- /dev/null +++ b/tests/queries/0_stateless/05298_var_uint_length_prefix.sql @@ -0,0 +1,37 @@ +-- `toVarUInt` encodes a number as an unsigned LEB128 `VarUInt`, `addVarUIntLengthPrefix` puts the length of a string in front of it as such a number, `removeVarUIntLengthPrefix` takes it off. + +SELECT 'toVarUInt'; +SELECT hex(toVarUInt(0)), hex(toVarUInt(127)), hex(toVarUInt(128)), hex(toVarUInt(300)), hex(toVarUInt(16383)), hex(toVarUInt(16384)), hex(toVarUInt(toUInt64(18446744073709551615))); +SELECT hex(toVarUInt(toUInt8(200))), hex(toVarUInt(toUInt16(60000))), hex(toVarUInt(toUInt32(4000000000))); +SELECT hex(toVarUInt(number)) FROM numbers(3); + +SELECT 'addVarUIntLengthPrefix'; +SELECT hex(addVarUIntLengthPrefix('abc')), hex(addVarUIntLengthPrefix('')), hex(addVarUIntLengthPrefix(toFixedString('ab', 4))); +SELECT hex(substring(addVarUIntLengthPrefix(repeat('a', 127)), 1, 2)), hex(substring(addVarUIntLengthPrefix(repeat('a', 128)), 1, 3)), hex(substring(addVarUIntLengthPrefix(repeat('a', 300)), 1, 3)); +SELECT length(addVarUIntLengthPrefix(repeat('a', 127))), length(addVarUIntLengthPrefix(repeat('a', 128))), length(addVarUIntLengthPrefix(repeat('a', 16384))); + +SELECT 'removeVarUIntLengthPrefix'; +SELECT removeVarUIntLengthPrefix(unhex('03616263')), hex(removeVarUIntLengthPrefix(unhex('00'))), removeVarUIntLengthPrefix(toFixedString(unhex('0161'), 2)); +SELECT 'a padded length is read as RowBinary reads it'; +SELECT removeVarUIntLengthPrefix(unhex('810061')), removeVarUIntLengthPrefix(unhex('81800061')); + +SELECT 'a round trip gives the same string'; +SELECT countIf(removeVarUIntLengthPrefix(addVarUIntLengthPrefix(s)) = s), count() FROM (SELECT repeat(char(number % 256), number * 37 % 40000) AS s FROM numbers(200)); +SELECT removeVarUIntLengthPrefix(addVarUIntLengthPrefix(CAST(NULL, 'Nullable(String)'))) IS NULL; +SELECT addVarUIntLengthPrefix(materialize('')) = unhex('00'); + +SELECT 'wrong lengths'; +SELECT removeVarUIntLengthPrefix(''); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('80')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('0261')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('016162')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('00ff')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('FFFFFFFFFFFFFFFFFF02')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('FFFFFFFFFFFFFFFFFFFF01')); -- { serverError INCORRECT_DATA } + +SELECT 'wrong argument types'; +SELECT toVarUInt(-1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toVarUInt(toInt64(1)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toVarUInt('1'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT addVarUIntLengthPrefix(1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT removeVarUIntLengthPrefix(1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.reference b/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.reference new file mode 100644 index 000000000000..71c5b73887c4 --- /dev/null +++ b/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.reference @@ -0,0 +1,11 @@ +it can be read after the length is added +5 +merged without a state column +5 +a state gives the bytes of its sketch +0301070B05080009 +a sketch that is read and written back is the same sketch +1 +a big sketch needs a longer length +1 1 +a state that is not a length and bytes is rejected diff --git a/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.sql b/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.sql new file mode 100644 index 000000000000..734ba2ee53d6 --- /dev/null +++ b/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.sql @@ -0,0 +1,26 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- A sketch that another system wrote without a length (for example, Spark `hll_sketch_agg` into Parquet) is read as a state with +-- `addVarUIntLengthPrefix`, and the bytes of a state are given to another system with `removeVarUIntLengthPrefix`. + +SELECT 'it can be read after the length is added'; +SELECT finalizeAggregation(CAST(addVarUIntLengthPrefix(unhex('0201070b03080508cbd7c2042bf2fb06862ff90d7581660781bc5d06')), 'AggregateFunction(uniqApacheHLL, UInt64)')); + +SELECT 'merged without a state column'; +SELECT uniqApacheHLLMerge(CAST(addVarUIntLengthPrefix(sketch), 'AggregateFunction(uniqApacheHLL, UInt64)')) +FROM (SELECT unhex('0201070b03080508cbd7c2042bf2fb06862ff90d7581660781bc5d06') AS sketch FROM numbers(3)); + +SELECT 'a state gives the bytes of its sketch'; +SELECT hex(substring(removeVarUIntLengthPrefix(toString(uniqApacheHLLState(11, 'HLL_8')(number))), 1, 8)) FROM numbers(10); + +SELECT 'a sketch that is read and written back is the same sketch'; +WITH unhex('0201070b03080508cbd7c2042bf2fb06862ff90d7581660781bc5d06') AS sketch +SELECT removeVarUIntLengthPrefix(toString(CAST(addVarUIntLengthPrefix(sketch), 'AggregateFunction(uniqApacheHLL, UInt64)'))) = sketch; + +SELECT 'a big sketch needs a longer length'; +WITH (SELECT removeVarUIntLengthPrefix(toString(uniqApacheHLLState(14, 'HLL_8')(number))) FROM numbers(100000)) AS sketch +SELECT length(sketch) > 16000, finalizeAggregation(CAST(addVarUIntLengthPrefix(sketch), 'AggregateFunction(uniqApacheHLL(14, \'HLL_8\'), UInt64)')) = uniqApacheHLL(14, 'HLL_8')(number) FROM numbers(100000); + +SELECT 'a state that is not a length and bytes is rejected'; +SELECT removeVarUIntLengthPrefix(toString(countState())); -- { serverError INCORRECT_DATA } diff --git a/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.reference b/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.reference new file mode 100644 index 000000000000..1bb602f6e9ef --- /dev/null +++ b/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.reference @@ -0,0 +1,2 @@ +OK without the length +5 diff --git a/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.sh b/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.sh new file mode 100755 index 000000000000..c9dcf3ca74d0 --- /dev/null +++ b/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-cpu-s390x +# no-fasttest -- compiled w/o datasketches + +# A sketch that another system wrote without a length in front of it is not a state of `uniqApacheHLL`: the state reader takes +# its first bytes for the length. `CORRUPTED_DATA` is always logged by the client, so this is a shell test, as in +# `05136_uniq_apache_hll_corrupted_state.sh`. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SKETCH="unhex('0201070b03080508cbd7c2042bf2fb06862ff90d7581660781bc5d06')" + +$CLICKHOUSE_CLIENT --query \ + "SELECT finalizeAggregation(CAST($SKETCH, 'AggregateFunction(uniqApacheHLL, UInt64)'))" 2>&1 \ + | grep -q -F 'CORRUPTED_DATA' && echo 'OK without the length' || echo 'FAIL without the length' + +$CLICKHOUSE_CLIENT --query \ + "SELECT finalizeAggregation(CAST(addVarUIntLengthPrefix($SKETCH), 'AggregateFunction(uniqApacheHLL, UInt64)'))" 2>&1 From 4e0c476bd5b4f013c383d345bf01ce529965a6a9 Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Mon, 12 Oct 2026 20:40:04 +0900 Subject: [PATCH 3/5] Fold the pending sketch of a `uniqApacheHLL` state before merging into it A state can hold a union and a sketch of inserts that came after a merge, for example when the rows of the tail of a block are added to a state that has already merged the lookup tables of the block, as the aggregation by a `UInt8` key does. `merge` took the new sketches first and folded the pending sketch at its end, so the older rows came after the newer ones. As the HIP estimate and the bytes of the sketch depend on this order, the pending rows are now folded into the union before the sketches are taken. The result of the default estimator can still differ with the size of the blocks, since the lookup tables merge the rows in another order; the composite estimator does not, and a test checks it. Related: https://github.com/ClickHouse/ClickHouse/pull/124667 Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: UnamedRus --- src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h | 6 ++++-- .../0_stateless/05297_uniq_apache_hll_estimator.reference | 2 ++ .../queries/0_stateless/05297_uniq_apache_hll_estimator.sql | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h index fc84f57013d8..b50a15502281 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h @@ -114,6 +114,10 @@ class HllSketchData datasketches::hll_union * u = getSkUnion(lg_config_k); + /// The rows that are already pending in this state come before `rhs`. They can be there after a merge, for example when + /// the rows of the tail of a block are added to a state that has already merged the lookup tables of the block. + foldUpdateIntoUnionIfNeeded(); + /// `rhs` may hold both sketches; take both without modifying it, in the order of `foldUpdateIntoUnionIfNeeded`: the union, then the /// updates. The registers of the result do not depend on the order, but the HIP estimate and the bytes of the sketch do. /// The union accepts a sketch of any type, and `HLL_8` is its own type, so there is no re-encoding. @@ -122,8 +126,6 @@ class HllSketchData if (rhs_has_update) u->update(*rhs.sk_update); - - foldUpdateIntoUnionIfNeeded(); } /// You can only call this for an empty object. The sketch is kept as it was written, with its own resolution and diff --git a/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference index 6b1729aa5df0..9b745b7aae4c 100644 --- a/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference +++ b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference @@ -10,4 +10,6 @@ the composite estimate of a sketch that is built by inserts does not depend on t the same for a state that is finalized, and for a state that is read 1 1 +the composite estimate for a UInt8 key does not depend on the size of the blocks +1 wrong parameters diff --git a/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql index f71c9ab3c93b..2145b6003b76 100644 --- a/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql +++ b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql @@ -46,6 +46,9 @@ SELECT count() = 3 AND uniqExact(r) = 1 FROM SELECT finalizeAggregation(CAST(toString(uniqApacheHLLState(11, 'HLL_8')(x)), 'AggregateFunction(uniqApacheHLL(11, \'HLL_8\', \'COMPOSITE\'), UInt64)')) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY cityHash64(x)) ); +SELECT 'the composite estimate for a UInt8 key does not depend on the size of the blocks'; +SELECT (SELECT arraySort(groupArray((k, r))) FROM (SELECT toUInt8(number % 7) AS k, uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(number) AS r FROM numbers(30000) GROUP BY k SETTINGS max_block_size = 1000)) = (SELECT arraySort(groupArray((k, r))) FROM (SELECT toUInt8(number % 7) AS k, uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(number) AS r FROM numbers(30000) GROUP BY k SETTINGS max_block_size = 65536)); + SELECT 'wrong parameters'; SELECT uniqApacheHLL(11, 'HLL_8', 'HIP')(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } SELECT uniqApacheHLL(11, 'HLL_8', 1)(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } From 3167a8f9deae4472b90a87ebcc1f2c80c32ffac2 Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Mon, 12 Oct 2026 20:40:04 +0900 Subject: [PATCH 4/5] Shorten the comment of `stateIsIndependentOfNullability` Related: https://github.com/ClickHouse/ClickHouse/pull/124667 Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: UnamedRus --- src/AggregateFunctions/IAggregateFunction.h | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 849ff37440d6..fb63d5bd28b1 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -417,16 +417,9 @@ class IAggregateFunction : public std::enable_shared_from_this getArgumentsThatCanBeOnlyNull() const { return {}; } - /** When the function is wrapped with the Null combinator (or the If combinator over Nullable arguments), the state of the - * wrapper has a flag byte and a type that names the Nullable argument. So the state of a Nullable column is not the state of a - * plain one: another implementation cannot read it, and the two cannot be mixed. - * - * A function that only skips NULL values, and whose state of no values is the empty state (example: uniqApacheHLL), - * does not need that. If it returns true here, the wrapper only skips NULL rows, and its state has the same bytes and the - * same type as the state of this function for a non-Nullable argument. - * - * It requires returns_default_when_only_null: the result of a function that must return NULL when only NULL values - * were seen needs the flag to tell it from the state of no rows. + /** If true, the `Null` combinator (and `If` over `Nullable` arguments) only skips `NULL` rows, and its state has the same bytes and + * type as the state of this function for a non-`Nullable` argument, without the flag byte. Only for a function whose state of no + * values is the empty state, for example `uniqApacheHLL`; it requires `returns_default_when_only_null`. */ virtual bool stateIsIndependentOfNullability() const { return false; } From 521419f237fb1e723df5b5cb58e15773275e5e5c Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Mon, 12 Oct 2026 21:58:32 +0900 Subject: [PATCH 5/5] Do not use `VAR_UINT_MAX_SIZE` in `removeVarUIntLengthPrefix` The constant exists in the `VarInt.h` of upstream master only, so the function did not compile on this branch. It is defined in the file under another name, so that it does not clash when the branch merges upstream. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: UnamedRus --- src/Functions/varUIntLengthPrefix.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Functions/varUIntLengthPrefix.cpp b/src/Functions/varUIntLengthPrefix.cpp index 593a97840d98..e4f0545e4c98 100644 --- a/src/Functions/varUIntLengthPrefix.cpp +++ b/src/Functions/varUIntLengthPrefix.cpp @@ -20,6 +20,9 @@ namespace ErrorCodes namespace { +/// The most bytes that `writeVarUInt` emits for a `UInt64`. +constexpr size_t var_uint_max_size = 10; + /// A string that is stored with its length in front of it, as `RowBinary` and `Native` store a `String` and as the state of /// some aggregate functions is stored, has the length as a `VarUInt`: an unsigned LEB128 number, 7 bits per byte, the lowest /// bits first, the high bit of every byte but the last one is set. The functions below convert between such a string and its @@ -109,12 +112,12 @@ class FunctionRemoveVarUIntLengthPrefix final : public IFunction UInt64 length = 0; size_t prefix_size = 0; bool complete = false; - while (prefix_size < value.size() && prefix_size < VAR_UINT_MAX_SIZE) + while (prefix_size < value.size() && prefix_size < var_uint_max_size) { const UInt64 byte = static_cast(value[prefix_size]); /// The tenth byte can only carry the highest bit of a `UInt64`. - if (prefix_size == VAR_UINT_MAX_SIZE - 1 && byte > 1) + if (prefix_size == var_uint_max_size - 1 && byte > 1) throw Exception(ErrorCodes::INCORRECT_DATA, "The length prefix in the argument of function {} does not fit UInt64", getName()); length |= (byte & 0x7F) << (7 * prefix_size);