diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp index 95144e70c22a..404991b18f24 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp @@ -21,10 +21,11 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( { uint8_t lg_config_k = 12; datasketches::target_hll_type target_type = datasketches::HLL_4; + bool use_composite_estimate = false; - if (params.size() > 2) + if (params.size() > 3) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Aggregate function {} accepts at most two parameters (lg_k, type).", name); + "Aggregate function {} accepts at most three parameters (lg_k, type, estimator).", name); if (!params.empty()) { @@ -35,7 +36,7 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( lg_config_k = static_cast(lg_k_param); } - if (params.size() == 2) + if (params.size() >= 2) { if (params[1].getType() != Field::Types::String) throw Exception(ErrorCodes::BAD_ARGUMENTS, @@ -53,6 +54,22 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( "Parameter type for aggregate function {} must be one of 'HLL_4', 'HLL_6', 'HLL_8'.", name); } + if (params.size() == 3) + { + if (params[2].getType() != Field::Types::String) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Parameter estimator for aggregate function {} must be a string.", name); + + const String estimator_param = params[2].safeGet(); + if (estimator_param == "DEFAULT") + use_composite_estimate = false; + else if (estimator_param == "COMPOSITE") + use_composite_estimate = true; + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Parameter estimator for aggregate function {} must be one of 'DEFAULT', 'COMPOSITE'.", name); + } + assertUnary(name, argument_types); const IDataType & argument_type = *argument_types[0]; @@ -60,35 +77,35 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( /// Unlike other decimals, `DateTime64` has an interoperable representation as epoch ticks. if (which.isDateTime64()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); /// Exclude wide integers: DataSketches has no portable representation for them. if (!which.isInt128() && !which.isInt256() && !which.isUInt128() && !which.isUInt256()) { AggregateFunctionPtr res(createWithNumericType( - argument_type, lg_config_k, target_type, argument_types, params)); + argument_type, lg_config_k, target_type, use_composite_estimate, argument_types, params)); if (res) return res; } if (which.isDate()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isDate32()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isDateTime()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isStringOrFixedString()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isUUID()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isIPv4()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); if (which.isIPv6()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); /// For `Nullable(Nothing)` the `Null` combinator replaces this function with `nothing`, but it must be created first. if (argument_type.onlyNull()) - return std::make_shared>(lg_config_k, target_type, argument_types, params); + return std::make_shared>(lg_config_k, target_type, use_composite_estimate, argument_types, params); throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument for aggregate function {}. Use uniq, uniqCombined or uniqHLL12 for unsupported types.", @@ -105,21 +122,27 @@ The `-State` and `-Merge` combinators exchange Apache DataSketches HLL sketches For interoperability, integers of at most 64 bits are hashed as 8-byte integers, floating-point values as IEEE-754 doubles, strings as raw bytes, `UUID` values as canonical 16 bytes, and `IPv6` addresses in network order. `Date`, `Date32`, `DateTime` and `DateTime64` use their underlying integer values. External producers must use the same unit; for example, `DateTime64(3)` uses epoch milliseconds. +An unsigned `UInt8`, `UInt16` or `UInt32` value is hashed as the same number as a 64-bit integer, as Java and Spark hash a number that they hold as a `long`. +The C++ library hashes `uint8_t`, `uint16_t` and `uint32_t` through the signed type of the same width, so for the values from 128, 32768 and 2147483648 respectively (and for `Date` from 32768, `DateTime` from 2147483648, and `IPv4` from 128.0.0.0) its sketch is another one. +To get the sketch of the C++ library, cast the argument to the signed type of the same width with a cast that wraps: `toInt8(x)`, `toInt16(x)` or `toInt32(x)`; for `Date` use `toInt16(toUInt16(x))`, for `DateTime` use `toInt32(x)`, and for `IPv4` use `toInt32(toUInt32(x))`. +Do not use `accurateCast`, which rejects these values. Unsupported types and multiple arguments are rejected. Use `uniq`, `uniqCombined` or `uniqHLL12` for those inputs. `NULL` values and empty strings are ignored, as in the Java, Python and C++ implementations and in Spark; unlike `uniq`, an empty string is not counted as a value. A `Nullable` argument gives the same state as a non-`Nullable` one, a bare DataSketches sketch, and the same state type, `AggregateFunction(uniqApacheHLL, T)`. -Merging can switch from the HIP estimator to the less accurate composite estimator, so results can depend on partitioning across threads, parts and shards. +Merging can switch from the HIP estimator to the less accurate composite estimator, and the HIP estimate of a union of small sketches depends on the order of the merges, so with the default estimator results can depend on partitioning across threads, parts and shards. +The `estimator` parameter `'COMPOSITE'` makes the result depend only on the registers of the sketch and not on the order of the inserts and of the merges, at the cost of a lower accuracy for sketches of a few hundred distinct values. It does not change the state. Merging a lower-resolution sketch permanently lowers the result's resolution, regardless of the declared `lg_k`. )"; - FunctionDocumentation::Syntax syntax = "uniqApacheHLL([lg_k, [type]])(x)"; + FunctionDocumentation::Syntax syntax = "uniqApacheHLL([lg_k, [type, [estimator]]])(x)"; FunctionDocumentation::Arguments arguments = { {"x", "Column to compute the number of distinct values of.", {"(U)Int8/16/32/64", "Enum", "BFloat16", "Float32", "Float64", "String", "FixedString", "UUID", "IPv4", "IPv6", "Date", "Date32", "DateTime", "DateTime64"}}, }; FunctionDocumentation::Parameters parameters = { {"lg_k", "Optional. Log-base-2 of the number of buckets, in range [4, 21]. Higher means better accuracy and more memory. Default: 12.", {"UInt8"}}, {"type", "Optional. Storage format of the sketch: 'HLL_4', 'HLL_6', or 'HLL_8'. Default: 'HLL_4'.", {"String"}}, + {"estimator", "Optional. The estimator used for the result: 'DEFAULT' (the estimator of the library, which can depend on the order of the inserts and of the merges) or 'COMPOSITE' (depends only on the registers of the sketch). Default: 'DEFAULT'.", {"String"}}, }; FunctionDocumentation::ReturnedValue returned_value = {"Returns the approximate number of distinct values.", {"UInt64"}}; FunctionDocumentation::Examples examples = { @@ -130,7 +153,9 @@ Merging a lower-resolution sketch permanently lowers the result's resolution, re FunctionDocumentation::Category category = FunctionDocumentation::Category::AggregateFunction; FunctionDocumentation documentation = {description, syntax, arguments, parameters, returned_value, examples, introduced_in, category}; - AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = false }; + /// The HIP estimate and the bytes of the sketch depend on the order of the updates and of the merges, so the planner must not + /// remove an `ORDER BY` below this function. + AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = true }; factory.registerFunction("uniqApacheHLL", {createAggregateFunctionUniqApacheHLL, documentation, properties}); } diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h index 76a0cd41de68..b50a15502281 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h @@ -4,14 +4,17 @@ #if USE_DATASKETCHES +#include #include #include #include #include #include #include +#include #include #include +#include #include #include #include @@ -72,7 +75,7 @@ class HllSketchData if (sk_union && sk_update) { sk_union->update(*sk_update); - sk_update.reset(); + sk_update = nullptr; } } @@ -88,38 +91,47 @@ class HllSketchData getSkUpdate(lg_config_k, tgt_type)->update(static_cast(data), size); } - UInt64 size() const + /// With `use_composite_estimate` the estimate is computed from the registers of the sketch only, so it does not depend on the order + /// of the inserts and of the merges. The estimate of the library is the HIP estimate while the sketch is in order, which is more + /// accurate for a small sketch, but it depends on that order. + UInt64 size(bool use_composite_estimate) const { foldUpdateIntoUnionIfNeeded(); /// Rounding preserves exact cardinalities despite floating-point error. if (sk_union) - return static_cast(std::llround(sk_union->get_estimate())); + return static_cast(std::llround(use_composite_estimate ? sk_union->get_composite_estimate() : sk_union->get_estimate())); if (sk_update) - return static_cast(std::llround(sk_update->get_estimate())); + return static_cast(std::llround(use_composite_estimate ? sk_update->get_composite_estimate() : sk_update->get_estimate())); return 0; } void merge(const HllSketchData & rhs, uint8_t lg_config_k) { - if (!rhs.sk_update && !rhs.sk_union) + const bool rhs_has_update = rhs.sk_update && !rhs.sk_update->is_empty(); + if (!rhs_has_update && !rhs.sk_union) return; datasketches::hll_union * u = getSkUnion(lg_config_k); - /// `rhs` may hold both sketches; take both without modifying it. - if (rhs.sk_update) - u->update(*rhs.sk_update); + /// The rows that are already pending in this state come before `rhs`. They can be there after a merge, for example when + /// the rows of the tail of a block are added to a state that has already merged the lookup tables of the block. + foldUpdateIntoUnionIfNeeded(); + /// `rhs` may hold both sketches; take both without modifying it, in the order of `foldUpdateIntoUnionIfNeeded`: the union, then the + /// updates. The registers of the result do not depend on the order, but the HIP estimate and the bytes of the sketch do. /// The union accepts a sketch of any type, and `HLL_8` is its own type, so there is no re-encoding. if (rhs.sk_union) u->update(rhs.sk_union->get_result(datasketches::HLL_8)); - foldUpdateIntoUnionIfNeeded(); + if (rhs_has_update) + u->update(*rhs.sk_update); } - /// You can only call this for an empty object. - void read(ReadBuffer & in, uint8_t lg_config_k) + /// You can only call this for an empty object. The sketch is kept as it was written, with its own resolution and + /// storage type: reading a state must not lose what writing it back would then persist. It is folded into a union, + /// at the declared resolution, only when it is merged. + void read(ReadBuffer & in) { datasketches::hll_sketch::vector_bytes bytes; readVectorBinary(bytes, in); @@ -129,7 +141,7 @@ class HllSketchData try { auto sk = datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()); - getSkUnion(lg_config_k)->update(std::move(sk)); + sk_update = std::make_unique(std::move(sk)); } catch (const DB::Exception &) { @@ -169,16 +181,19 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper uint8_t lg_config_k; datasketches::target_hll_type target_type; + bool use_composite_estimate; public: AggregateFunctionUniqApacheHLL( uint8_t lg_config_k_, datasketches::target_hll_type target_type_, + bool use_composite_estimate_, const DataTypes & argument_types_, const Array & params_) : Base(argument_types_, params_, std::make_shared()) , lg_config_k(lg_config_k_) , target_type(target_type_) + , use_composite_estimate(use_composite_estimate_) { } @@ -199,7 +214,7 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper /// Other DataSketches implementations ignore empty strings, and the sketch of a mixed /// input must be the same as theirs. - if (value.size() == 0) + if (value.empty()) return; data.insertData(value.data(), value.size(), lg_config_k, target_type); @@ -210,11 +225,11 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper if constexpr (std::is_same_v) { - /// Convert the two host-order halves of a `UUID` to canonical bytes. - const UInt64 halves[2] = { - std::byteswap(UUIDHelpers::getHighBytes(value)), - std::byteswap(UUIDHelpers::getLowBytes(value)), - }; + /// The two halves of a `UUID` are numbers in host order; the canonical bytes are their big-endian + /// representation on every platform. + UInt64 halves[2] = {UUIDHelpers::getHighBytes(value), UUIDHelpers::getLowBytes(value)}; + transformEndianness(halves[0]); + transformEndianness(halves[1]); data.insertData(reinterpret_cast(halves), sizeof(halves), lg_config_k, target_type); } else if constexpr (std::is_same_v) @@ -249,6 +264,21 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper [](const auto & lhs, const auto & rhs_type) { return removeNullable(lhs)->equals(*removeNullable(rhs_type)); }); } + /// The parameters do not change what a state means, so every spelling of them is the same type wherever a common type is + /// needed (`UNION ALL`, arrays, `if`) and wherever a state is accepted by the name of its type. The default parameters + /// stand for all of them. + DataTypePtr getNormalizedStateType() const override + { + DataTypes normalized_argument_types; + normalized_argument_types.reserve(this->argument_types.size()); + for (const auto & type : this->argument_types) + normalized_argument_types.emplace_back(type->getNormalizedType()); + + AggregateFunctionProperties properties; + auto function = AggregateFunctionFactory::instance().get("uniqApacheHLL", NullsAction::EMPTY, normalized_argument_types, {}, properties); + return std::make_shared(function, normalized_argument_types, Array{}); + } + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override { this->data(place).merge(this->data(rhs), lg_config_k); @@ -261,12 +291,12 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override { - this->data(place).read(buf, lg_config_k); + this->data(place).read(buf); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - assert_cast(to).getData().push_back(this->data(place).size()); + assert_cast(to).getData().push_back(this->data(place).size(use_composite_estimate)); } }; diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 849ff37440d6..fb63d5bd28b1 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -417,16 +417,9 @@ class IAggregateFunction : public std::enable_shared_from_this getArgumentsThatCanBeOnlyNull() const { return {}; } - /** When the function is wrapped with the Null combinator (or the If combinator over Nullable arguments), the state of the - * wrapper has a flag byte and a type that names the Nullable argument. So the state of a Nullable column is not the state of a - * plain one: another implementation cannot read it, and the two cannot be mixed. - * - * A function that only skips NULL values, and whose state of no values is the empty state (example: uniqApacheHLL), - * does not need that. If it returns true here, the wrapper only skips NULL rows, and its state has the same bytes and the - * same type as the state of this function for a non-Nullable argument. - * - * It requires returns_default_when_only_null: the result of a function that must return NULL when only NULL values - * were seen needs the flag to tell it from the state of no rows. + /** If true, the `Null` combinator (and `If` over `Nullable` arguments) only skips `NULL` rows, and its state has the same bytes and + * type as the state of this function for a non-`Nullable` argument, without the flag byte. Only for a function whose state of no + * values is the empty state, for example `uniqApacheHLL`; it requires `returns_default_when_only_null`. */ virtual bool stateIsIndependentOfNullability() const { return false; } diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index 22291d140905..d5e10987d3a9 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -63,7 +63,9 @@ void registerAggregateFunctionSumMap(AggregateFunctionFactory &); void registerAggregateFunctionsUniq(AggregateFunctionFactory &); void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &); void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &); -#if USE_DATASKETCHES +/// Apache DataSketches hashes values and serializes sketches in the byte order of the host, so `uniqApacheHLL` is registered only on +/// little-endian platforms, as if DataSketches were not built. +#if USE_DATASKETCHES && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ void registerAggregateFunctionUniqApacheHLL(AggregateFunctionFactory &); #endif void registerAggregateFunctionTopK(AggregateFunctionFactory &); @@ -179,7 +181,7 @@ void registerAggregateFunctions() registerAggregateFunctionsUniq(factory); registerAggregateFunctionUniqCombined(factory); registerAggregateFunctionUniqUpTo(factory); -#if USE_DATASKETCHES +#if USE_DATASKETCHES && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ registerAggregateFunctionUniqApacheHLL(factory); #endif registerAggregateFunctionTopK(factory); diff --git a/src/Functions/varUIntLengthPrefix.cpp b/src/Functions/varUIntLengthPrefix.cpp new file mode 100644 index 000000000000..e4f0545e4c98 --- /dev/null +++ b/src/Functions/varUIntLengthPrefix.cpp @@ -0,0 +1,267 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; +} + +namespace +{ + +/// The most bytes that `writeVarUInt` emits for a `UInt64`. +constexpr size_t var_uint_max_size = 10; + +/// A string that is stored with its length in front of it, as `RowBinary` and `Native` store a `String` and as the state of +/// some aggregate functions is stored, has the length as a `VarUInt`: an unsigned LEB128 number, 7 bits per byte, the lowest +/// bits first, the high bit of every byte but the last one is set. The functions below convert between such a string and its +/// contents, and encode one number. + +void checkIsStringOrFixedString(const String & function_name, const DataTypePtr & type) +{ + if (!isStringOrFixedString(type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}, expected String or FixedString", type->getName(), function_name); +} + +class FunctionAddVarUIntLengthPrefix final : public IFunction +{ +public: + static constexpr auto name = "addVarUIntLengthPrefix"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + checkIsStringOrFixedString(getName(), arguments[0]); + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const IColumn & column = *arguments[0].column; + + auto result = ColumnString::create(); + auto & chars = result->getChars(); + auto & offsets = result->getOffsets(); + offsets.resize(input_rows_count); + chars.reserve(column.byteSize() + input_rows_count * 2); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const std::string_view value = column.getDataAt(row); + const size_t prefix_size = getLengthOfVarUInt(value.size()); + + const size_t old_size = chars.size(); + chars.resize(old_size + prefix_size + value.size()); + + char * pos = reinterpret_cast(chars.data() + old_size); + pos = writeVarUInt(value.size(), pos); + if (!value.empty()) + memcpy(pos, value.data(), value.size()); + + offsets[row] = chars.size(); + } + + return result; + } +}; + +class FunctionRemoveVarUIntLengthPrefix final : public IFunction +{ +public: + static constexpr auto name = "removeVarUIntLengthPrefix"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + checkIsStringOrFixedString(getName(), arguments[0]); + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const IColumn & column = *arguments[0].column; + + auto result = ColumnString::create(); + result->reserve(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const std::string_view value = column.getDataAt(row); + + UInt64 length = 0; + size_t prefix_size = 0; + bool complete = false; + while (prefix_size < value.size() && prefix_size < var_uint_max_size) + { + const UInt64 byte = static_cast(value[prefix_size]); + + /// The tenth byte can only carry the highest bit of a `UInt64`. + if (prefix_size == var_uint_max_size - 1 && byte > 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "The length prefix in the argument of function {} does not fit UInt64", getName()); + + length |= (byte & 0x7F) << (7 * prefix_size); + ++prefix_size; + + if (!(byte & 0x80)) + { + complete = true; + break; + } + } + + if (!complete) + throw Exception(ErrorCodes::INCORRECT_DATA, "The argument of function {} does not start with a complete VarUInt length", getName()); + + const size_t rest = value.size() - prefix_size; + if (length != rest) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "The length prefix in the argument of function {} is {}, but {} bytes follow it", + getName(), + length, + rest); + + result->insertData(value.data() + prefix_size, rest); + } + + return result; + } +}; + +class FunctionToVarUInt final : public IFunction +{ +public: + static constexpr auto name = "toVarUInt"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!WhichDataType(arguments[0]).isNativeUInt()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument of function {}, expected UInt8, UInt16, UInt32 or UInt64", + arguments[0]->getName(), + getName()); + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const IColumn & column = *arguments[0].column; + + auto result = ColumnString::create(); + auto & chars = result->getChars(); + auto & offsets = result->getOffsets(); + offsets.resize(input_rows_count); + chars.reserve(input_rows_count * 2); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const UInt64 value = column.getUInt(row); + + const size_t old_size = chars.size(); + chars.resize(old_size + getLengthOfVarUInt(value)); + writeVarUInt(value, reinterpret_cast(chars.data() + old_size)); + + offsets[row] = chars.size(); + } + + return result; + } +}; + +} + +REGISTER_FUNCTION(VarUIntLengthPrefix) +{ + { + FunctionDocumentation::Description description = R"( +Puts the length of a string in front of it, encoded as a `VarUInt` (an unsigned LEB128 number). +This is how `RowBinary` and `Native` store a `String`, and how the state of some aggregate functions (for example `uniqApacheHLL` and `uniqTheta`) stores an external sketch. +It is the inverse of [`removeVarUIntLengthPrefix`](#removeVarUIntLengthPrefix). + +A sketch that was written by another system, without the length, can be read as a state of such a function after adding the prefix: `CAST(addVarUIntLengthPrefix(sketch) AS AggregateFunction(...))`. +)"; + FunctionDocumentation::Syntax syntax = "addVarUIntLengthPrefix(s)"; + FunctionDocumentation::Arguments arguments = {{"s", "The string.", {"String", "FixedString"}}}; + FunctionDocumentation::ReturnedValue returned_value = {"The string with its length in front of it.", {"String"}}; + FunctionDocumentation::Examples examples = { + {"Usage example", "SELECT hex(addVarUIntLengthPrefix('abc'))", "03616263"}, + }; + FunctionDocumentation::IntroducedIn introduced_in = {26, 6}; + FunctionDocumentation::Category category = FunctionDocumentation::Category::String; + FunctionDocumentation documentation = {description, syntax, arguments, {}, returned_value, examples, introduced_in, category}; + + factory.registerFunction(documentation); + } + + { + FunctionDocumentation::Description description = R"( +Removes the length that is stored in front of a string as a `VarUInt` (an unsigned LEB128 number), and returns the string. +It is the inverse of [`addVarUIntLengthPrefix`](#addVarUIntLengthPrefix). + +The length must be complete and equal to the number of bytes that follow it, otherwise the function throws an exception. +The encoding of the length is read as the `RowBinary` reader reads it, so a length that is padded with zero groups is accepted. + +It gives another system the bytes of a sketch that the state of an aggregate function stores, for example `removeVarUIntLengthPrefix(toString(state))`. +Apply it only to a state that is stored as a length followed by bytes, such as the state of `uniqApacheHLL`. +)"; + FunctionDocumentation::Syntax syntax = "removeVarUIntLengthPrefix(s)"; + FunctionDocumentation::Arguments arguments = {{"s", "The string that starts with its length.", {"String", "FixedString"}}}; + FunctionDocumentation::ReturnedValue returned_value = {"The string without its length.", {"String"}}; + FunctionDocumentation::Examples examples = { + {"Usage example", "SELECT removeVarUIntLengthPrefix(unhex('03616263'))", "abc"}, + }; + FunctionDocumentation::IntroducedIn introduced_in = {26, 6}; + FunctionDocumentation::Category category = FunctionDocumentation::Category::String; + FunctionDocumentation documentation = {description, syntax, arguments, {}, returned_value, examples, introduced_in, category}; + + factory.registerFunction(documentation); + } + + { + FunctionDocumentation::Description description = R"( +Encodes an unsigned integer as a `VarUInt` (an unsigned LEB128 number): 7 bits per byte, the lowest bits first, the high bit of every byte but the last one is set. +It takes from one to ten bytes. +)"; + FunctionDocumentation::Syntax syntax = "toVarUInt(x)"; + FunctionDocumentation::Arguments arguments = {{"x", "The number.", {"UInt8", "UInt16", "UInt32", "UInt64"}}}; + FunctionDocumentation::ReturnedValue returned_value = {"The bytes of the encoded number.", {"String"}}; + FunctionDocumentation::Examples examples = { + {"Usage example", "SELECT hex(toVarUInt(300))", "AC02"}, + }; + FunctionDocumentation::IntroducedIn introduced_in = {26, 6}; + FunctionDocumentation::Category category = FunctionDocumentation::Category::String; + FunctionDocumentation documentation = {description, syntax, arguments, {}, returned_value, examples, introduced_in, category}; + + factory.registerFunction(documentation); + } +} + +} diff --git a/tests/queries/0_stateless/04327_uniq_apache_hll.sql b/tests/queries/0_stateless/04327_uniq_apache_hll.sql index cf1111734501..7d2fa6f1c145 100644 --- a/tests/queries/0_stateless/04327_uniq_apache_hll.sql +++ b/tests/queries/0_stateless/04327_uniq_apache_hll.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-cpu-s390x -- ^ DataSketches is not built in fast-test builds. SELECT 'accuracy'; @@ -37,7 +37,7 @@ SELECT uniqApacheHLL(3)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_O SELECT uniqApacheHLL(22)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } SELECT uniqApacheHLL(-1)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } SELECT uniqApacheHLL(12, 'HLL_9')(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } -SELECT uniqApacheHLL(12, 'HLL_4', 1)(number) FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT uniqApacheHLL(12, 'HLL_4', 'DEFAULT', 1)(number) FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT 'argument types'; -- Small counts stay in coupon mode, where the estimate is exact. diff --git a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh index 80c16c7e17f6..650dcae83ec2 100755 --- a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh +++ b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-cpu-s390x # no-fasttest -- compiled w/o datasketches # Malformed states must raise `CORRUPTED_DATA`, not a logical exception. diff --git a/tests/queries/0_stateless/05137_uniq_apache_hll_hash_borders.sql b/tests/queries/0_stateless/05137_uniq_apache_hll_hash_borders.sql index 589ec1b16a7d..3fa830a84749 100644 --- a/tests/queries/0_stateless/05137_uniq_apache_hll_hash_borders.sql +++ b/tests/queries/0_stateless/05137_uniq_apache_hll_hash_borders.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-cpu-s390x -- ^ DataSketches is not built in fast-test builds. -- Pins how every supported argument type is hashed, on the border values of its range. diff --git a/tests/queries/0_stateless/05138_uniq_apache_hll_null_and_empty.sql b/tests/queries/0_stateless/05138_uniq_apache_hll_null_and_empty.sql index 94282a2c346b..557c6c393606 100644 --- a/tests/queries/0_stateless/05138_uniq_apache_hll_null_and_empty.sql +++ b/tests/queries/0_stateless/05138_uniq_apache_hll_null_and_empty.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-cpu-s390x -- ^ DataSketches is not built in fast-test builds. -- `NULL` values and empty strings are ignored by the Apache DataSketches implementations in Java, Python diff --git a/tests/queries/0_stateless/05139_uniq_apache_hll_nullable_state_type.sql b/tests/queries/0_stateless/05139_uniq_apache_hll_nullable_state_type.sql index c75d1cf0153f..1c76cd9d4170 100644 --- a/tests/queries/0_stateless/05139_uniq_apache_hll_nullable_state_type.sql +++ b/tests/queries/0_stateless/05139_uniq_apache_hll_nullable_state_type.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-cpu-s390x -- ^ DataSketches is not built in fast-test builds. -- The state of a `Nullable` argument is the same bare DataSketches sketch as the state of a plain one, diff --git a/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.reference b/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.reference new file mode 100644 index 000000000000..b06e8260028f --- /dev/null +++ b/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.reference @@ -0,0 +1,10 @@ +a state read as a type with a lower lg_k and another storage type is written back unchanged +1 +the same for the default parameters +1 +the estimate of the imported state is the estimate of the sketch +1 +merging brings it down to the declared resolution +1 +an empty imported state merges to nothing +3 diff --git a/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.sql b/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.sql new file mode 100644 index 000000000000..291ca2ed2f18 --- /dev/null +++ b/tests/queries/0_stateless/05295_uniq_apache_hll_import_keeps_sketch.sql @@ -0,0 +1,30 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- A sketch that is read as a state of another type, for example one with a lower `lg_k`, is kept as it was written. +-- Reading must not lose anything that writing the state back would then persist. The sketch is brought down to the +-- declared resolution only when it is merged. + +SELECT 'a state read as a type with a lower lg_k and another storage type is written back unchanged'; +WITH (SELECT hex(toString(uniqApacheHLLState(14, 'HLL_6')(number))) FROM numbers(100000)) AS h +SELECT hex(toString(CAST(unhex(h), 'AggregateFunction(uniqApacheHLL(8, \'HLL_4\'), UInt64)'))) = h; + +SELECT 'the same for the default parameters'; +WITH (SELECT hex(toString(uniqApacheHLLState(14, 'HLL_8')(number))) FROM numbers(100000)) AS h +SELECT hex(toString(CAST(unhex(h), 'AggregateFunction(uniqApacheHLL, UInt64)'))) = h; + +SELECT 'the estimate of the imported state is the estimate of the sketch'; +WITH (SELECT uniqApacheHLLState(14)(number) FROM numbers(100000)) AS s +SELECT finalizeAggregation(CAST(unhex(hex(toString(s))), 'AggregateFunction(uniqApacheHLL(8), UInt64)')) = finalizeAggregation(s); + +SELECT 'merging brings it down to the declared resolution'; +WITH (SELECT hex(toString(uniqApacheHLLState(14)(number))) FROM numbers(100000)) AS h +SELECT length(toString(uniqApacheHLLMergeState(8)(CAST(unhex(h), 'AggregateFunction(uniqApacheHLL(8), UInt64)')))) < length(unhex(h)) / 10; + +SELECT 'an empty imported state merges to nothing'; +SELECT uniqApacheHLLMerge(s) FROM +( + SELECT CAST(unhex('080201070C030C0000'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s + UNION ALL + SELECT uniqApacheHLLState(number) AS s FROM numbers(3) +); diff --git a/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.reference b/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.reference new file mode 100644 index 000000000000..b8b4ec1525f5 --- /dev/null +++ b/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.reference @@ -0,0 +1,12 @@ +UNION ALL of states with different parameters +9 +an array of such states has one element type +Array(AggregateFunction(uniqApacheHLL(8), UInt64)) +if between such states +AggregateFunction(uniqApacheHLL(8), UInt64) +a state with a Nullable argument is the same type as well +Array(AggregateFunction(uniqApacheHLL(8), UInt64)) +a different argument type is a different type +Array(Variant(AggregateFunction(uniqApacheHLL(8), UInt64), AggregateFunction(uniqApacheHLL, String))) +changing the parameters of a column is applied +AggregateFunction(uniqApacheHLL(8), UInt64) 5 diff --git a/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.sql b/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.sql new file mode 100644 index 000000000000..c84cdae87935 --- /dev/null +++ b/tests/queries/0_stateless/05296_uniq_apache_hll_state_types_unify.sql @@ -0,0 +1,36 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- The parameters of `uniqApacheHLL` (`lg_k` and the storage type) do not change what a state means: every sketch carries +-- its own configuration, so states of any parameters can be merged. The types of such states are therefore the same type +-- for the places that need a common type, and not only convertible with `CAST`. + +SELECT 'UNION ALL of states with different parameters'; +SELECT uniqApacheHLLMerge(s) FROM +( + SELECT uniqApacheHLLState(8)(toUInt64(number)) AS s FROM numbers(3) + UNION ALL + SELECT uniqApacheHLLState(14, 'HLL_6')(toUInt64(number + 10)) AS s FROM numbers(3) + UNION ALL + SELECT uniqApacheHLLState(toUInt64(number + 20)) AS s FROM numbers(3) +); + +SELECT 'an array of such states has one element type'; +SELECT toTypeName([uniqApacheHLLState(8)(toUInt64(1)), uniqApacheHLLState(14, 'HLL_6')(toUInt64(2)), uniqApacheHLLState(toUInt64(3))]) FROM numbers(1); + +SELECT 'if between such states'; +SELECT toTypeName(if(1, uniqApacheHLLState(8)(number), uniqApacheHLLState(14, 'HLL_6')(number))) FROM numbers(1); + +SELECT 'a state with a Nullable argument is the same type as well'; +SELECT toTypeName([uniqApacheHLLState(8)(toNullable(toUInt64(1))), uniqApacheHLLState(toUInt64(2))]) FROM numbers(1); + +SELECT 'a different argument type is a different type'; +SELECT toTypeName([uniqApacheHLLState(8)(toUInt64(1)), uniqApacheHLLState(toString(2))]) FROM numbers(1); + +SELECT 'changing the parameters of a column is applied'; +DROP TABLE IF EXISTS hll_state_types; +CREATE TABLE hll_state_types (k UInt8, s AggregateFunction(uniqApacheHLL, UInt64)) ENGINE = AggregatingMergeTree ORDER BY k; +INSERT INTO hll_state_types SELECT 1, uniqApacheHLLState(number) FROM numbers(5); +ALTER TABLE hll_state_types MODIFY COLUMN s AggregateFunction(uniqApacheHLL(8), UInt64); +SELECT toTypeName(s), uniqApacheHLLMerge(8)(s) FROM hll_state_types GROUP BY toTypeName(s); +DROP TABLE hll_state_types; diff --git a/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference new file mode 100644 index 000000000000..9b745b7aae4c --- /dev/null +++ b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.reference @@ -0,0 +1,15 @@ +the composite estimate of a merged state does not depend on the order of the merges +1 +the estimator does not change the state, and states of any estimator have one type where a common type is needed +1 +2 +a small sketch has the same estimate +100 100 +the composite estimate of a sketch that is built by inserts does not depend on the order of the inserts +1 +the same for a state that is finalized, and for a state that is read +1 +1 +the composite estimate for a UInt8 key does not depend on the size of the blocks +1 +wrong parameters diff --git a/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql new file mode 100644 index 000000000000..2145b6003b76 --- /dev/null +++ b/tests/queries/0_stateless/05297_uniq_apache_hll_estimator.sql @@ -0,0 +1,55 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- The HIP estimate of the library depends on the order of the inserts and of the merges. The third parameter, `estimator`, chooses +-- the estimate: `DEFAULT` is the one of the library, `COMPOSITE` is computed from the registers of the sketch only, so it does not +-- depend on the order. Only the second is checked here, because the first one differs between orders only sometimes. + +SET max_threads = 1; + +-- 8 sketches of 100 values each, neighbours overlap by half, 450 distinct values in total, merged in three orders + +SELECT 'the composite estimate of a merged state does not depend on the order of the merges'; +SELECT count() = 3 AND uniqExact(r) = 1 AND any(r) BETWEEN 430 AND 470 FROM +( + SELECT uniqApacheHLLMerge(11, 'HLL_8', 'COMPOSITE')(s) AS r FROM (SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(0) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(1) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(2) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(3) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(4) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(5) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(6) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(7) * 50 + number) AS s FROM numbers(100)) UNION ALL + SELECT uniqApacheHLLMerge(11, 'HLL_8', 'COMPOSITE')(s) AS r FROM (SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(7) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(6) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(5) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(4) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(3) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(2) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(1) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(0) * 50 + number) AS s FROM numbers(100)) UNION ALL + SELECT uniqApacheHLLMerge(11, 'HLL_8', 'COMPOSITE')(s) AS r FROM (SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(3) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(6) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(0) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(5) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(1) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(7) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(2) * 50 + number) AS s FROM numbers(100) UNION ALL SELECT uniqApacheHLLState(11, 'HLL_8')(toUInt64(4) * 50 + number) AS s FROM numbers(100)) +); + +SELECT 'the estimator does not change the state, and states of any estimator have one type where a common type is needed'; +SELECT hex(toString(uniqApacheHLLState(11, 'HLL_8', 'COMPOSITE')(number))) = hex(toString(uniqApacheHLLState(11, 'HLL_8', 'DEFAULT')(number))) FROM numbers(1000); +SELECT length([uniqApacheHLLState(8, 'HLL_6', 'COMPOSITE')(toUInt64(1)), uniqApacheHLLState(toUInt64(2))]); + +SELECT 'a small sketch has the same estimate'; +SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(number), uniqApacheHLL(11, 'HLL_8', 'DEFAULT')(number) FROM numbers(100); + +SELECT 'the composite estimate of a sketch that is built by inserts does not depend on the order of the inserts'; +SELECT count() = 3 AND uniqExact(r) = 1 AND any(r) BETWEEN 2800 AND 3200 FROM +( + SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(x) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY x) UNION ALL + SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(x) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY -x) UNION ALL + SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(x) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY cityHash64(x)) +); + +SELECT 'the same for a state that is finalized, and for a state that is read'; +SELECT count() = 3 AND uniqExact(r) = 1 FROM +( + SELECT finalizeAggregation(uniqApacheHLLState(11, 'HLL_8', 'COMPOSITE')(x)) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY x) UNION ALL + SELECT finalizeAggregation(uniqApacheHLLState(11, 'HLL_8', 'COMPOSITE')(x)) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY -x) UNION ALL + SELECT finalizeAggregation(uniqApacheHLLState(11, 'HLL_8', 'COMPOSITE')(x)) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY cityHash64(x)) +); +SELECT count() = 3 AND uniqExact(r) = 1 FROM +( + SELECT finalizeAggregation(CAST(toString(uniqApacheHLLState(11, 'HLL_8')(x)), 'AggregateFunction(uniqApacheHLL(11, \'HLL_8\', \'COMPOSITE\'), UInt64)')) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY x) UNION ALL + SELECT finalizeAggregation(CAST(toString(uniqApacheHLLState(11, 'HLL_8')(x)), 'AggregateFunction(uniqApacheHLL(11, \'HLL_8\', \'COMPOSITE\'), UInt64)')) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY -x) UNION ALL + SELECT finalizeAggregation(CAST(toString(uniqApacheHLLState(11, 'HLL_8')(x)), 'AggregateFunction(uniqApacheHLL(11, \'HLL_8\', \'COMPOSITE\'), UInt64)')) AS r FROM (SELECT number AS x FROM numbers(3000) ORDER BY cityHash64(x)) +); + +SELECT 'the composite estimate for a UInt8 key does not depend on the size of the blocks'; +SELECT (SELECT arraySort(groupArray((k, r))) FROM (SELECT toUInt8(number % 7) AS k, uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(number) AS r FROM numbers(30000) GROUP BY k SETTINGS max_block_size = 1000)) = (SELECT arraySort(groupArray((k, r))) FROM (SELECT toUInt8(number % 7) AS k, uniqApacheHLL(11, 'HLL_8', 'COMPOSITE')(number) AS r FROM numbers(30000) GROUP BY k SETTINGS max_block_size = 65536)); + +SELECT 'wrong parameters'; +SELECT uniqApacheHLL(11, 'HLL_8', 'HIP')(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } +SELECT uniqApacheHLL(11, 'HLL_8', 1)(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } +SELECT uniqApacheHLL(11, 'HLL_8', 'COMPOSITE', 1)(number) FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } diff --git a/tests/queries/0_stateless/05298_var_uint_length_prefix.reference b/tests/queries/0_stateless/05298_var_uint_length_prefix.reference new file mode 100644 index 000000000000..d5b6150623a6 --- /dev/null +++ b/tests/queries/0_stateless/05298_var_uint_length_prefix.reference @@ -0,0 +1,20 @@ +toVarUInt +00 7F 8001 AC02 FF7F 808001 FFFFFFFFFFFFFFFFFF01 +C801 E0D403 80D0ACF30E +00 +01 +02 +addVarUIntLengthPrefix +03616263 00 0461620000 +7F61 800161 AC0261 +128 130 16387 +removeVarUIntLengthPrefix +abc a +a padded length is read as RowBinary reads it +a a +a round trip gives the same string +200 200 +1 +1 +wrong lengths +wrong argument types diff --git a/tests/queries/0_stateless/05298_var_uint_length_prefix.sql b/tests/queries/0_stateless/05298_var_uint_length_prefix.sql new file mode 100644 index 000000000000..e0cfdb9b0016 --- /dev/null +++ b/tests/queries/0_stateless/05298_var_uint_length_prefix.sql @@ -0,0 +1,37 @@ +-- `toVarUInt` encodes a number as an unsigned LEB128 `VarUInt`, `addVarUIntLengthPrefix` puts the length of a string in front of it as such a number, `removeVarUIntLengthPrefix` takes it off. + +SELECT 'toVarUInt'; +SELECT hex(toVarUInt(0)), hex(toVarUInt(127)), hex(toVarUInt(128)), hex(toVarUInt(300)), hex(toVarUInt(16383)), hex(toVarUInt(16384)), hex(toVarUInt(toUInt64(18446744073709551615))); +SELECT hex(toVarUInt(toUInt8(200))), hex(toVarUInt(toUInt16(60000))), hex(toVarUInt(toUInt32(4000000000))); +SELECT hex(toVarUInt(number)) FROM numbers(3); + +SELECT 'addVarUIntLengthPrefix'; +SELECT hex(addVarUIntLengthPrefix('abc')), hex(addVarUIntLengthPrefix('')), hex(addVarUIntLengthPrefix(toFixedString('ab', 4))); +SELECT hex(substring(addVarUIntLengthPrefix(repeat('a', 127)), 1, 2)), hex(substring(addVarUIntLengthPrefix(repeat('a', 128)), 1, 3)), hex(substring(addVarUIntLengthPrefix(repeat('a', 300)), 1, 3)); +SELECT length(addVarUIntLengthPrefix(repeat('a', 127))), length(addVarUIntLengthPrefix(repeat('a', 128))), length(addVarUIntLengthPrefix(repeat('a', 16384))); + +SELECT 'removeVarUIntLengthPrefix'; +SELECT removeVarUIntLengthPrefix(unhex('03616263')), hex(removeVarUIntLengthPrefix(unhex('00'))), removeVarUIntLengthPrefix(toFixedString(unhex('0161'), 2)); +SELECT 'a padded length is read as RowBinary reads it'; +SELECT removeVarUIntLengthPrefix(unhex('810061')), removeVarUIntLengthPrefix(unhex('81800061')); + +SELECT 'a round trip gives the same string'; +SELECT countIf(removeVarUIntLengthPrefix(addVarUIntLengthPrefix(s)) = s), count() FROM (SELECT repeat(char(number % 256), number * 37 % 40000) AS s FROM numbers(200)); +SELECT removeVarUIntLengthPrefix(addVarUIntLengthPrefix(CAST(NULL, 'Nullable(String)'))) IS NULL; +SELECT addVarUIntLengthPrefix(materialize('')) = unhex('00'); + +SELECT 'wrong lengths'; +SELECT removeVarUIntLengthPrefix(''); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('80')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('0261')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('016162')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('00ff')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('FFFFFFFFFFFFFFFFFF02')); -- { serverError INCORRECT_DATA } +SELECT removeVarUIntLengthPrefix(unhex('FFFFFFFFFFFFFFFFFFFF01')); -- { serverError INCORRECT_DATA } + +SELECT 'wrong argument types'; +SELECT toVarUInt(-1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toVarUInt(toInt64(1)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toVarUInt('1'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT addVarUIntLengthPrefix(1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT removeVarUIntLengthPrefix(1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.reference b/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.reference new file mode 100644 index 000000000000..71c5b73887c4 --- /dev/null +++ b/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.reference @@ -0,0 +1,11 @@ +it can be read after the length is added +5 +merged without a state column +5 +a state gives the bytes of its sketch +0301070B05080009 +a sketch that is read and written back is the same sketch +1 +a big sketch needs a longer length +1 1 +a state that is not a length and bytes is rejected diff --git a/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.sql b/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.sql new file mode 100644 index 000000000000..734ba2ee53d6 --- /dev/null +++ b/tests/queries/0_stateless/05299_var_uint_length_prefix_uniq_apache_hll.sql @@ -0,0 +1,26 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- A sketch that another system wrote without a length (for example, Spark `hll_sketch_agg` into Parquet) is read as a state with +-- `addVarUIntLengthPrefix`, and the bytes of a state are given to another system with `removeVarUIntLengthPrefix`. + +SELECT 'it can be read after the length is added'; +SELECT finalizeAggregation(CAST(addVarUIntLengthPrefix(unhex('0201070b03080508cbd7c2042bf2fb06862ff90d7581660781bc5d06')), 'AggregateFunction(uniqApacheHLL, UInt64)')); + +SELECT 'merged without a state column'; +SELECT uniqApacheHLLMerge(CAST(addVarUIntLengthPrefix(sketch), 'AggregateFunction(uniqApacheHLL, UInt64)')) +FROM (SELECT unhex('0201070b03080508cbd7c2042bf2fb06862ff90d7581660781bc5d06') AS sketch FROM numbers(3)); + +SELECT 'a state gives the bytes of its sketch'; +SELECT hex(substring(removeVarUIntLengthPrefix(toString(uniqApacheHLLState(11, 'HLL_8')(number))), 1, 8)) FROM numbers(10); + +SELECT 'a sketch that is read and written back is the same sketch'; +WITH unhex('0201070b03080508cbd7c2042bf2fb06862ff90d7581660781bc5d06') AS sketch +SELECT removeVarUIntLengthPrefix(toString(CAST(addVarUIntLengthPrefix(sketch), 'AggregateFunction(uniqApacheHLL, UInt64)'))) = sketch; + +SELECT 'a big sketch needs a longer length'; +WITH (SELECT removeVarUIntLengthPrefix(toString(uniqApacheHLLState(14, 'HLL_8')(number))) FROM numbers(100000)) AS sketch +SELECT length(sketch) > 16000, finalizeAggregation(CAST(addVarUIntLengthPrefix(sketch), 'AggregateFunction(uniqApacheHLL(14, \'HLL_8\'), UInt64)')) = uniqApacheHLL(14, 'HLL_8')(number) FROM numbers(100000); + +SELECT 'a state that is not a length and bytes is rejected'; +SELECT removeVarUIntLengthPrefix(toString(countState())); -- { serverError INCORRECT_DATA } diff --git a/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.reference b/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.reference new file mode 100644 index 000000000000..1bb602f6e9ef --- /dev/null +++ b/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.reference @@ -0,0 +1,2 @@ +OK without the length +5 diff --git a/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.sh b/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.sh new file mode 100755 index 000000000000..c9dcf3ca74d0 --- /dev/null +++ b/tests/queries/0_stateless/05300_var_uint_length_prefix_without_length.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-cpu-s390x +# no-fasttest -- compiled w/o datasketches + +# A sketch that another system wrote without a length in front of it is not a state of `uniqApacheHLL`: the state reader takes +# its first bytes for the length. `CORRUPTED_DATA` is always logged by the client, so this is a shell test, as in +# `05136_uniq_apache_hll_corrupted_state.sh`. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SKETCH="unhex('0201070b03080508cbd7c2042bf2fb06862ff90d7581660781bc5d06')" + +$CLICKHOUSE_CLIENT --query \ + "SELECT finalizeAggregation(CAST($SKETCH, 'AggregateFunction(uniqApacheHLL, UInt64)'))" 2>&1 \ + | grep -q -F 'CORRUPTED_DATA' && echo 'OK without the length' || echo 'FAIL without the length' + +$CLICKHOUSE_CLIENT --query \ + "SELECT finalizeAggregation(CAST(addVarUIntLengthPrefix($SKETCH), 'AggregateFunction(uniqApacheHLL, UInt64)'))" 2>&1 diff --git a/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.reference b/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.reference new file mode 100644 index 000000000000..c21307d22faa --- /dev/null +++ b/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.reference @@ -0,0 +1,6 @@ +uniq 0 +uniqApacheHLL 1 +uniqApacheHLL params 1 +uniqApacheHLLState 1 +uniqApacheHLLMerge 1 +uniqApacheHLLIf 1 diff --git a/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.sql b/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.sql new file mode 100644 index 000000000000..b16d71d9ad40 --- /dev/null +++ b/tests/queries/0_stateless/05301_uniq_apache_hll_order_dependent.sql @@ -0,0 +1,14 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- The HIP estimate and the bytes of a `uniqApacheHLL` sketch depend on the order of the updates and of the merges, so the planner must keep an `ORDER BY` below it, +-- also below its `-State`, `-Merge` and `-If` forms. `uniq` is the control: its result does not depend on the order, so the sorting below it is removed. + +SET query_plan_remove_redundant_sorting = 1; +SET optimize_aggregators_of_group_by_keys = 0; +SELECT 'uniq', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniq(x) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); +SELECT 'uniqApacheHLL', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLL(x) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); +SELECT 'uniqApacheHLL params', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLL(14, 'HLL_8')(x) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); +SELECT 'uniqApacheHLLState', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLLState(x) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); +SELECT 'uniqApacheHLLMerge', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLLMerge(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(10) GROUP BY number % 3 ORDER BY number % 3 DESC)); +SELECT 'uniqApacheHLLIf', countIf(explain LIKE '%Sorting%') FROM (EXPLAIN SELECT uniqApacheHLLIf(x, x > 2) FROM (SELECT number AS x FROM numbers(10) ORDER BY number DESC)); diff --git a/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.reference b/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.reference new file mode 100644 index 000000000000..6664c4a9c9d2 --- /dev/null +++ b/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.reference @@ -0,0 +1,17 @@ +by value, as Java and Spark hash a long +1 +1 +1 +a cast to the signed type of the same width gives the sketch of the C++ library +1 +1 +1 +1 +1 +1 +the two are different sketches for these values +1 +the values below the signed range are not affected +1 +1 +a cast that checks the range rejects these values diff --git a/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.sql b/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.sql new file mode 100644 index 000000000000..d114cdbf49be --- /dev/null +++ b/tests/queries/0_stateless/05302_uniq_apache_hll_unsigned_cast.sql @@ -0,0 +1,28 @@ +-- Tags: no-fasttest, no-cpu-s390x +-- ^ DataSketches is not built in fast-test builds. + +-- An unsigned `UInt8`, `UInt16` or `UInt32` value is hashed as the same number as a 64-bit integer. The C++ library hashes such a value +-- through the signed type of the same width, and its sketch is what the documented casts to that type give. + +SELECT 'by value, as Java and Spark hash a long'; +SELECT hex(toString(uniqApacheHLLState(toUInt32(3000000000)))) = hex(toString(uniqApacheHLLState(toInt64(3000000000)))); +SELECT hex(toString(uniqApacheHLLState(toUInt16(50000)))) = hex(toString(uniqApacheHLLState(toInt64(50000)))); +SELECT hex(toString(uniqApacheHLLState(toUInt8(200)))) = hex(toString(uniqApacheHLLState(toInt64(200)))); + +SELECT 'a cast to the signed type of the same width gives the sketch of the C++ library'; +SELECT hex(toString(uniqApacheHLLState(toInt32(toUInt32(3000000000))))) = hex(toString(uniqApacheHLLState(toInt64(-1294967296)))); +SELECT hex(toString(uniqApacheHLLState(toInt16(toUInt16(50000))))) = hex(toString(uniqApacheHLLState(toInt64(-15536)))); +SELECT hex(toString(uniqApacheHLLState(toInt8(toUInt8(200))))) = hex(toString(uniqApacheHLLState(toInt64(-56)))); +SELECT hex(toString(uniqApacheHLLState(toInt32(toUInt32(toIPv4('192.168.0.1')))))) = hex(toString(uniqApacheHLLState(toInt64(-1062731775)))); +SELECT hex(toString(uniqApacheHLLState(toInt32(toDateTime(2147483648, 'UTC'))))) = hex(toString(uniqApacheHLLState(toInt64(-2147483648)))); +SELECT hex(toString(uniqApacheHLLState(toInt16(toUInt16(toDate(32768)))))) = hex(toString(uniqApacheHLLState(toInt64(-32768)))); + +SELECT 'the two are different sketches for these values'; +SELECT hex(toString(uniqApacheHLLState(toUInt32(3000000000)))) != hex(toString(uniqApacheHLLState(toInt32(toUInt32(3000000000))))); + +SELECT 'the values below the signed range are not affected'; +SELECT hex(toString(uniqApacheHLLState(toUInt32(2147483647)))) = hex(toString(uniqApacheHLLState(toInt32(2147483647)))); +SELECT hex(toString(uniqApacheHLLState(toIPv4('10.0.0.1')))) = hex(toString(uniqApacheHLLState(toInt32(toUInt32(toIPv4('10.0.0.1')))))); + +SELECT 'a cast that checks the range rejects these values'; +SELECT accurateCast(toUInt32(3000000000), 'Int32'); -- { serverError CANNOT_CONVERT_TYPE }