From a1610ddb16a94dadb8f7a2e35060348de6c75078 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 15 Sep 2026 23:25:19 +0000 Subject: [PATCH 001/296] Backport #115670 to 26.8: Fix substreams cache key collision between streams of one column --- src/DataTypes/IDataType.cpp | 2 +- .../Serializations/ISerialization.cpp | 22 ++- src/DataTypes/Serializations/ISerialization.h | 9 +- .../MergeTree/MergeTreeReaderWide.cpp | 2 +- src/Storages/StorageLog.cpp | 4 +- ...7_substreams_cache_key_collision.reference | 184 ++++++++++++++++++ .../04927_substreams_cache_key_collision.sql | 167 ++++++++++++++++ 7 files changed, 376 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/04927_substreams_cache_key_collision.reference create mode 100644 tests/queries/0_stateless/04927_substreams_cache_key_collision.sql diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index ccb9ccad65a67..ccc8e24295bef 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -163,7 +163,7 @@ std::unique_ptr IDataType::getSubcolumnData( size_t prefix_len = i + 1; if (!subpath[i].visited && ISerialization::hasSubcolumnForPath(subpath, prefix_len)) { - auto name = ISerialization::getSubcolumnNameForStream(subpath, prefix_len, false, initial_array_level); + auto name = ISerialization::getSubcolumnNameForStream(subpath, prefix_len, initial_array_level); /// Create data from path only if it's requested subcolumn. /// Use the first exact match to be consistent with ColumnsDescription::addSubcolumns /// which also keeps the first subcolumn when there are name collisions diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 7b1602de58b7f..12aca2a07c32a 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -471,20 +471,26 @@ String ISerialization::getFileNameForRenamedColumnStream(const NameAndTypePair & return getFileNameForRenamedColumnStream(column_from.getNameInStorage(), column_to.getNameInStorage(), file_name); } -String ISerialization::getSubcolumnNameForStream(const SubstreamPath & path, bool encode_sparse_stream, size_t initial_array_level) +String ISerialization::getSubcolumnNameForStream(const SubstreamPath & path) { - return getSubcolumnNameForStream(path, path.size(), encode_sparse_stream, initial_array_level); + return getSubcolumnNameForStream(path, path.size()); } -String ISerialization::getSubcolumnNameForStream(const SubstreamPath & path, size_t prefix_len, bool encode_sparse_stream, size_t initial_array_level) +String ISerialization::getSubcolumnNameForStream(const SubstreamPath & path, size_t prefix_len, size_t initial_array_level) { - auto subcolumn_name = getNameForSubstreamPath("", path.begin(), path.begin() + prefix_len, false, encode_sparse_stream, false, initial_array_level); + auto subcolumn_name = getNameForSubstreamPath("", path.begin(), path.begin() + prefix_len, false, false, false, initial_array_level); if (!subcolumn_name.empty()) subcolumn_name = subcolumn_name.substr(1); // It starts with a dot. return subcolumn_name; } +String ISerialization::getSubstreamsCacheKeyForStream(const SubstreamPath & path) +{ + /// Unlike the subcolumn name, this rendering is injective, so two substreams never share a cache slot. + return getNameForSubstreamPath("", path.begin(), path.end(), /*escape_for_file_name=*/true, /*encode_sparse_stream=*/true, /*escape_variant_substreams=*/true); +} + namespace { @@ -530,7 +536,7 @@ void ISerialization::addElementToSubstreamsCache(ISerialization::SubstreamsCache if (!cache) return; - cache->insert_or_assign(getSubcolumnNameForStream(path, true), std::move(element)); + cache->insert_or_assign(getSubstreamsCacheKeyForStream(path), std::move(element)); } ISerialization::ISubstreamsCacheElement * ISerialization::getElementFromSubstreamsCache(ISerialization::SubstreamsCache * cache, const ISerialization::SubstreamPath & path) @@ -538,7 +544,7 @@ ISerialization::ISubstreamsCacheElement * ISerialization::getElementFromSubstrea if (!cache) return nullptr; - auto it = cache->find(getSubcolumnNameForStream(path, true)); + auto it = cache->find(getSubstreamsCacheKeyForStream(path)); return it == cache->end() ? nullptr : it->second.get(); } @@ -547,7 +553,7 @@ void ISerialization::addToSubstreamsDeserializeStatesCache(SubstreamsDeserialize if (!cache) return; - cache->emplace(getSubcolumnNameForStream(path, true), state); + cache->emplace(getSubstreamsCacheKeyForStream(path), state); } ISerialization::DeserializeBinaryBulkStatePtr ISerialization::getFromSubstreamsDeserializeStatesCache(SubstreamsDeserializeStatesCache * cache, const SubstreamPath & path) @@ -555,7 +561,7 @@ ISerialization::DeserializeBinaryBulkStatePtr ISerialization::getFromSubstreamsD if (!cache) return nullptr; - auto it = cache->find(getSubcolumnNameForStream(path, true)); + auto it = cache->find(getSubstreamsCacheKeyForStream(path)); return it == cache->end() ? nullptr : it->second; } diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index 938c3ef9a5262..abfe3cdce0b32 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -699,8 +699,13 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro static String getFileNameForRenamedColumnStream(const NameAndTypePair & column_from, const NameAndTypePair & column_to, const String & file_name); static String getFileNameForRenamedColumnStream(const String & name_from, const String & name_to, const String & file_name); - static String getSubcolumnNameForStream(const SubstreamPath & path, bool encode_sparse_stream = false, size_t initial_array_level = 0); - static String getSubcolumnNameForStream(const SubstreamPath & path, size_t prefix_len, bool encode_sparse_stream = false, size_t initial_array_level = 0); + static String getSubcolumnNameForStream(const SubstreamPath & path); + static String getSubcolumnNameForStream(const SubstreamPath & path, size_t prefix_len, size_t initial_array_level = 0); + /// Rejects a stale `(path, true)` call, which would otherwise silently bind `true` to `prefix_len`. + static String getSubcolumnNameForStream(const SubstreamPath & path, bool) = delete; + + /// Key of a stream in SubstreamsCache and SubstreamsDeserializeStatesCache. + static String getSubstreamsCacheKeyForStream(const SubstreamPath & path); static void addColumnWithNumReadRowsToSubstreamsCache(SubstreamsCache * cache, const SubstreamPath & path, ColumnPtr column, size_t num_read_rows); static std::optional> getColumnWithNumReadRowsFromSubstreamsCache(SubstreamsCache * cache, const SubstreamPath & path); diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 6785042a07c59..71bd9c2aeaa1d 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -332,7 +332,7 @@ ReadBuffer * MergeTreeReaderWide::getStream( ISerialization::SubstreamsCache & cache) { /// If substream have already been read. - if (cache.contains(ISerialization::getSubcolumnNameForStream(substream_path))) + if (cache.contains(ISerialization::getSubstreamsCacheKeyForStream(substream_path))) return nullptr; auto stream_name = IMergeTreeDataPart::getStreamNameForColumn(name_and_type, substream_path, ".bin", checksums, storage_settings); diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index dd5ceb02d1204..9fc213141e70e 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -304,7 +304,7 @@ void LogSource::readPrefix(const NameAndTypePair & name_and_type, ISerialization ISerialization::DeserializeBinaryBulkSettings settings; settings.getter = [&](const ISerialization::SubstreamPath & path) -> ReadBuffer * { - if (cache.contains(ISerialization::getSubcolumnNameForStream(path))) + if (cache.contains(ISerialization::getSubstreamsCacheKeyForStream(path))) return nullptr; String data_file_name = ISerialization::getFileNameForStream(name_and_type, path, {}); @@ -333,7 +333,7 @@ void LogSource::readData(const NameAndTypePair & name_and_type, MutableColumnPtr settings.getter = [&] (const ISerialization::SubstreamPath & path) -> ReadBuffer * { - if (cache.contains(ISerialization::getSubcolumnNameForStream(path))) + if (cache.contains(ISerialization::getSubstreamsCacheKeyForStream(path))) return nullptr; String data_file_name = ISerialization::getFileNameForStream(name_and_type, path, {}); diff --git a/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference b/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference new file mode 100644 index 0000000000000..ba91c81af2da0 --- /dev/null +++ b/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference @@ -0,0 +1,184 @@ +[(100),(200)] ('abc',99) +[(300)] ('de',98) +[] ('',97) +4 +Row 1: +────── +c1: ([(100),(200)],1) +c2: ([('abc')],2) +c3: ([('abc')],3) +c4: ([([10,20])],4) +c5: ([(1),(NULL)],5) +c6: ([[(10),(20)]],6) +c7: ([[(10),(20)]],7) +c8: ([(1),NULL],8) +c9: ({'k':[(1),(2)]},9) +c10: (('abc',99),10) +c11: (([10,20],99),11) +c12: ((5,7),12) +Row 1: +────── +c1.x: [(100),(200)] +c2.x: [('abc')] +c3.x: [('abc')] +c4.x: [([10,20])] +c5.x: [(1),(NULL)] +c6.x: [[(10),(20)]] +c7.x: [[(10),(20)]] +c8.x: [(1),NULL] +c9.x: {'k':[(1),(2)]} +c10.x: ('abc',99) +c11.x: ([10,20],99) +c12.x: (5,7) +Row 1: +────── +c1: ([(100),(200)],1) +c2: ([('abc')],2) +c3: ([('abc')],3) +c4: ([([10,20])],4) +c5: ([(1),(NULL)],5) +c6: ([[(10),(20)]],6) +c7: ([[(10),(20)]],7) +c8: ([(1),NULL],8) +c9: ({'k':[(1),(2)]},9) +c10: (('abc',99),10) +c11: (([10,20],99),11) +c12: ((5,7),12) +Row 1: +────── +c1.x: [(100),(200)] +c2.x: [('abc')] +c3.x: [('abc')] +c4.x: [([10,20])] +c5.x: [(1),(NULL)] +c6.x: [[(10),(20)]] +c7.x: [[(10),(20)]] +c8.x: [(1),NULL] +c9.x: {'k':[(1),(2)]} +c10.x: ('abc',99) +c11.x: ([10,20],99) +c12.x: (5,7) +Row 1: +────── +c1: ('{"object_shared_data":{"0":{"size0":1}}}',1) +c2: ('{"a":[1,2],"a":{"size0":7}}',2) +c3: ('{"a":1,"a":{"null":7}}',3) +c4: ('{"a":"abc","a":{"size":7}}',4) +c5: ('{"a":1,"a":{"Int64":7}}',5) +c6: ('{"a":1,"a":{"Int64":7}}',6) +c7: ('{"a":{"b":0},"a":{"b":1}}',7) +Row 1: +────── +c1.x: { + "object_shared_data": { + "0": { + "size0": 1 + } + } +} +c2.x: { + "a": [1,2], + "a": { + "size0": 7 + } +} +c3.x: { + "a": 1, + "a": { + "null": 7 + } +} +c4.x: { + "a": "abc", + "a": { + "size": 7 + } +} +c5.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c6.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c7.x: { + "a": { + "b": 0 + }, + "a": { + "b": 1 + } +} +Row 1: +────── +c1: ('{"object_shared_data":{"0":{"size0":1}}}',1) +c2: ('{"a":[1,2],"a":{"size0":7}}',2) +c3: ('{"a":1,"a":{"null":7}}',3) +c4: ('{"a":"abc","a":{"size":7}}',4) +c5: ('{"a":1,"a":{"Int64":7}}',5) +c6: ('{"a":1,"a":{"Int64":7}}',6) +c7: ('{"a":{"b":0},"a":{"b":1}}',7) +Row 1: +────── +c1.x: { + "object_shared_data": { + "0": { + "size0": 1 + } + } +} +c2.x: { + "a": [1,2], + "a": { + "size0": 7 + } +} +c3.x: { + "a": 1, + "a": { + "null": 7 + } +} +c4.x: { + "a": "abc", + "a": { + "size": 7 + } +} +c5.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c6.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c7.x: { + "a": { + "b": 0 + }, + "a": { + "b": 1 + } +} +12495667661173940433 30 14365225858895003680 +5579088841599825588 20 3931234904921333272 +11007181315820846960 20 16997049742924259240 3194560342723893250 +9842334541689766390 20 3194560342723893250 +1328710051673613252 0 11053694533033933970 +9067169503920147426 190 +1 +9469320946673274046 2 13032090342589669733 2 2268159345449755554 10 +[(100),(200)] ('abc',99) +[(300)] ('de',98) +[(100),(200)] ('abc',99) +[(300)] ('de',98) diff --git a/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql b/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql new file mode 100644 index 0000000000000..044b3a0d321be --- /dev/null +++ b/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql @@ -0,0 +1,167 @@ +-- Two streams of one column whose files differ (`c.size0` and `c%2Esize0` for Array(Tuple(`size0` UInt64))) +-- used to share a substreams cache slot, so one of them got the other's column. The data on disk is correct. +-- The wide reader uses the cache for every read, the compact reader only for subcolumn reads. + +SET enable_nullable_tuple_type = 1; + +DROP TABLE IF EXISTS t_cache_key; + +CREATE TABLE t_cache_key (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_cache_key VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)), ([], ('', 97)); +SELECT c, d FROM t_cache_key; + +-- A merge reads the parts the same way. +INSERT INTO t_cache_key VALUES ([(400)], ('f', 96)); +OPTIMIZE TABLE t_cache_key FINAL; +SELECT count() FROM t_cache_key; + +DROP TABLE t_cache_key; + +DROP TABLE IF EXISTS t_cache_wide; +DROP TABLE IF EXISTS t_cache_compact; + +-- Every column is wrapped in a Tuple: the compact reader allocates a substreams cache only for subcolumn +-- reads of a part that has substream marks, hence `write_marks_for_substreams_in_compact_parts` below. +CREATE TABLE t_cache_wide +( + -- A Tuple element named like the array sizes of an enclosing Array. + c1 Tuple(x Array(Tuple(`size0` UInt64)), y UInt8), + c2 Tuple(x Array(Tuple(`size0` String)), y UInt8), + c3 Tuple(x Array(Tuple(`size0` LowCardinality(String))), y UInt8), + c4 Tuple(x Array(Tuple(`size0` Array(UInt64))), y UInt8), + c5 Tuple(x Array(Tuple(`size0` Nullable(UInt64))), y UInt8), + c6 Tuple(x Array(Array(Tuple(`size0` UInt64))), y UInt8), + c7 Tuple(x Array(Array(Tuple(`size1` UInt64))), y UInt8), + c8 Tuple(x Array(Nullable(Tuple(`size0` Int64))), y UInt8), + c9 Tuple(x Map(String, Array(Tuple(`size1` UInt64))), y UInt8), + -- A Tuple element named like an automatic subcolumn of a sibling element. + c10 Tuple(x Tuple(`a` String, `a.size` UInt64), y UInt8), + c11 Tuple(x Tuple(`a` Array(UInt64), `a.size0` UInt64), y UInt8), + c12 Tuple(x Tuple(`a` Nullable(UInt64), `a.null` UInt8), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; + +CREATE TABLE t_cache_compact +( + c1 Tuple(x Array(Tuple(`size0` UInt64)), y UInt8), + c2 Tuple(x Array(Tuple(`size0` String)), y UInt8), + c3 Tuple(x Array(Tuple(`size0` LowCardinality(String))), y UInt8), + c4 Tuple(x Array(Tuple(`size0` Array(UInt64))), y UInt8), + c5 Tuple(x Array(Tuple(`size0` Nullable(UInt64))), y UInt8), + c6 Tuple(x Array(Array(Tuple(`size0` UInt64))), y UInt8), + c7 Tuple(x Array(Array(Tuple(`size1` UInt64))), y UInt8), + c8 Tuple(x Array(Nullable(Tuple(`size0` Int64))), y UInt8), + c9 Tuple(x Map(String, Array(Tuple(`size1` UInt64))), y UInt8), + c10 Tuple(x Tuple(`a` String, `a.size` UInt64), y UInt8), + c11 Tuple(x Tuple(`a` Array(UInt64), `a.size0` UInt64), y UInt8), + c12 Tuple(x Tuple(`a` Nullable(UInt64), `a.null` UInt8), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = '10G', min_rows_for_wide_part = 1000000000, write_marks_for_substreams_in_compact_parts = 1; + +INSERT INTO t_cache_wide SELECT ([tuple(100), tuple(200)], 1), ([tuple('abc')], 2), ([tuple('abc')], 3), ([tuple([10, 20])], 4), ([tuple(1), tuple(NULL)], 5), ([[tuple(10), tuple(20)]], 6), ([[tuple(10), tuple(20)]], 7), ([tuple(1), NULL], 8), (map('k', [tuple(1), tuple(2)]), 9), (('abc', 99), 10), (([10, 20], 99), 11), ((5, 7), 12); +INSERT INTO t_cache_compact SELECT ([tuple(100), tuple(200)], 1), ([tuple('abc')], 2), ([tuple('abc')], 3), ([tuple([10, 20])], 4), ([tuple(1), tuple(NULL)], 5), ([[tuple(10), tuple(20)]], 6), ([[tuple(10), tuple(20)]], 7), ([tuple(1), NULL], 8), (map('k', [tuple(1), tuple(2)]), 9), (('abc', 99), 10), (([10, 20], 99), 11), ((5, 7), 12); + +SELECT * FROM t_cache_wide FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x, c8.x, c9.x, c10.x, c11.x, c12.x FROM t_cache_wide FORMAT Vertical; +SELECT * FROM t_cache_compact FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x, c8.x, c9.x, c10.x, c11.x, c12.x FROM t_cache_compact FORMAT Vertical; + +DROP TABLE t_cache_wide; +DROP TABLE t_cache_compact; + +-- The same with JSON typed paths named like a substream of a sibling path or like an internal stream. +DROP TABLE IF EXISTS t_cache_json_wide; +DROP TABLE IF EXISTS t_cache_json_compact; + +CREATE TABLE t_cache_json_wide +( + c1 Tuple(x JSON(`object_shared_data.0.size0` Int64), y UInt8), + c2 Tuple(x JSON(`a` Array(Int64), `a.size0` Int64), y UInt8), + c3 Tuple(x JSON(`a` Nullable(Int64), `a.null` Int64), y UInt8), + c4 Tuple(x JSON(`a` String, `a.size` Int64), y UInt8), + c5 Tuple(x JSON(`a` Dynamic, `a.Int64` Int64), y UInt8), + c6 Tuple(x JSON(`a` Variant(Int64, String), `a.Int64` Int64), y UInt8), + c7 Tuple(x JSON(`a` JSON(`b` Int64), `a.b` Int64), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; + +CREATE TABLE t_cache_json_compact +( + c1 Tuple(x JSON(`object_shared_data.0.size0` Int64), y UInt8), + c2 Tuple(x JSON(`a` Array(Int64), `a.size0` Int64), y UInt8), + c3 Tuple(x JSON(`a` Nullable(Int64), `a.null` Int64), y UInt8), + c4 Tuple(x JSON(`a` String, `a.size` Int64), y UInt8), + c5 Tuple(x JSON(`a` Dynamic, `a.Int64` Int64), y UInt8), + c6 Tuple(x JSON(`a` Variant(Int64, String), `a.Int64` Int64), y UInt8), + c7 Tuple(x JSON(`a` JSON(`b` Int64), `a.b` Int64), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = '10G', min_rows_for_wide_part = 1000000000, write_marks_for_substreams_in_compact_parts = 1; + +INSERT INTO t_cache_json_wide VALUES (('{"object_shared_data.0.size0" : 1}', 1), ('{"a" : [1, 2], "a.size0" : 7}', 2), ('{"a" : 1, "a.null" : 7}', 3), ('{"a" : "abc", "a.size" : 7}', 4), ('{"a" : 1, "a.Int64" : 7}', 5), ('{"a" : 1, "a.Int64" : 7}', 6), ('{"a" : {"b" : 1}}', 7)); +INSERT INTO t_cache_json_compact VALUES (('{"object_shared_data.0.size0" : 1}', 1), ('{"a" : [1, 2], "a.size0" : 7}', 2), ('{"a" : 1, "a.null" : 7}', 3), ('{"a" : "abc", "a.size" : 7}', 4), ('{"a" : 1, "a.Int64" : 7}', 5), ('{"a" : 1, "a.Int64" : 7}', 6), ('{"a" : {"b" : 1}}', 7)); + +SELECT * FROM t_cache_json_wide FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x FROM t_cache_json_wide FORMAT Vertical; +SELECT * FROM t_cache_json_compact FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x FROM t_cache_json_compact FORMAT Vertical; + +DROP TABLE t_cache_json_wide; +DROP TABLE t_cache_json_compact; + +-- Whole column and subcolumn reads must keep the same key: the wide reader keys streams by file name, so +-- without sharing the second read continues from where the first stopped. Needs several granules. +DROP TABLE IF EXISTS t_cache_sharing; + +SET flatten_nested = 0; + +CREATE TABLE t_cache_sharing +( + i UInt64, + arr Array(Nullable(UInt64)), + arr2 Array(Array(String)), + m Map(String, UInt64), + n Nested(a UInt64, b String), + s Nullable(String), + j JSON(`p` Int64) +) +ENGINE = MergeTree ORDER BY i SETTINGS min_bytes_for_wide_part = 0, index_granularity = 3; + +INSERT INTO t_cache_sharing SELECT number, range(number % 4), [[toString(number)]], map('k', number), [(number, toString(number))], toString(number), '{"p" : ' || toString(number) || '}' FROM numbers(20); + +SELECT sum(cityHash64(arr)), sum(arr.size0), sum(cityHash64(arr.null)) FROM t_cache_sharing; +SELECT sum(cityHash64(arr2)), sum(arr2.size0), sum(cityHash64(arr2.size1)) FROM t_cache_sharing; +SELECT sum(cityHash64(m)), sum(m.size0), sum(cityHash64(m.keys)), sum(cityHash64(m.values)) FROM t_cache_sharing; +SELECT sum(cityHash64(n)), sum(n.size0), sum(cityHash64(n.a)) FROM t_cache_sharing; +SELECT sum(cityHash64(s)), sum(s.null), sum(cityHash64(s.size)) FROM t_cache_sharing; +SELECT sum(cityHash64(j)), sum(j.p) FROM t_cache_sharing; + +DROP TABLE t_cache_sharing; + +-- Sparse elements are a separate substream, which the key must keep distinct from the column itself. +DROP TABLE IF EXISTS t_cache_sparse; + +CREATE TABLE t_cache_sparse (i UInt64, s String, arr Array(UInt64), t Tuple(a UInt64, b String)) +ENGINE = MergeTree ORDER BY i +SETTINGS min_bytes_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 0.1, index_granularity = 3; + +INSERT INTO t_cache_sparse SELECT number, if(number % 10 = 0, 'x', ''), if(number % 10 = 0, [number], []), (if(number % 10 = 0, number, 0), '') FROM numbers(20); + +SELECT sum(serialization_kind = 'Sparse') > 0 FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_cache_sparse' AND active; +SELECT sum(cityHash64(s)), sum(s.size), sum(cityHash64(arr)), sum(arr.size0), sum(cityHash64(t)), sum(t.a) FROM t_cache_sparse; + +DROP TABLE t_cache_sparse; + +-- StorageLog and StorageTinyLog use the same cache. +DROP TABLE IF EXISTS t_cache_log; +DROP TABLE IF EXISTS t_cache_tiny_log; + +CREATE TABLE t_cache_log (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) ENGINE = Log; +CREATE TABLE t_cache_tiny_log (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) ENGINE = TinyLog; +INSERT INTO t_cache_log VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)); +INSERT INTO t_cache_tiny_log VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)); +SELECT c, d FROM t_cache_log ORDER BY c; +SELECT c, d FROM t_cache_tiny_log ORDER BY c; + +DROP TABLE t_cache_log; +DROP TABLE t_cache_tiny_log; From 2b6bc6a09cfe26a12344d208366e2a95efdabcba Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 16 Sep 2026 07:45:58 +0000 Subject: [PATCH 002/296] Backport #119977 to 26.8: Do not enable Iceberg compaction settings in `enableAllExperimentalSettings` --- ci/jobs/scripts/check_style/experimental_settings_ignore.txt | 2 ++ src/Databases/enableAllExperimentalSettings.cpp | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt index 8cd361e36c138..4c4a0ec48a103 100644 --- a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt +++ b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt @@ -87,3 +87,5 @@ enable_vector_similarity_index allow_nullable_tuple_in_extracted_subcolumns allow_lossy_numeric_supertype allow_metadata_only_named_tuple_alter +allow_experimental_cleanup_old_data_files_compaction +allow_experimental_iceberg_compaction diff --git a/src/Databases/enableAllExperimentalSettings.cpp b/src/Databases/enableAllExperimentalSettings.cpp index 6b542bf1e641d..931012f3710d3 100644 --- a/src/Databases/enableAllExperimentalSettings.cpp +++ b/src/Databases/enableAllExperimentalSettings.cpp @@ -62,8 +62,6 @@ void enableAllExperimentalSettings(ContextMutablePtr context) context->setSetting("allow_experimental_time_series_aggregate_functions", 1); context->setSetting("allow_experimental_lightweight_update", 1); context->setSetting("allow_insert_into_iceberg", 1); - context->setSetting("allow_experimental_iceberg_compaction", 1); - context->setSetting("allow_experimental_cleanup_old_data_files_compaction", 1); context->setSetting("allow_iceberg_remove_orphan_files", 1); context->setSetting("allow_experimental_expire_snapshots", 1); context->setSetting("allow_experimental_delta_lake_writes", 1); From 21b00f2f6e793738193e02c000376a5a7e1010b7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 16 Sep 2026 10:11:45 +0000 Subject: [PATCH 003/296] Backport #120025 to 26.8: Fix unparseable metadata for a NATS table defined only by a named collection --- src/Storages/NATS/StorageNATS.cpp | 3 ++- .../04891_nats_credential_file_path_restriction.sql | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index 43913a8fc5f48..5e2f139a9c764 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -1196,7 +1196,8 @@ void registerStorageNATS(StorageFactory & factory) else if (!args.storage_def->settings) throw Exception(ErrorCodes::BAD_ARGUMENTS, "NATS engine must have settings"); - nats_settings->loadFromQuery(*args.storage_def); + if (args.storage_def->settings) + nats_settings->loadFromQuery(*args.storage_def); /// A credential source assigned in the `SETTINGS` clause is query-level even when the named /// collection provides the same key: the clause is applied on top of the collection values, diff --git a/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql b/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql index bebc4d7ac63d0..6c6549b72bde1 100644 --- a/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql +++ b/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql @@ -75,12 +75,10 @@ CREATE NAMED COLLECTION 04891_nats_existing_sql_collection AS nats_startup_connect_tries = 0, nats_reconnect_wait = 1; -- A full-definition `ATTACH` is validated exactly like `CREATE`, and unlike `CREATE` it tolerates -- a failing connection attempt, so the table exists afterwards and its metadata can be replayed. --- The `SETTINGS` clause carries an unrelated key on purpose: an engine definition whose settings --- are all inherited from the named collection is stored with an empty `SETTINGS` clause, which the --- metadata reload then fails to parse. +-- All of its settings come from the named collection, so the stored definition must carry no +-- `SETTINGS` clause at all. ATTACH TABLE nats_file_from_existing_sql_collection UUID 'c6d2423a-9ab2-4a37-8e56-10e479541002' (key UInt64) -ENGINE = NATS(04891_nats_existing_sql_collection) -SETTINGS nats_num_consumers = 1; +ENGINE = NATS(04891_nats_existing_sql_collection); DETACH TABLE nats_file_from_existing_sql_collection; ALTER NAMED COLLECTION 04891_nats_existing_sql_collection SET nats_credential_file = '/etc/passwd'; ATTACH TABLE nats_file_from_existing_sql_collection; -- { serverError BAD_ARGUMENTS } From c694da511190e0da58f5df7d90e217c3f51b7a2a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 16 Sep 2026 11:36:12 +0000 Subject: [PATCH 004/296] Backport #120239 to 26.8: Mask the base-url password of a URL database engine --- src/Databases/DatabaseURL.cpp | 3 ++- src/Parsers/FunctionSecretArgumentsFinder.cpp | 5 +++++ tests/integration/test_mask_sensitive_info/test.py | 2 ++ .../0_stateless/04627_url_database_engine.reference | 9 ++++++++- .../0_stateless/04627_url_database_engine.sh | 13 ++++++++++++- 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/Databases/DatabaseURL.cpp b/src/Databases/DatabaseURL.cpp index e48390e7254e5..b08e1332830f6 100644 --- a/src/Databases/DatabaseURL.cpp +++ b/src/Databases/DatabaseURL.cpp @@ -264,9 +264,10 @@ class StorageURLDatabaseTable final : public StorageProxy DatabaseURL::DatabaseURL(const String & name_, const String & base_url_, ContextPtr context_) : IDatabase(name_), WithContext(context_->getGlobalContext()), base_url(base_url_) { + /// Not echoed back: password masking anchors on the `://` this value lacks, so it would log the password. if (!base_url.empty() && !hasURLScheme(base_url)) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The base URL of a URL database must contain a scheme (e.g. https://), got: {}", base_url); + "The base URL of a URL database must contain a scheme (e.g. https://)"); } String DatabaseURL::getTableURL(const String & name) const diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 1431838817b56..6215a2933f436 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -1123,6 +1123,11 @@ void FunctionSecretArgumentsFinder::findDatabaseEngineSecretArguments() { findBackupDatabaseSecretArguments(); } + else if (engine_name == "URL") + { + /// URL('base_url') + findURLSecretArguments(); + } } void FunctionSecretArgumentsFinder::findMySQLDatabaseSecretArguments() diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index a9ffa2e37f80e..d16d8f5623f74 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -444,6 +444,7 @@ def test_create_database(): f"Backup('', S3('http://minio1:9001/root/data/backup', 'minio', '{password}'))", "DNS_ERROR", ), + f"URL('https://username:{password}@localhost:11111/x/')", ] def make_test_case(i): @@ -472,6 +473,7 @@ def make_test_case(i): "CREATE DATABASE database3 ENGINE = S3(named_collection_2, secret_access_key = '[HIDDEN]', access_key_id = 'minio')", # "CREATE DATABASE database4 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')", "CREATE DATABASE database4 ENGINE = Backup('', S3('http://minio1:9001/root/data/backup', 'minio', '[HIDDEN]'))", + "CREATE DATABASE database5 ENGINE = URL('https://username:[HIDDEN]@localhost:11111/x/')", ], must_not_contain=[password], ) diff --git a/tests/queries/0_stateless/04627_url_database_engine.reference b/tests/queries/0_stateless/04627_url_database_engine.reference index 5be03016e8065..9b1a3484096ab 100644 --- a/tests/queries/0_stateless/04627_url_database_engine.reference +++ b/tests/queries/0_stateless/04627_url_database_engine.reference @@ -23,5 +23,12 @@ ENGINE = URL('http://server/') --- URL database engine on the server Ok. Ok. ---- the base URL must contain a scheme +--- the base URL must contain a scheme, and the rejected value is not echoed back (last line counts the leaks) must contain a scheme +must contain a scheme +0 +--- the base-url password is masked in the display surfaces (the last line counts the leaks) +URL(\'https://user:[HIDDEN]@localhost:11111/x/\') +CREATE DATABASE secret_db +ENGINE = URL('https://user:[HIDDEN]@localhost:11111/x/') +0 diff --git a/tests/queries/0_stateless/04627_url_database_engine.sh b/tests/queries/0_stateless/04627_url_database_engine.sh index baaf4615ed99c..6c2a3cb403510 100755 --- a/tests/queries/0_stateless/04627_url_database_engine.sh +++ b/tests/queries/0_stateless/04627_url_database_engine.sh @@ -54,5 +54,16 @@ ${CLICKHOUSE_CLIENT} -q "SELECT * FROM ${WEB_DB}.\`ping\`" ${CLICKHOUSE_CLIENT} -q "SELECT * FROM ${WEB_DB}.\`${SERVER_URL}/ping\`" ${CLICKHOUSE_CLIENT} -q "DROP DATABASE ${WEB_DB}" -echo '--- the base URL must contain a scheme' +echo '--- the base URL must contain a scheme, and the rejected value is not echoed back (last line counts the leaks)' ${CLICKHOUSE_LOCAL} -q "CREATE DATABASE bad ENGINE = URL('localhost/dir/')" 2>&1 | grep -oF 'must contain a scheme' | head -1 +${CLICKHOUSE_LOCAL} -q "CREATE DATABASE bad ENGINE = URL('user:SEKRIT_PW@localhost/dir/')" 2>&1 | grep -oF 'must contain a scheme' | head -1 +${CLICKHOUSE_LOCAL} -q "CREATE DATABASE bad ENGINE = URL('user:SEKRIT_PW@localhost/dir/')" 2>&1 | grep -c SEKRIT_PW + +echo '--- the base-url password is masked in the display surfaces (the last line counts the leaks)' +SECRET_DB="${CLICKHOUSE_DATABASE}_04627_secret" +${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS ${SECRET_DB}" +${CLICKHOUSE_CLIENT} -q "CREATE DATABASE ${SECRET_DB} ENGINE = URL('https://user:SEKRIT_PW@localhost:11111/x/')" +${CLICKHOUSE_CLIENT} -q "SELECT engine_full FROM system.databases WHERE name = '${SECRET_DB}'" +${CLICKHOUSE_CLIENT} -q "SHOW CREATE DATABASE ${SECRET_DB} FORMAT TabSeparatedRaw" | sed "s|${SECRET_DB}|secret_db|" +${CLICKHOUSE_CLIENT} -q "SHOW CREATE DATABASE ${SECRET_DB}" | grep -c SEKRIT_PW +${CLICKHOUSE_CLIENT} -q "DROP DATABASE ${SECRET_DB}" From de7e84061a45a172c275d54dfb1f25cb76ed1894 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 16 Sep 2026 13:06:57 +0000 Subject: [PATCH 005/296] Backport #119001 to 26.8: Restore selected right join key from left under join_use_nulls --- src/Processors/QueryPlan/JoinStepLogical.cpp | 73 +++++- .../03357_join_pk_sharding.reference | 4 +- ...03402_concurrent_right_full_join.reference | 4 +- ...oin_use_nulls_right_key_restored.reference | 224 ++++++++++++++++++ ...5137_join_use_nulls_right_key_restored.sql | 99 ++++++++ 5 files changed, 399 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.reference create mode 100644 tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.sql diff --git a/src/Processors/QueryPlan/JoinStepLogical.cpp b/src/Processors/QueryPlan/JoinStepLogical.cpp index 9ca69c92f9acd..88dc3f137ea74 100644 --- a/src/Processors/QueryPlan/JoinStepLogical.cpp +++ b/src/Processors/QueryPlan/JoinStepLogical.cpp @@ -809,6 +809,45 @@ static void predicateOperandsToCommonType( } } +/// Under `join_use_nulls`, a right column selected from a LEFT or FULL JOIN is output through `toNullable(x)` +/// (see `addToNullableIfNeeded`). When that column is also a join key, joining on the `Nullable` node makes +/// it the single right column that is both the key and the output, which the join restores from the left +/// key. Joining on the plain input instead leaves the `Nullable` wrapper as a payload column next to the +/// key: a whole extra column where the join keeps keys only in its arena, and a second count of the same +/// bytes toward the spill threshold where it saves the key columns too. +static void preferNullableRightKey( + JoinActionRef & right_node, + const JoinPlanningContext & planning_context, + std::vector> & shared_runtime_filter_descriptors) +{ + /// The `Join` engine and a dictionary are looked up by the key they declare. + if (planning_context.is_storage_join) + return; + + const auto * input = right_node.getNode(); + if (input->type != ActionsDAG::ActionType::INPUT) + return; + + auto it = planning_context.actions_after_join_map.find(input->result_name); + if (it == planning_context.actions_after_join_map.end()) + return; + + const auto * to_nullable = it->second; + if (to_nullable->type != ActionsDAG::ActionType::FUNCTION || to_nullable->children.size() != 1 + || to_nullable->children.front() != input || to_nullable->function_base->getName() != "toNullable") + return; + + /// The build-side key name is the rendezvous with the shared runtime filter descriptors, as in + /// `predicateOperandsToCommonType`. + String name_before = right_node.getColumnName(); + right_node = JoinActionRef::transform({right_node}, [to_nullable](auto &, auto &&) { return to_nullable; }); + for (auto & descriptor : shared_runtime_filter_descriptors) + { + if (descriptor.second == name_before) + descriptor.second = right_node.getColumnName(); + } +} + static bool addJoinPredicatesToTableJoin(std::vector & predicates, TableJoin::JoinOnClause & table_join_clause, std::vector & used_expressions, const JoinSettings & join_settings, const JoinPlanningContext & planning_context, std::vector> & shared_runtime_filter_descriptors) @@ -830,6 +869,8 @@ static bool addJoinPredicatesToTableJoin(std::vector & predicates predicateOperandsToCommonType(lhs, rhs, join_settings, planning_context, shared_runtime_filter_descriptors); bool null_safe_comparison = JoinConditionOperator::NullSafeEquals == predicate_op; + if (!null_safe_comparison) + preferNullableRightKey(rhs, planning_context, shared_runtime_filter_descriptors); if (null_safe_comparison && isNullableOrLowCardinalityNullable(lhs.getType()) && isNullableOrLowCardinalityNullable(rhs.getType())) { /** @@ -1656,6 +1697,34 @@ static QueryPlanNode buildPhysicalJoinImpl( for (const auto * node : dag_inputs) name_to_nodes[node->result_name].push_back(node); + /// An input that only feeds a used expression, such as the `toNullable(x)` key under `join_use_nulls` + /// or a key cast to a common type, is not passed to the join as a column of its own: the join would + /// keep it as payload for nothing. `ActionsDAG::updateHeader` drops such consumed inputs anyway. + std::unordered_set consumed_inputs; + { + std::unordered_set used_nodes; + for (const auto & expression : used_expressions) + used_nodes.insert(expression.getNode()); + + std::stack stack; + for (const auto * node : used_nodes) + for (const auto * child : node->children) + stack.push(child); + while (!stack.empty()) + { + const auto * node = stack.top(); + stack.pop(); + if (node->type == ActionsDAG::ActionType::INPUT) + { + if (!used_nodes.contains(node)) + consumed_inputs.insert(node); + continue; + } + for (const auto * child : node->children) + stack.push(child); + } + } + for (const auto * child : children) { for (const auto & column : *child->step->getOutputHeader()) @@ -1669,8 +1738,10 @@ static QueryPlanNode buildPhysicalJoinImpl( fmt::join(children | std::views::transform([](const auto & c) { return fmt::format("[{}]", c->step->getOutputHeader()->dumpNames()); }), ", "), expression_actions.getActionsDAG()->dumpDAG()); - used_expressions.emplace_back(input_it->second.front(), expression_actions); + const auto * input = input_it->second.front(); input_it->second.pop_front(); + if (!consumed_inputs.contains(input)) + used_expressions.emplace_back(input, expression_actions); } } diff --git a/tests/queries/0_stateless/03357_join_pk_sharding.reference b/tests/queries/0_stateless/03357_join_pk_sharding.reference index 110b13f33c486..a411a626b93d1 100644 --- a/tests/queries/0_stateless/03357_join_pk_sharding.reference +++ b/tests/queries/0_stateless/03357_join_pk_sharding.reference @@ -210,13 +210,13 @@ Expression ((Project names + Projection)) Expression (Post Join Actions) Join (JOIN FillRightFirst) Algorithm: ConcurrentHashJoin - Clauses: [(multiply(__table1.a, 2_UInt8), plus(__table1.b, __table1.c), __table1.d) = (multiply(__table3.a, 2_UInt8), multiply(__table3.c, 2_UInt8), __table3.d)] + Clauses: [(multiply(__table1.a, 2_UInt8), plus(__table1.b, __table1.c), __table1.d) = (multiply(__table3.a, 2_UInt8), multiply(__table3.c, 2_UInt8), toNullable(__table3.d))] Sharding: [(multiply(a, 2) = multiply(a, 2)), (plus(b, c) = multiply(c, 2))] Expression (Left Pre Join Actions) Expression (Post Join Actions) Join (JOIN FillRightFirst) Algorithm: ConcurrentHashJoin - Clauses: [(multiply(__table1.a, 2_UInt8), __table1.d, plus(__table1.b, __table1.c)) = (plus(__table2.c, __table2.d), __table2.a, multiply(__table2.b, 2_UInt8))] + Clauses: [(multiply(__table1.a, 2_UInt8), __table1.d, plus(__table1.b, __table1.c)) = (plus(__table2.c, __table2.d), toNullable(__table2.a), multiply(__table2.b, 2_UInt8))] Sharding: [(multiply(a, 2) = plus(c, d)), (plus(b, c) = multiply(b, 2))] Expression (Left Pre Join Actions) Expression (Change column names to column identifiers) diff --git a/tests/queries/0_stateless/03402_concurrent_right_full_join.reference b/tests/queries/0_stateless/03402_concurrent_right_full_join.reference index 302682474cfc0..866ff4078b735 100644 --- a/tests/queries/0_stateless/03402_concurrent_right_full_join.reference +++ b/tests/queries/0_stateless/03402_concurrent_right_full_join.reference @@ -38,8 +38,8 @@ Positions: 4 0 1 Actions: INPUT : 0 -> __table1.id UInt32 : 0 INPUT : 1 -> __table1.value String : 1 FUNCTION toNullable(__table1.id : 0) -> toNullable(__table1.id) Nullable(UInt32) : 2 - FUNCTION toNullable(__table1.value : 1) -> toNullable(__table1.value) Nullable(String) : 3 - Positions: 0 2 3 1 + FUNCTION toNullable(__table1.value :: 1) -> toNullable(__table1.value) Nullable(String) : 3 + Positions: 0 2 3 Expression (Change column names to column identifiers) Actions: INPUT : 0 -> id UInt32 : 0 INPUT : 1 -> value String : 1 diff --git a/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.reference b/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.reference new file mode 100644 index 0000000000000..13c43b066be50 --- /dev/null +++ b/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.reference @@ -0,0 +1,224 @@ +-- right side of the plan carries only the Nullable key +Header: __table2.k Nullable(String) +Header: __table2.k Nullable(String) +__table2.k Nullable(String) +toNullable(__table2.k) Nullable(String) +Expression (Right Pre Join Actions) +Header: toNullable(__table2.k) Nullable(String) +Header: __table2.k String +-- LEFT +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +Nullable(String) 6 +-- LEFT, key only +a +c +c +c +c +\N +-- LEFT ANY +a a +b \N +c c +c c +-- LEFT SEMI +a a +c c +c c +-- LEFT ANTI +b \N +-- FULL +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +-- LEFT with residual ON condition +a \N \N +b \N \N +c c 31 +c c 31 +-- LEFT with a residual ON condition on the right key itself +a \N \N +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +-- LEFT with WHERE on the right key +b \N +c c +c c +c c +c c +-- null-safe key +a a +b \N +c c +c c +c c +c c +-- parallel_hash +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +-- grace_hash +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +-- spilling wrapper +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +-- partial_merge +Header: __table2.k Nullable(String) +Header: __table2.k Nullable(String) +__table2.k Nullable(String) +toNullable(__table2.k) Nullable(String) +Expression (Right Pre Join Actions) +Header: toNullable(__table2.k) Nullable(String) +Header: __table2.k String +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +a a +b \N +c c +c c +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +a a +b \N +c c +c c +c c +c c +LowCardinality(Nullable(String)) +-- auto, switched to partial_merge +Header: __table2.k Nullable(String) +Header: __table2.k Nullable(String) +__table2.k Nullable(String) +toNullable(__table2.k) Nullable(String) +Expression (Right Pre Join Actions) +Header: toNullable(__table2.k) Nullable(String) +Header: __table2.k String +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +a a +b \N +c c +c c +c c +c c +-- full_sorting_merge +Header: __table2.k Nullable(String) +Header: __table2.k Nullable(String) +__table2.k Nullable(String) +toNullable(__table2.k) Nullable(String) +Header: toNullable(__table2.k) Nullable(String) +Expression (Right Pre Join Actions) +Header: toNullable(__table2.k) Nullable(String) +Header: __table2.k String +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +-- LowCardinality key +a a +b \N +c c +c c +c c +c c +LowCardinality(Nullable(String)) +-- key with a type conversion +1 10 +2 \N +3 30 +4 40 +1 10 +2 \N +3 30 +4 40 +-- derived right key +a a +b \N +c c +c c +c c +c c +-- WITH TOTALS +a 1 +c 14 +\N 2 + +\N 17 +a 1 10 +b 2 \N +c 7 61 +d \N 40 + +\N 10 111 +a a +b \N +c c + + +-- swapped +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 diff --git a/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.sql b/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.sql new file mode 100644 index 0000000000000..e4a19097ef0ae --- /dev/null +++ b/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.sql @@ -0,0 +1,99 @@ +-- Tags: no-parallel-replicas +-- The EXPLAIN queries filter plan headers by column name, and parallel replicas add extra plan steps that print the same header. + +-- https://github.com/ClickHouse/ClickHouse/issues/118738 +-- With join_use_nulls, a selected right join key is joined on as its Nullable output column, so the right +-- side carries one column that the join restores from the left key, instead of the plain key plus the +-- Nullable wrapper as a payload column. + +SET join_use_nulls = 1; +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS l; +DROP TABLE IF EXISTS r; +CREATE TABLE l (k String, v UInt32) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE r (k String, w UInt32) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO l VALUES ('a', 1), ('b', 2), ('c', 3), ('c', 4); +INSERT INTO r VALUES ('a', 10), ('c', 30), ('c', 31), ('d', 40); + +SELECT '-- right side of the plan carries only the Nullable key'; +SELECT trimLeft(explain) FROM ( + EXPLAIN header = 1 + SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k + SETTINGS explain_query_plan_default = 'legacy', query_plan_join_swap_table = 0, join_algorithm = 'hash' +) WHERE explain LIKE '%Right Pre Join Actions%' OR explain LIKE '%__table2.k%'; + +SELECT '-- LEFT'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT toTypeName(r.k), count() FROM l LEFT JOIN r ON l.k = r.k GROUP BY 1 SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT, key only'; +SELECT r.k FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT ANY'; +SELECT l.k, r.k FROM l LEFT ANY JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT SEMI'; +SELECT l.k, r.k FROM l LEFT SEMI JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT ANTI'; +SELECT l.k, r.k FROM l LEFT ANTI JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- FULL'; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT with residual ON condition'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k AND r.w > 30 ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT with a residual ON condition on the right key itself'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k AND r.k != 'a' ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT with WHERE on the right key'; +SELECT l.k, r.k FROM l LEFT JOIN r ON l.k = r.k WHERE r.k IS NULL ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT l.k, r.k FROM l LEFT JOIN r ON l.k = r.k WHERE r.k = 'c' ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- null-safe key'; +SELECT l.k, r.k FROM l LEFT JOIN r ON l.k IS NOT DISTINCT FROM r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- parallel_hash'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'parallel_hash'; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'parallel_hash'; +SELECT '-- grace_hash'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'grace_hash', grace_hash_join_initial_buckets = 4; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'grace_hash', grace_hash_join_initial_buckets = 4; +SELECT '-- spilling wrapper'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash', max_bytes_before_external_join = 100000000; +SELECT '-- partial_merge'; +SELECT trimLeft(explain) FROM ( + EXPLAIN header = 1 + SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k + SETTINGS explain_query_plan_default = 'legacy', query_plan_join_swap_table = 0, join_algorithm = 'partial_merge' +) WHERE explain LIKE '%Right Pre Join Actions%' OR explain LIKE '%__table2.k%'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'partial_merge'; +SELECT l.k, r.k FROM l LEFT ANY JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'partial_merge'; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'partial_merge'; +SELECT l.k, r.k FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'partial_merge'; +SELECT toTypeName(r.k) FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k LIMIT 1 SETTINGS join_algorithm = 'partial_merge'; +SELECT '-- auto, switched to partial_merge'; +SELECT trimLeft(explain) FROM ( + EXPLAIN header = 1 + SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k + SETTINGS explain_query_plan_default = 'legacy', query_plan_join_swap_table = 0, join_algorithm = 'auto' +) WHERE explain LIKE '%Right Pre Join Actions%' OR explain LIKE '%__table2.k%'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'auto', max_rows_in_join = 2, join_overflow_mode = 'break'; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'auto', max_rows_in_join = 2, join_overflow_mode = 'break'; +SELECT l.k, r.k FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'auto', max_rows_in_join = 2, join_overflow_mode = 'break'; +SELECT '-- full_sorting_merge'; +SELECT trimLeft(explain) FROM ( + EXPLAIN header = 1 + SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k + SETTINGS explain_query_plan_default = 'legacy', query_plan_join_swap_table = 0, join_algorithm = 'full_sorting_merge' +) WHERE explain LIKE '%Right Pre Join Actions%' OR explain LIKE '%__table2.k%'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'full_sorting_merge'; +SELECT '-- LowCardinality key'; +SELECT l.k, r.k FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT toTypeName(r.k) FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k LIMIT 1 SETTINGS join_algorithm = 'hash'; +SELECT '-- key with a type conversion'; +SELECT l.v, r.w FROM l LEFT JOIN r ON l.v * 10 = r.w ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT l.v, r.w FROM l LEFT JOIN r ON toInt64(l.v) * 10 = r.w ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- derived right key'; +SELECT l.k, r.k FROM l LEFT JOIN r ON l.k = lower(upper(r.k)) ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- WITH TOTALS'; +SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k WITH TOTALS ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT k, sum(v), sum(w) FROM (SELECT k, sum(v) AS v FROM l GROUP BY k WITH TOTALS) AS l FULL JOIN (SELECT k, sum(w) AS w FROM r GROUP BY k WITH TOTALS) AS r USING (k) GROUP BY k WITH TOTALS ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT l.k, r.k FROM (SELECT k FROM l GROUP BY k WITH TOTALS) AS l LEFT JOIN (SELECT k FROM r GROUP BY k WITH TOTALS) AS r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- swapped'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash', query_plan_join_swap_table = 1; + +DROP TABLE l; +DROP TABLE r; From 846c8f283dc492d3235450ae108e6f51d4af229a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 16 Sep 2026 13:30:11 +0000 Subject: [PATCH 006/296] Backport #119868 to 26.8: Add `validate_group_by_all_key_types` to gate the `GROUP BY ALL` key type validation --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 30 +++++++++++++++---- src/Analyzer/Resolve/QueryAnalyzer.h | 2 +- src/Core/Settings.cpp | 8 +++++ src/Core/SettingsChangesHistory.cpp | 1 + ...02_group_by_all_suspicious_types.reference | 6 ++++ .../04602_group_by_all_suspicious_types.sql | 28 +++++++++++++++++ 6 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 4d11a1aa8156e..25e58030e9e3b 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -108,6 +108,7 @@ namespace Setting extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions; extern const SettingsBool allow_suspicious_types_in_group_by; extern const SettingsBool allow_suspicious_types_in_order_by; + extern const SettingsBool validate_group_by_all_key_types; extern const SettingsBool allow_experimental_correlated_subqueries; extern const SettingsString implicit_table_at_top_level; extern const SettingsBool parallel_replicas_for_cluster_engines; @@ -4076,7 +4077,7 @@ void registerNullableGroupByKeys(const QueryTreeNodes & group_by_keys, Identifie /** Resolve GROUP BY clause. */ -void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope) +void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope, bool validate_key_types) { QueryTreeNodes nullable_group_by_keys; @@ -4098,7 +4099,10 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR { for (const auto & group_by_elem : grouping_set->as()->getNodes()) { - validateGroupByKeyType(group_by_elem->getResultType(), scope); + if (validate_key_types) + validateGroupByKeyType(group_by_elem->getResultType(), scope); + /// Outside the guard: the promotion to Nullable is what `group_by_use_nulls` asks for, + /// independently of whether the key types are validated. if (scope.group_by_use_nulls) nullable_group_by_keys.push_back(group_by_elem); } @@ -4117,7 +4121,9 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) { - validateGroupByKeyType(group_by_elem->getResultType(), scope); + if (validate_key_types) + validateGroupByKeyType(group_by_elem->getResultType(), scope); + /// Outside the guard, for the same reason as in the grouping-sets branch above. if (scope.group_by_use_nulls) nullable_group_by_keys.push_back(group_by_elem); } @@ -6527,6 +6533,10 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier NamesAndTypes projection_columns; + /// `expandGroupByAll` clears the flag, and under `group_by_use_nulls` it runs before the grouping keys + /// are resolved, so the ALL-ness has to be remembered here to still be known at either validation site. + const bool query_is_group_by_all = query_node_typed.isGroupByAll(); + if (!scope.group_by_use_nulls) { projection_columns = resolveProjectionExpressionNodeList(query_node_typed.getProjectionNode(), scope); @@ -6582,7 +6592,11 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier resolveExpressionNode(query_node_typed.getWhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); if (query_node_typed.hasGroupBy()) - resolveGroupByNode(query_node_typed, scope); + resolveGroupByNode( + query_node_typed, + scope, + /* validate_key_types */ !query_is_group_by_all + || scope.context->getSettingsRef()[Setting::validate_group_by_all_key_types]); if (query_node_typed.hasHaving()) resolveExpressionNode(query_node_typed.getHaving(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -6742,8 +6756,12 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier { expandTuplesInList(query_node_typed.getGroupBy().getNodes()); - for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) - validateGroupByKeyType(group_by_elem->getResultType(), scope); + /// Only the acceptance check is optional; the tuple expansion above is not. + if (scope.context->getSettingsRef()[Setting::validate_group_by_all_key_types]) + { + for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) + validateGroupByKeyType(group_by_elem->getResultType(), scope); + } } tryMoveNonAggregateHavingPredicatesToWhere(query_node, scope); diff --git a/src/Analyzer/Resolve/QueryAnalyzer.h b/src/Analyzer/Resolve/QueryAnalyzer.h index d3f1db45ca8ec..129c38311cd72 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.h +++ b/src/Analyzer/Resolve/QueryAnalyzer.h @@ -270,7 +270,7 @@ class QueryAnalyzer void validateSortingKeyType(const DataTypePtr & sorting_key_type, const IdentifierResolveScope & scope) const; - void resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope); + void resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope, bool validate_key_types); void validateGroupByKeyType(const DataTypePtr & group_by_key_type, const IdentifierResolveScope & scope) const; diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 75a3e21b1aee6..064ae514eb49b 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -1210,6 +1210,14 @@ Allows or restricts using [Variant](/reference/data-types/variant) and [Dynamic] )", 0) \ DECLARE(Bool, allow_suspicious_types_in_order_by, false, R"( Allows or restricts using [Variant](/reference/data-types/variant) and [Dynamic](/reference/data-types/dynamic) types in ORDER BY keys. +)", 0) \ + DECLARE(Bool, validate_group_by_all_key_types, true, R"( +Controls whether the grouping keys that `GROUP BY ALL` expands the `SELECT` expressions into are checked against [allow_suspicious_types_in_group_by](#allow_suspicious_types_in_group_by). Disable it to restore the behavior of versions before 26.7, which accepted a [Variant](/reference/data-types/variant) or [Dynamic](/reference/data-types/dynamic) key written as `GROUP BY ALL`, for example a grouping key that is an untyped JSON subpath. Takes effect only when the analyzer is enabled (`enable_analyzer = 1`, the default); with the old analyzer such a key was rejected before 26.7 as well, so there is nothing to restore there. An explicit `GROUP BY` is unaffected and keeps rejecting such a key either way, so this is narrower than setting `allow_suspicious_types_in_group_by`, which also permits them in an explicit `GROUP BY`. + +Possible values: + +- 0 - The key types `GROUP BY ALL` expands into are not validated. +- 1 - They are validated, as an explicit `GROUP BY` validates its own keys. )", 0) \ DECLARE(Bool, use_variant_default_implementation_for_comparisons, true, R"( Enables or disables default implementation for Variant type in comparison functions. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 910d9b7efe4e5..18e6042c5617e 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -155,6 +155,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.7", { + {"validate_group_by_all_key_types", false, true, "New setting gating the validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into. 26.7 started rejecting a `Variant`/`Dynamic` key there, which earlier versions accepted with the analyzer enabled (`enable_analyzer = 1`, the default; the old analyzer rejected such a key before 26.7 as well), so the previous value is `false` and `compatibility` with a version before 26.7 restores the earlier acceptance."}, {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, {"query_plan_optimize_lazy_materialization_for_object_storage", false, true, "New setting to use lazy materialization for `ORDER BY ... LIMIT n` queries reading Parquet files from object storage (including Iceberg tables)."}, {"iceberg_compaction_max_rows_in_data_file", std::numeric_limits::max(), std::numeric_limits::max(), "New setting for the max rows of an iceberg data file produced by compaction, separate from the insert-time limit."}, diff --git a/tests/queries/0_stateless/04602_group_by_all_suspicious_types.reference b/tests/queries/0_stateless/04602_group_by_all_suspicious_types.reference index e0a24a12d8546..cb962acd614c7 100644 --- a/tests/queries/0_stateless/04602_group_by_all_suspicious_types.reference +++ b/tests/queries/0_stateless/04602_group_by_all_suspicious_types.reference @@ -1,2 +1,8 @@ 1 ('1',1) +1 +(1,2) +1 +\N +1 +v0 diff --git a/tests/queries/0_stateless/04602_group_by_all_suspicious_types.sql b/tests/queries/0_stateless/04602_group_by_all_suspicious_types.sql index f2e23cb156ae2..d622b03c603c3 100644 --- a/tests/queries/0_stateless/04602_group_by_all_suspicious_types.sql +++ b/tests/queries/0_stateless/04602_group_by_all_suspicious_types.sql @@ -16,3 +16,31 @@ SELECT tuple(d, 1) FROM (SELECT 1::Dynamic AS d) GROUP BY tuple(d, 1) SETTINGS a -- Allowed once the setting permits suspicious types. SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 1; SELECT tuple(d, 1) FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 1; + +-- `validate_group_by_all_key_types` gates the key types that `GROUP BY ALL` expands into, and only +-- those: with it off `GROUP BY ALL` accepts a suspicious key type again while an explicit `GROUP BY` +-- keeps rejecting it, which is what distinguishes this setting from `allow_suspicious_types_in_group_by`. +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY d SETTINGS allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; -- { serverError ILLEGAL_COLUMN } + +-- The tuple expansion is not gated, so a tuple grouping key is still unwrapped into its elements with +-- the validation off and an `ORDER BY` of the same tuple still finds them in the aggregated block. +SELECT tuple(c0, c1) AS t FROM (SELECT 1 c0, 2 c1) v0 GROUP BY ALL ORDER BY t + SETTINGS validate_group_by_all_key_types = 0, optimize_injective_functions_in_group_by = 0; + +-- Under `group_by_use_nulls` with a `WITH ROLLUP` modifier the `GROUP BY ALL` keys are expanded before +-- they are resolved, so they are validated on the same path as an explicit `GROUP BY`. The setting +-- reaches that path too, while an explicit `GROUP BY` there stays unconditional. The second row of the +-- accepted arm is the rollup total, whose key is NULL because `group_by_use_nulls` promoted it. +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL WITH ROLLUP SETTINGS group_by_use_nulls = 1, allow_suspicious_types_in_group_by = 0; -- { serverError ILLEGAL_COLUMN } +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL WITH ROLLUP SETTINGS group_by_use_nulls = 1, allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY d WITH ROLLUP SETTINGS group_by_use_nulls = 1, allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; -- { serverError ILLEGAL_COLUMN } + +-- `compatibility` with a version before 26.7 restores the earlier acceptance; with 26.7 itself it does +-- not, because 26.7 is the version that started rejecting such a key. +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 0, compatibility = '26.6'; +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 0, compatibility = '26.7'; -- { serverError ILLEGAL_COLUMN } + +-- The reported shape: an untyped JSON subpath is a `Dynamic` grouping key, so it is gated the same way. +SELECT c1.p1 FROM (SELECT '{"p1":"v0"}'::JSON AS c1) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; +SELECT c1.p1 FROM (SELECT '{"p1":"v0"}'::JSON AS c1) GROUP BY c1.p1 SETTINGS allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; -- { serverError ILLEGAL_COLUMN } From 93451b5a06dcebdc91c2b55337a0c699baa09654 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 16 Sep 2026 15:56:03 +0000 Subject: [PATCH 007/296] Backport #119873 to 26.8: Combine small blocks into one Arrow IPC record batch on output --- .../BuzzHouse/Generator/SessionSettings.cpp | 2 + src/Core/FormatFactorySettings.h | 18 ++ src/Core/SettingsChangesHistory.cpp | 2 + src/Formats/FormatFactory.cpp | 2 + src/Formats/FormatSettings.h | 2 + .../ArrowIPC/ArrowIPCBlockInputFormat.cpp | 4 + .../ArrowIPC/ArrowIPCBlockOutputFormat.cpp | 47 +++++ .../Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h | 8 + ...w_output_record_batch_coalescing.reference | 65 +++++++ ...11_arrow_output_record_batch_coalescing.sh | 160 ++++++++++++++++++ 10 files changed, 310 insertions(+) create mode 100644 tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.reference create mode 100755 tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.sh diff --git a/src/Client/BuzzHouse/Generator/SessionSettings.cpp b/src/Client/BuzzHouse/Generator/SessionSettings.cpp index 51211ceebf795..a78720dcab77f 100644 --- a/src/Client/BuzzHouse/Generator/SessionSettings.cpp +++ b/src/Client/BuzzHouse/Generator/SessionSettings.cpp @@ -2021,6 +2021,7 @@ void loadFuzzerServerSettings(const FuzzConfig & fc) "min_insert_block_size_bytes", "min_insert_block_size_bytes_for_materialized_views", "min_joined_block_size_bytes", + "output_format_arrow_record_batch_size_bytes", "output_format_parquet_row_group_size_bytes", "page_cache_block_size", "page_cache_max_coalesced_bytes", @@ -2079,6 +2080,7 @@ void loadFuzzerServerSettings(const FuzzConfig & fc) "min_table_rows_to_use_projection_index", "number_of_mutations_to_delay", "number_of_mutations_to_throw", + "output_format_arrow_record_batch_size", "output_format_parquet_data_page_size", "output_format_parquet_row_group_size", "output_format_pretty_max_rows", diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index c37da84008e30..5bc1e8766ec44 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -1501,6 +1501,24 @@ Write Date values as plain 16-bit numbers (read back as UInt16), instead of conv )", 0) \ DECLARE(Bool, output_format_arrow_unsupported_types_as_binary, true, R"( Output types having no conversion as raw binary data. If false - such types would raise UNKNOWN_TYPE exception. +)", 0) \ + DECLARE(UInt64, output_format_arrow_record_batch_size, 0, R"( +Target number of rows per record batch for the `Arrow` and `ArrowStream` output formats. Combining small blocks reduces metadata and buffer-padding overhead, particularly for queries with selective filters. + +Blocks accumulate until this target or [output_format_arrow_record_batch_size_bytes](#output_format_arrow_record_batch_size_bytes) is reached. A block that already meets the row or byte target is written separately, without splitting. If you set a row target, combined batches contain fewer than twice that many rows, but a single input block can be larger. + +Buffering blocks can increase memory use and delay output. If the result never reaches either target, `ArrowStream` writes the first record batch only when the query finishes, though it can write the schema earlier. Leave both targets at `0` to write record batches as blocks arrive. + +`0` (the default) disables the row target. Try `65409` as a starting value. +)", 0) \ + DECLARE(UInt64, output_format_arrow_record_batch_size_bytes, 0, R"( +Target record batch size for the `Arrow` and `ArrowStream` output formats, measured in bytes of accumulated block data. This uses the same measure as [min_insert_block_size_bytes](/reference/settings/settings#min_insert_block_size_bytes). A batch is written when either this target or [output_format_arrow_record_batch_size](#output_format_arrow_record_batch_size) is reached. + +Note that `LowCardinality` columns can produce Arrow batches much larger or smaller than this byte target. Repeated values expand in the output unless [output_format_arrow_low_cardinality_as_dictionary](#output_format_arrow_low_cardinality_as_dictionary) is enabled. Filtered blocks can also retain large dictionaries, so even a block with very few rows can reach the target and be written separately. For these columns, use [output_format_arrow_record_batch_size](#output_format_arrow_record_batch_size) to control the row count and set the byte target to `0`. + +Buffering blocks can increase memory use and delay the first record batch until the query finishes. Leave both targets at `0` to write record batches as blocks arrive. + +`0` (the default) disables the byte target. Try `1048576` (1 MiB) as a starting value. )", 0) \ \ DECLARE(Bool, output_format_orc_string_as_string, true, R"( diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 910d9b7efe4e5..75d274f67ff39 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -152,6 +152,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"enable_function_early_short_circuit", false, false, "New setting"}, {"merge_tree_prefetch_json_shared_data_substreams", true, true, "New setting to control prefetching of JSON shared data substreams that are read by seeking to a mark in Wide parts."}, {"iceberg_compaction_commit_batch_size", 100, 100, "New setting"}, + {"output_format_arrow_record_batch_size", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target row count. The default `0` preserves one record batch per block."}, + {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, }); addSettingsChanges(settings_changes_history, "26.7", { diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 17befc8e3313f..a36d3af0f3e35 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -348,6 +348,8 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.arrow.output_compression_method = settings[Setting::output_format_arrow_compression_method]; format_settings.arrow.output_date_as_uint16 = settings[Setting::output_format_arrow_date_as_uint16]; format_settings.arrow.output_unsupported_types_as_binary = settings[Setting::output_format_arrow_unsupported_types_as_binary]; + format_settings.arrow.output_record_batch_rows = settings[Setting::output_format_arrow_record_batch_size]; + format_settings.arrow.output_record_batch_bytes = settings[Setting::output_format_arrow_record_batch_size_bytes]; format_settings.orc.allow_missing_columns = settings[Setting::input_format_orc_allow_missing_columns]; format_settings.orc.row_batch_size = settings[Setting::input_format_orc_row_batch_size]; format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_orc_skip_columns_with_unsupported_types_in_schema_inference]; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 52ac7ff553175..42f62e34fb22c 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -199,6 +199,8 @@ struct FormatSettings ArrowCompression output_compression_method = ArrowCompression::NONE; bool output_date_as_uint16 = false; bool output_unsupported_types_as_binary = true; + UInt64 output_record_batch_rows = 0; + UInt64 output_record_batch_bytes = 0; } arrow{}; struct AvroSchemaRegistryTimeouts diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockInputFormat.cpp index f4eca795c1145..031004f4dfeff 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockInputFormat.cpp @@ -1235,6 +1235,8 @@ cat forex_eurusd.arrow | clickhouse-client --query="INSERT INTO some_table FORMA | `output_format_arrow_compression_method` | Compression method for Arrow output format. Supported codecs: lz4_frame, zstd, none (uncompressed) | `lz4_frame` | | `output_format_arrow_fixed_string_as_fixed_byte_array` | Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns. | `1` | | `output_format_arrow_low_cardinality_as_dictionary` | Enable output LowCardinality type as Dictionary Arrow type | `0` | +| `output_format_arrow_record_batch_size` | Target rows per record batch when combining small blocks. Buffering can increase memory use and delay the first batch until the query finishes. `0` disables the row target. | `0` | +| `output_format_arrow_record_batch_size_bytes` | Target bytes of accumulated block data per record batch. Buffering can increase memory use and delay the first batch until the query finishes. `0` disables the byte target. | `0` | | `output_format_arrow_string_as_string` | Use Arrow String type instead of Binary for String columns | `1` | | `output_format_arrow_unsupported_types_as_binary` | Output a type that has no Arrow equivalent (e.g. `BFloat16`, `AggregateFunction`) as raw binary data. If false, such a type raises an exception. | `1` | | `output_format_arrow_use_64_bit_indexes_for_dictionary` | Always use 64 bit integers for dictionary indexes in Arrow format | `0` | @@ -1369,6 +1371,8 @@ the blog post | `output_format_arrow_date_as_uint16` | Write Date values as plain 16-bit numbers (read back as UInt16), instead of converting to a 32-bit Arrow DATE32 type (read back as Date32). | `0` | | `output_format_arrow_fixed_string_as_fixed_byte_array` | Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns. | `1` | | `output_format_arrow_low_cardinality_as_dictionary` | Enable output LowCardinality type as Dictionary Arrow type | `0` | +| `output_format_arrow_record_batch_size` | Target rows per record batch when combining small blocks. Buffering can increase memory use and delay the first batch until the query finishes. `0` disables the row target. | `0` | +| `output_format_arrow_record_batch_size_bytes` | Target bytes of accumulated block data per record batch. Buffering can increase memory use and delay the first batch until the query finishes. `0` disables the byte target. | `0` | | `output_format_arrow_string_as_string` | Use Arrow String type instead of Binary for String columns | `1` | | `output_format_arrow_unsupported_types_as_binary` | Output a type that has no Arrow equivalent (e.g. `BFloat16`, `AggregateFunction`) as raw binary data. If false, such a type raises an exception. | `1` | | `output_format_arrow_use_64_bit_indexes_for_dictionary` | Always use 64 bit integers for dictionary indexes in Arrow format | `0` | diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp index f448d28199aef..6253eb086fd8d 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp @@ -353,8 +353,49 @@ std::pair ArrowIPCBlockOutputFormat::substituteDictionar void ArrowIPCBlockOutputFormat::consume(Chunk chunk) { + /// Not in `writeChunk`: a reader gets the schema on the first chunk even when batches are combined. writeSchemaIfNeeded(); + const size_t target_rows = format_settings.arrow.output_record_batch_rows; + const size_t target_bytes = format_settings.arrow.output_record_batch_bytes; + if (!target_rows && !target_bytes) + { + writeChunk(std::move(chunk)); + return; + } + + if (chunk.getNumRows() == 0) + return; + + /// `ColumnConst::insertRangeFrom` advances the row count without copying the value, and a lazily sparse + /// or replicated column cannot be appended to a materialized one. It also has to precede the size + /// measured below, which `ColumnConst::byteSize` reports as a single stored value. + materializeChunk(chunk); + + auto reached = [&](size_t rows, size_t bytes) + { return (target_rows && rows >= target_rows) || (target_bytes && bytes >= target_bytes); }; + + /// A chunk that already fills a batch is written as it stands, so a large block is neither copied nor + /// combined with its neighbours. `Chunk::append` also requires at least one column. + if (chunk.getNumColumns() == 0 || reached(chunk.getNumRows(), chunk.bytes())) + { + if (staged.getNumRows()) + writeChunk(std::move(staged)); + writeChunk(std::move(chunk)); + return; + } + + if (staged.getNumRows()) + staged.append(chunk); + else + staged = std::move(chunk); + + if (reached(staged.getNumRows(), staged.bytes())) + writeChunk(std::move(staged)); +} + +void ArrowIPCBlockOutputFormat::writeChunk(Chunk chunk) +{ const size_t num_rows = chunk.getNumRows(); const Columns & columns = chunk.getColumns(); @@ -378,6 +419,10 @@ void ArrowIPCBlockOutputFormat::consume(Chunk chunk) void ArrowIPCBlockOutputFormat::finalizeImpl() { + /// The whole output of a result that never reached a target leaves through here. + if (staged.getNumRows()) + writeChunk(std::move(staged)); + /// Make sure even an empty result produces a valid stream/file (schema, then EOS or footer). writeSchemaIfNeeded(); @@ -399,6 +444,8 @@ void ArrowIPCBlockOutputFormat::finalizeImpl() void ArrowIPCBlockOutputFormat::resetFormatterImpl() { + /// `finalizeImpl` drains it on every normal path; this also covers a reset after an interrupted one. + staged.clear(); message_writer.emplace(out); schema_written = false; dictionary_blocks.clear(); diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h index 13cc336dac926..2be27e359c1f9 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h @@ -43,6 +43,9 @@ class ArrowIPCBlockOutputFormat final : public IOutputFormat void finalizeImpl() override; void resetFormatterImpl() override; + /// Encodes one chunk as exactly one record batch, preceded by the dictionary batches it references. + void writeChunk(Chunk chunk); + void writeSchemaIfNeeded(); /// Writes one encapsulated message for an encoded batch (a record batch, or a dictionary batch /// when `dictionary_id` is set), returning its location for recording an Arrow file `Block`. @@ -94,6 +97,11 @@ class ArrowIPCBlockOutputFormat final : public IOutputFormat }; ArrowIPC::DictPlans column_dict_plans; VectorWithMemoryTracking dictionary_states; + + /// Rows of consecutive chunks that are individually smaller than the configured record batch target, + /// to be written as one record batch. The first chunk becomes the accumulator itself, so a staged batch + /// can hold one source block's allocations until it is written. + Chunk staged; }; } diff --git a/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.reference b/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.reference new file mode 100644 index 0000000000000..ec8e6ce9d8bc9 --- /dev/null +++ b/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.reference @@ -0,0 +1,65 @@ +--- one batch per block by default --- +batches: 64 +64 2016 1 +--- both targets set: a result below both leaves as one batch --- +batches: 1 +64 2016 1 +--- row target alone: 64 rows in batches of 16 --- +batches: 4 +64 2016 1 +--- byte target alone: 8 bytes per row, so batches of 8 rows --- +batches: 8 +64 2016 1 +--- a combined batch may exceed the target: two 40-row blocks under a 41-row target --- +batches: 1, rows per batch: [80] +80 3160 1 +--- blocks that already reach the target are neither merged nor split --- +batches: 4, rows per batch: [65409, 65409, 65409, 3773] +200000 19999900000 1 +--- a block reaching the target flushes what is staged instead of absorbing it --- +batches: 3, rows per batch: [3, 65409, 254] +65666 6450733123 1 +--- a multi-column result keeps every value, in order, when combined --- +batches: 1 +7 0 \N +7 1 1 +7 2 2 +7 0 3 +7 1 \N +7 2 5 +7 0 6 +7 1 7 +--- dictionary-encoded LowCardinality spread over several combined batches --- +batches: 4, rows per batch: [4, 4, 4, 4] +0 2 +1 2 +2 2 +3 2 +shared 8 +--- the same dictionary column combined into one batch --- +batches: 1, rows per batch: [16] +0 2 +1 2 +2 2 +3 2 +shared 8 +--- the byte target counts the bytes the block holds, which for LowCardinality is a deduplicated value --- +batches: 1, rows per batch: [100] +100 1 100000 +batches: 50 +--- the row criterion still applies when the byte criterion is set, and bounds that batch --- +batches: 13, rows per batch: [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4] +100 1 100000 +--- a block keeping the dictionary of the block it was filtered out of reaches the byte target alone --- +batches: 4, rows per batch: [1, 1, 1, 1] +4 4 1000 +batches: 1, rows per batch: [4] +--- the row target alone combines those same blocks --- +batches: 1, rows per batch: [4] +4 4 1000 +--- the Arrow file format coalesces too, and its footer stays consistent --- +batches: 1 +64 2016 1 +--- an Arrow file footer with one Block per combined batch --- +batches: 4, rows per batch: [16, 16, 16, 16] +64 2016 1 diff --git a/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.sh b/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.sh new file mode 100755 index 0000000000000..1398fdf849954 --- /dev/null +++ b/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Tests `output_format_arrow_record_batch_size` and `output_format_arrow_record_batch_size_bytes`, which +# combine consecutive blocks that are individually smaller than the target into one Arrow IPC record batch. +# Every arm asserts an exact batch count plus the rows read back, so that combining cannot lose, duplicate +# or reorder rows. +# +# `numbers` (never `numbers_mt`) with `max_block_size` pinned in the query's own SETTINGS, which beats +# the test runner's randomization, is what makes the sequence of blocks reaching the writer, and therefore +# the batch counts, reproducible. `max_threads = 1` keeps the source and the read-back single-stream, so +# that neither the batch counts nor the `in_order` check below can depend on scheduling. +# +# The result is written with `FORMAT ArrowStream` redirected to a file, never with +# `INSERT INTO FUNCTION file(...)`: `ArrowStream` prefers large blocks, so the INSERT path would squash to +# `min_insert_block_size_rows` and deliver the whole fixture as one block, leaving every arm at one batch +# whether or not the feature works. + +FILE="${CLICKHOUSE_TMP}/05211_arrow_batch_coalescing.arrows" + +# An older pyarrow cannot read the sentinel written for an uncompressed lz4_frame body, and the batch +# counts must not depend on the reader's codec support. +COMMON="max_threads = 1, output_format_arrow_compression_method = 'none'" + +# Prints the number of record batches, and with a second argument also the rows in each of them. +arrow_batches() { + python3 - "$FILE" "$1" "${2-}" <<'PY' +import sys +import pyarrow as pa + +path, mode, detail = sys.argv[1], sys.argv[2], sys.argv[3] +with pa.OSFile(path, "rb") as source: + reader = pa.ipc.open_file(source) if mode == "file" else pa.ipc.open_stream(source) + if mode == "file": + rows = [reader.get_batch(i).num_rows for i in range(reader.num_record_batches)] + else: + rows = [batch.num_rows for batch in reader] +out = "batches: %d" % len(rows) +if detail: + out += ", rows per batch: %s" % rows +print(out) +PY +} + +# The count and the checksum prove no row was lost or duplicated; `in_order` proves that combining +# appended rows instead of reordering them, since every fixture below is produced in ascending order. +oracle() { + ${CLICKHOUSE_LOCAL} --query "SELECT count(), sum(number), groupArray(number) = arraySort(groupArray(number)) AS in_order FROM file('${FILE}', '${1-ArrowStream}') SETTINGS max_threads = 1" +} + +# The same, for the `LowCardinality(String)` arms whose fixture is not a sequence of numbers: the row count, +# the distinct values and their total length prove the combined batch still describes every row. +string_oracle() { + ${CLICKHOUSE_LOCAL} --query "SELECT count(), uniqExact(s), sum(length(s)) FROM file('${FILE}', 'ArrowStream') SETTINGS max_threads = 1" +} + +echo "--- one batch per block by default ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON} FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +oracle + +echo "--- both targets set: a result below both leaves as one batch ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 1048576 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +oracle + +echo "--- row target alone: 64 rows in batches of 16 ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 16, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +oracle + +echo "--- byte target alone: 8 bytes per row, so batches of 8 rows ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 0, output_format_arrow_record_batch_size_bytes = 64 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +oracle + +echo "--- a combined batch may exceed the target: two 40-row blocks under a 41-row target ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(80) SETTINGS max_block_size = 40, ${COMMON}, output_format_arrow_record_batch_size = 41, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +oracle + +echo "--- blocks that already reach the target are neither merged nor split ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(200000) SETTINGS max_block_size = 65409, ${COMMON}, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +oracle + +echo "--- a block reaching the target flushes what is staged instead of absorbing it ---" +# The three blocks are 3, 65409 and 254 rows. `toString(number) IN (...)` rather than `number < 3` keeps +# the condition out of reach of the range analysis of `numbers`, which would otherwise generate exactly +# the matching rows and deliver them as two full blocks, leaving no small block in front of the big one. +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(131072) WHERE toString(number) IN ('0', '1', '2') OR number > 65408 SETTINGS max_block_size = 65409, ${COMMON}, output_format_arrow_record_batch_size = 1000, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +oracle + +echo "--- a multi-column result keeps every value, in order, when combined ---" +${CLICKHOUSE_LOCAL} --query "SELECT 7 AS c, toLowCardinality(toString(number % 3)) AS lc, if(number % 4 = 0, NULL, number)::Nullable(UInt64) AS n FROM numbers(8) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 65409 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${FILE}', 'ArrowStream') SETTINGS max_threads = 1" + +echo "--- dictionary-encoded LowCardinality spread over several combined batches ---" +# Every batch after the first mixes a value the stream's dictionary already carries with one it does not, so +# each emits a dictionary delta and has the indexes of a merged column remapped against the accumulated dictionary. +DICT_FIXTURE="toLowCardinality(if(number % 4 < 2, 'shared', toString(intDiv(number, 4)))) AS lc FROM numbers(16)" +DICT_READBACK="SELECT lc, count() FROM file('${FILE}', 'ArrowStream') GROUP BY lc ORDER BY lc SETTINGS max_threads = 1" +${CLICKHOUSE_LOCAL} --query "SELECT ${DICT_FIXTURE} SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 1, output_format_arrow_record_batch_size = 4 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +${CLICKHOUSE_LOCAL} --query "${DICT_READBACK}" + +echo "--- the same dictionary column combined into one batch ---" +${CLICKHOUSE_LOCAL} --query "SELECT ${DICT_FIXTURE} SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 1, output_format_arrow_record_batch_size = 65409 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +${CLICKHOUSE_LOCAL} --query "${DICT_READBACK}" + +echo "--- the byte target counts the bytes the block holds, which for LowCardinality is a deduplicated value ---" +# The byte target counts one index per row plus the dictionary once (`ColumnLowCardinality::byteSize`), while +# the encoder writes one value per row: documented on the setting, hence asserted here rather than filed as a bug. +${CLICKHOUSE_LOCAL} --query "SELECT toLowCardinality(repeat('x', 1000)) AS s FROM numbers(100) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 0, output_format_arrow_record_batch_size = 0, output_format_arrow_record_batch_size_bytes = 1500 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +string_oracle +${CLICKHOUSE_LOCAL} --query "SELECT repeat('x', 1000) AS s FROM numbers(100) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 0, output_format_arrow_record_batch_size_bytes = 1500 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream + +echo "--- the row criterion still applies when the byte criterion is set, and bounds that batch ---" +${CLICKHOUSE_LOCAL} --query "SELECT toLowCardinality(repeat('x', 1000)) AS s FROM numbers(100) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 0, output_format_arrow_record_batch_size = 8, output_format_arrow_record_batch_size_bytes = 1500 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +string_oracle + +echo "--- a block keeping the dictionary of the block it was filtered out of reaches the byte target alone ---" +# `ColumnLowCardinality::filter` keeps the whole source dictionary, so each surviving row measures the +# ~64 KB dictionary its 250-row block built and is written as its own record batch, which also ends the +# batches the row target set here would otherwise have combined. The `OR` is what keeps the filter above +# the projection that builds `s`, so the dictionary is built for the whole block; `length(s) = 0` is +# never true. The `String` arm below has the same values and the same targets and is combined. +LC_SOURCE_DICT="SELECT s FROM (SELECT toLowCardinality(repeat(toString(number), 100)) AS s, number AS n FROM numbers(1000)) WHERE (n % 250 = 0) OR (length(s) = 0)" +${CLICKHOUSE_LOCAL} --query "${LC_SOURCE_DICT} SETTINGS max_block_size = 250, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 0, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 8192 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +string_oracle +${CLICKHOUSE_LOCAL} --query "SELECT s FROM (SELECT repeat(toString(number), 100) AS s, number AS n FROM numbers(1000)) WHERE (n % 250 = 0) OR (length(s) = 0) SETTINGS max_block_size = 250, ${COMMON}, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 8192 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail + +echo "--- the row target alone combines those same blocks ---" +${CLICKHOUSE_LOCAL} --query "${LC_SOURCE_DICT} SETTINGS max_block_size = 250, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 0, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +string_oracle + +echo "--- the Arrow file format coalesces too, and its footer stays consistent ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 65409 FORMAT Arrow" > "${FILE}" +arrow_batches file +oracle Arrow + +echo "--- an Arrow file footer with one Block per combined batch ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 16 FORMAT Arrow" > "${FILE}" +arrow_batches file detail +oracle Arrow + +rm -f "${FILE}" From 2300225c9d255e9480a3b78072659449d2bd226f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 16 Sep 2026 16:01:17 +0000 Subject: [PATCH 008/296] Backport #119032 to 26.8: Text index: make `mergeTreeTextIndex` stricter when a row policy applies --- src/Storages/StorageMergeTreeTextIndex.cpp | 66 ++++++++++--------- src/Storages/StorageMergeTreeTextIndex.h | 4 +- .../TableFunctionMergeTreeTextIndex.cpp | 54 ++++++++++++--- ..._index_mergeTreeTextIndex_access.reference | 2 +- ...46_text_index_mergeTreeTextIndex_access.sh | 2 +- ...rgeTreeTextIndex_create_table_as.reference | 1 + ...dex_mergeTreeTextIndex_create_table_as.sql | 5 ++ ...x_mergeTreeTextIndex_interserver.reference | 2 + ...xt_index_mergeTreeTextIndex_interserver.sh | 46 +++++++++++++ ...rgeTreeTextIndex_metadata_access.reference | 10 +++ ...ndex_mergeTreeTextIndex_metadata_access.sh | 64 ++++++++++++++++++ ..._index_mergeTreeTextIndex_remote.reference | 9 +++ ...53_text_index_mergeTreeTextIndex_remote.sh | 60 +++++++++++++++++ ...eTextIndex_row_policy_any_column.reference | 4 ++ ...ergeTreeTextIndex_row_policy_any_column.sh | 56 ++++++++++++++++ 15 files changed, 342 insertions(+), 43 deletions(-) create mode 100644 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference create mode 100644 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql create mode 100644 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference create mode 100755 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh create mode 100644 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference create mode 100755 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh create mode 100644 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference create mode 100755 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh create mode 100644 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference create mode 100755 tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh diff --git a/src/Storages/StorageMergeTreeTextIndex.cpp b/src/Storages/StorageMergeTreeTextIndex.cpp index 8c16dd595e1da..2ebf5b4a57ce7 100644 --- a/src/Storages/StorageMergeTreeTextIndex.cpp +++ b/src/Storages/StorageMergeTreeTextIndex.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,6 @@ #include #include #include -#include namespace DB { @@ -361,7 +361,10 @@ void ReadFromMergeTreeTextIndex::applyFilters(ActionDAGNodes added_filter_nodes) void ReadFromMergeTreeTextIndex::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - auto filtered_parts = VirtualColumnUtils::filterDataPartsWithExpression(storage->data_parts, virtual_columns_filter); + /// Taken at read time: the storage outlives the query in a table created from the function before that was forbidden. + auto data_parts = dynamic_cast(*storage->source_table).getDataPartsVectorForInternalUsage(); + std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); + auto filtered_parts = VirtualColumnUtils::filterDataPartsWithExpression(data_parts, virtual_columns_filter); if (filtered_parts.empty()) { @@ -407,13 +410,9 @@ StorageMergeTreeTextIndex::StorageMergeTreeTextIndex( , source_table(source_table_) , text_index(std::move(text_index_)) { - const auto * merge_tree = dynamic_cast(source_table.get()); - if (!merge_tree) + if (!dynamic_cast(source_table.get())) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeTextIndex expected MergeTree table, got: {}", source_table->getName()); - data_parts = merge_tree->getDataPartsVectorForInternalUsage(); - std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); - StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns); storage_metadata.setVirtuals(createVirtuals()); @@ -428,6 +427,34 @@ VirtualColumnsDescription StorageMergeTreeTextIndex::createVirtuals() return desc; } +void StorageMergeTreeTextIndex::checkAccess(const ContextPtr & context, const StorageID & source_storage_id, const IMergeTreeIndex & index) +{ + /// The checks below are for the user who runs the query, so a shard of a distributed query may run it only as the + /// initiating user: authenticated by the interserver secret, or reached by `remote(...)` as the same user, which the + /// initiator confirms by pushing its roles (it does not when it rewrote the initial user to the connection user). + const auto & client_info = context->getClientInfo(); + const bool same_user = client_info.initial_user == client_info.current_user && client_info.current_roles.has_value(); + if (client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY + && client_info.interface != ClientInfo::Interface::TCP_INTERSERVER && !same_user) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Table function `mergeTreeTextIndex` checks the access of the user who runs the query, so a shard of a " + "distributed query can execute it only as the initiating user: through a cluster with an interserver secret, " + "or through `remote` with the credentials of that user"); + + context->checkAccess(AccessType::SELECT, source_storage_id, index.getColumnsRequiredForIndexCalc()); + + /// The index is built over all rows of a part, so it contains tokens of the rows a row policy hides, + /// regardless of which columns the policy filters on. The policy cannot be applied to the dictionary. + auto row_policy_filter = context->getRowPolicyFilter( + source_storage_id.getDatabaseName(), source_storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot read from `mergeTreeTextIndex` because a row policy is applied on table {}. " + "The text index covers all rows of the table, so reading its tokens would violate the row policy", + source_storage_id.getNameForLogs()); +} + void StorageMergeTreeTextIndex::readImpl( QueryPlan & query_plan, const Names & column_names, @@ -438,30 +465,7 @@ void StorageMergeTreeTextIndex::readImpl( size_t max_block_size, size_t num_streams) { - auto source_storage_id = source_table->getStorageID(); - auto required_columns = text_index->getColumnsRequiredForIndexCalc(); - context->checkAccess(AccessType::SELECT, source_storage_id, required_columns); - /// If the row policy filter references any column required for building the index, - /// reading from the text index would expose tokens derived from those columnsand violate the row policy. - auto row_policy_filter = context->getRowPolicyFilter(source_storage_id.getDatabaseName(), source_storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); - - if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) - { - RequiredSourceColumnsVisitor::Data columns_context; - RequiredSourceColumnsVisitor(columns_context).visit(row_policy_filter->expression); - NameSet row_policy_columns = columns_context.requiredColumns(); - - for (const auto & column_name : required_columns) - { - if (row_policy_columns.contains(column_name)) - { - throw Exception(ErrorCodes::ACCESS_DENIED, - "Cannot read from `mergeTreeTextIndex` because a row policy on column `{}` " - "is applied on table {}. Reading text index tokens could violate the row policy", - column_name, source_storage_id.getNameForLogs()); - } - } - } + checkAccess(context, source_table->getStorageID(), *text_index); auto sample_block = std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names)); auto this_ptr = std::static_pointer_cast(shared_from_this()); diff --git a/src/Storages/StorageMergeTreeTextIndex.h b/src/Storages/StorageMergeTreeTextIndex.h index 92245724d26ea..f39db5f33dfae 100644 --- a/src/Storages/StorageMergeTreeTextIndex.h +++ b/src/Storages/StorageMergeTreeTextIndex.h @@ -34,12 +34,14 @@ class StorageMergeTreeTextIndex final : public StorageWithCommonVirtualColumns static VirtualColumnsDescription createVirtuals(); + /// Throws if the user may not read the tokens of the index of the table `source_storage_id`. + static void checkAccess(const ContextPtr & context, const StorageID & source_storage_id, const IMergeTreeIndex & index); + private: friend class ReadFromMergeTreeTextIndex; StoragePtr source_table; MergeTreeIndexPtr text_index; - MergeTreeData::DataPartsVector data_parts; }; } diff --git a/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp b/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp index 60a024e9a02fe..29f2a251d9e12 100644 --- a/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -10,13 +11,16 @@ #include #include #include +#include #include +#include namespace DB { namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; @@ -28,6 +32,10 @@ class TableFunctionMergeTreeTextIndex : public ITableFunction static constexpr auto name = "mergeTreeTextIndex"; std::string getName() const override { return name; } + /// The returned storage holds its source table's storage object, so a persisted table would keep the source undroppable. + /// A persisted definition would also resolve the source table under the global context or the engine credentials. + bool canBeUsedToCreateTable() const override { return false; } + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; @@ -39,6 +47,10 @@ class TableFunctionMergeTreeTextIndex : public ITableFunction ColumnsDescription cached_columns, bool is_insert_query) const override; + /// Resolves the index of the source table and checks that the user may read it. + std::pair resolveIndex(ContextPtr context) const; + static ColumnsDescription getColumns(); + const char * getStorageEngineName() const override { return ""; @@ -78,7 +90,7 @@ static std::shared_ptr getDictionaryCompressionType() return std::make_shared(std::move(values)); } -ColumnsDescription TableFunctionMergeTreeTextIndex::getActualTableStructure(ContextPtr, bool /*is_insert_query*/) const +ColumnsDescription TableFunctionMergeTreeTextIndex::getColumns() { return ColumnsDescription{{ {"part_name", std::make_shared()}, @@ -92,13 +104,19 @@ ColumnsDescription TableFunctionMergeTreeTextIndex::getActualTableStructure(Cont }}; } -StoragePtr TableFunctionMergeTreeTextIndex::executeImpl( - const ASTPtr & /*ast_function*/, - ContextPtr context, - const std::string & table_name, - ColumnsDescription /*cached_columns*/, - bool is_insert_query) const +std::pair TableFunctionMergeTreeTextIndex::resolveIndex(ContextPtr context) const { + /// A table persisted before that was forbidden resolves the function under the load context, which has no user. + if (!context->getUserID()) + { + context = CurrentThread::tryGetQueryContext(); + if (!context) + throw Exception(ErrorCodes::ACCESS_DENIED, "Table function 'mergeTreeTextIndex' cannot check the access of the user outside of a query"); + } + + /// Otherwise the errors below would reveal the engine and the indexes of a table the user cannot see. + context->checkAccess(AccessType::SHOW_TABLES, source_database, source_table); + auto source_table_ptr = DatabaseCatalog::instance().getTable(StorageID{source_database, source_table}, context); auto metadata_snapshot = source_table_ptr->getInMemoryMetadataPtr(context, false); const auto & index_desc = metadata_snapshot->getSecondaryIndices().getByName(source_index_name); @@ -114,14 +132,32 @@ StoragePtr TableFunctionMergeTreeTextIndex::executeImpl( throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeTextIndex expected MergeTree table, got: {}", source_table_ptr->getName()); auto text_index = MergeTreeIndexFactory::instance().get(metadata_snapshot, index_desc, *merge_tree->getSettings()); - auto columns = getActualTableStructure(context, is_insert_query); + StorageMergeTreeTextIndex::checkAccess(context, source_table_ptr->getStorageID(), *text_index); + return {std::move(source_table_ptr), std::move(text_index)}; +} + +ColumnsDescription TableFunctionMergeTreeTextIndex::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const +{ + /// The structure is static, but this is where e.g. `remote` over a local shard checks the access of the user. + resolveIndex(context); + return getColumns(); +} + +StoragePtr TableFunctionMergeTreeTextIndex::executeImpl( + const ASTPtr & /*ast_function*/, + ContextPtr context, + const std::string & table_name, + ColumnsDescription /*cached_columns*/, + bool /*is_insert_query*/) const +{ + auto [source_table_ptr, text_index] = resolveIndex(context); StorageID storage_id(getDatabaseName(), table_name); auto res = std::make_shared( std::move(storage_id), std::move(source_table_ptr), std::move(text_index), - std::move(columns)); + getColumns()); res->startup(); return res; diff --git a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference index 14efafd0c46c2..d219eb8e6a852 100644 --- a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference +++ b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference @@ -9,5 +9,5 @@ ACCESS_DENIED OK ACCESS_DENIED ACCESS_DENIED -OK +ACCESS_DENIED ACCESS_DENIED diff --git a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh index 87e86c4edc6c0..ae07f79cd8468 100755 --- a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh +++ b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh @@ -66,7 +66,7 @@ check_access "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_ab)" ## Row policy tests $CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name;" -# Row policy on column `a`: idx_a and idx_ab denied, idx_b allowed +# Row policy on column `a`: every index is denied, because each dictionary contains tokens of the hidden rows $CLICKHOUSE_CLIENT -q "CREATE ROW POLICY p1_03917 ON $CLICKHOUSE_DATABASE.tab FOR SELECT USING a = 'hello' TO $user_name;" check_access "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_a)" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql new file mode 100644 index 0000000000000..09357fa59d889 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql @@ -0,0 +1,5 @@ +-- A persisted table over `mergeTreeTextIndex` would resolve the source table without the reader's grants. +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE tab_index AS mergeTreeTextIndex(currentDatabase(), tab, idx_s); -- { serverError BAD_ARGUMENTS } +CREATE TABLE tab_index (part_name String, token String) AS mergeTreeTextIndex(currentDatabase(), tab, idx_s); -- { serverError BAD_ARGUMENTS } +SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 'tab_index'; diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference new file mode 100644 index 0000000000000..8734b92528426 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference @@ -0,0 +1,2 @@ +ACCESS_DENIED +['hidden','token'] diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh new file mode 100755 index 0000000000000..e6052c5ed95c7 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# The interserver secret needs the SSL library, which the fast test build does not have. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Through an interserver connection the shard authenticates the initiating user, so the checks of +# `mergeTreeTextIndex` apply to that user instead of refusing the query as over an ordinary connection. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO tab VALUES ('hidden token'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT REMOTE ON *.* TO $user_name; +GRANT CREATE TEMPORARY TABLE ON *.* TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +query="SELECT arraySort(groupUniqArray(token)) FROM cluster('test_cluster_interserver_secret', mergeTreeTextIndex('$CLICKHOUSE_DATABASE', 'tab', 'idx_s')) SETTINGS prefer_localhost_replica = 0" + +run_as_user "$query" + +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +run_as_user "$query" + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference new file mode 100644 index 0000000000000..361650ff1a48e --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference @@ -0,0 +1,10 @@ +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +BAD_ARGUMENTS +BAD_ARGUMENTS +NOT_IMPLEMENTED diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh new file mode 100755 index 0000000000000..f49d9906b7bad --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Without any grant on the table, `mergeTreeTextIndex` must not reveal whether the table +# is a MergeTree table or which indexes it has: every error must be ACCESS_DENIED. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab +( + a String, + b String, + INDEX idx_text a TYPE text(tokenizer = 'splitByNonAlpha'), + INDEX idx_set b TYPE set(0) +) +ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE tab_memory (a String) ENGINE = Memory; + +INSERT INTO tab VALUES ('hello', 'world'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "OK" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_set)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_missing)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab_memory, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab_missing, idx_text)" +run_as_user "INSERT INTO FUNCTION mergeTreeTextIndex(currentDatabase(), tab, idx_text) SELECT 'p', 'x', 'raw', 1, 1, 0, 0, 0" + +# A grant on any column implies SHOW TABLES, so index metadata becomes visible, +# while reading the index still requires SELECT on the indexed column. +$CLICKHOUSE_CLIENT -q "GRANT SELECT(b) ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_set)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_missing)" + +# The index is read-only: the access check runs before the write is rejected. +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION mergeTreeTextIndex(currentDatabase(), tab, idx_text) SELECT 'p', 'x', 'raw', 1, 1, 0, 0, 0" 2>&1 | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP TABLE tab_memory; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference new file mode 100644 index 0000000000000..80d7d44bd4660 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference @@ -0,0 +1,9 @@ +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +2 +2 +2 diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh new file mode 100755 index 0000000000000..0bd207645ea78 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `remote` over a local shard validates the access of the current user while inferring the structure of a nested +# table function, and may then route the query over loopback with other credentials. `mergeTreeTextIndex` has a +# static structure, so it has to check the access of the user itself there. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO tab VALUES ('hidden token'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT REMOTE ON *.* TO $user_name; +GRANT CREATE TEMPORARY TABLE ON *.* TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +index="mergeTreeTextIndex('$CLICKHOUSE_DATABASE', 'tab', 'idx_s')" +query="SELECT count() FROM remote('127.0.0.1:$CLICKHOUSE_PORT_TCP', $index)" +same_user_query="SELECT count() FROM remote('127.0.0.1:$CLICKHOUSE_PORT_TCP', $index, '$user_name', 'password')" + +function run_remote_as_user() +{ + for localhost_replica in 0 1; do + run_as_user "$query SETTINGS prefer_localhost_replica = $localhost_replica" + done + for localhost_replica in 0 1; do + run_as_user "$same_user_query SETTINGS prefer_localhost_replica = $localhost_replica" + done +} + +run_as_user "DESCRIBE TABLE $index" +run_remote_as_user + +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +# Over an ordinary connection the shard runs the query as the user of the connection, which the function refuses +# unless that is the initiating user; with the local shortcut it runs as the user itself. +run_remote_as_user + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference new file mode 100644 index 0000000000000..2efecfaa49217 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference @@ -0,0 +1,4 @@ +['visible apple phrase'] +0 +ACCESS_DENIED +['apple','hidden','phrase','visible','zebra'] diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh new file mode 100755 index 0000000000000..d5a3aa5698697 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A row policy on a column that is not indexed must still deny reading the text index: +# the dictionary contains the tokens of the rows the policy hides. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab +( + tenant_id String, + doc_id UInt64, + secret_text String, + INDEX idx_secret secret_text TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY doc_id +SETTINGS index_granularity = 1; + +INSERT INTO tab VALUES ('tenant_a', 1, 'visible apple phrase'), ('tenant_b', 2, 'hidden zebra phrase'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name; +CREATE ROW POLICY p_05153 ON $CLICKHOUSE_DATABASE.tab FOR SELECT USING tenant_id = 'tenant_a' TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + elif echo "$output" | grep -q "ACCESS_DENIED"; then + echo "ACCESS_DENIED" + else + echo "$output" + fi +} + +run_as_user "SELECT groupArray(secret_text) FROM tab" +run_as_user "SELECT count() FROM tab WHERE hasToken(secret_text, 'zebra')" +run_as_user "SELECT arraySort(groupArray(token)) FROM mergeTreeTextIndex(currentDatabase(), tab, idx_secret)" + +$CLICKHOUSE_CLIENT -q "DROP ROW POLICY p_05153 ON $CLICKHOUSE_DATABASE.tab" + +run_as_user "SELECT arraySort(groupArray(token)) FROM mergeTreeTextIndex(currentDatabase(), tab, idx_secret)" + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" From a0f67cdaf8ffdfddda0251e520bd705465de38c1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 16 Sep 2026 16:43:46 +0000 Subject: [PATCH 009/296] Update autogenerated version to 26.8.6.5 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index bfe1c884e16f6..99f0c1d87ca71 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54518) +SET(VERSION_REVISION 54519) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 6) -SET(VERSION_GITHASH ec5605431dacfc812affc406cc81ca398ca68174) -SET(VERSION_DESCRIBE v26.8.6.1-lts) -SET(VERSION_STRING 26.8.6.1) +SET(VERSION_PATCH 7) +SET(VERSION_GITHASH 4c31daf60f098627e7f9a867bfa2ab5889aafd74) +SET(VERSION_DESCRIBE v26.8.7.1-lts) +SET(VERSION_STRING 26.8.7.1) # end of autochange From fd5856a9d72cb647072ce4904b4d44ef9d42e7a8 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 16 Sep 2026 21:42:19 +0200 Subject: [PATCH 010/296] Record `validate_group_by_all_key_types` in the 26.8 settings-history block The 26.8 backport carries the entry in the 26.7 block, which is what `compatibility` needs: a version before 26.7 restores the earlier acceptance of a `Variant`/`Dynamic` key written as `GROUP BY ALL`. The `settings_changes_history` style check additionally requires every setting the change touches to be recorded under the current version block, so add the no-op record (previous value equal to the new one) under 26.8 as well, mirroring what master carries under 26.9. --- src/Core/SettingsChangesHistory.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 18e6042c5617e..29530ec728c48 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -43,6 +43,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// Note: please check if the key already exists to prevent duplicate entries. addSettingsChanges(settings_changes_history, "26.8", { + {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, From 102678e89bab1b7e602099425e27dedc2d27dc3c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 01:07:02 +0000 Subject: [PATCH 011/296] Backport #117329 to 26.8: Add a setting to forbid listing namespaces in catalogs --- src/Common/FailPoint.cpp | 1 + src/Databases/DataLake/DatabaseDataLake.cpp | 5 + .../DataLake/DatabaseDataLakeSettings.cpp | 1 + src/Databases/DataLake/RestCatalog.cpp | 40 +++-- src/Databases/DataLake/RestCatalog.h | 13 +- src/Databases/DataLake/S3TablesCatalog.cpp | 2 +- .../DataLake/tests/gtest_rest_catalog.cpp | 138 +++++++++++++++- .../StorageObjectStorageSource.cpp | 7 + .../test_lifetime_sources_bug.py | 156 ++++++++++++++++++ .../test_variant_type.py | 75 +++++++++ 10 files changed, 420 insertions(+), 18 deletions(-) create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_lifetime_sources_bug.py create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_variant_type.py diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 1e58899e7cc92..25428b51a6307 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -343,6 +343,7 @@ static struct InitFiu PAUSEABLE(keeper_changelog_readahead_serve_wait) \ PAUSEABLE(keeper_changelog_readahead_park_armed) \ PAUSEABLE(keeper_changelog_readahead_pre_drain) \ + PAUSEABLE(object_storage_source_pause_before_virtual_columns) \ REGULAR(keeper_changelog_readahead_fill_exception) \ REGULAR(distributed_plan_record_failure_while_starting_tasks) \ ONCE(zk_send_thread_request_window_throw) \ diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index e812f591c57a7..38c41f1e182e9 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -95,6 +95,7 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsString google_adc_quota_project_id; extern const DatabaseDataLakeSettingsString google_adc_credentials_file; extern const DatabaseDataLakeSettingsBool force_add_bucket; + extern const DatabaseDataLakeSettingsBool flat_namespaces; } namespace Setting @@ -263,6 +264,7 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::auth_header], settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, + settings[DatabaseDataLakeSetting::flat_namespaces].value, Context::getGlobalContextInstance()); break; } @@ -278,6 +280,7 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::auth_header], settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, + settings[DatabaseDataLakeSetting::flat_namespaces].value, Context::getGlobalContextInstance()); break; } @@ -294,6 +297,7 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::auth_header], settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, + settings[DatabaseDataLakeSetting::flat_namespaces].value, Context::getGlobalContextInstance()); break; } @@ -315,6 +319,7 @@ void DatabaseDataLake::initialize() const onelake_auth_scope, settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, + settings[DatabaseDataLakeSetting::flat_namespaces].value, Context::getGlobalContextInstance()); break; } diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index 421eae178c53f..99a03bba5db31 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -51,6 +51,7 @@ namespace ErrorCodes DECLARE(String, dlf_access_key_id, "", "Access id of DLF token for Paimon REST Catalog", 0) \ DECLARE(String, dlf_access_key_secret, "", "Access secret of DLF token for Paimon REST Catalog", 0) \ DECLARE(Bool, force_add_bucket, false, "When constructing object-storage URLs from the catalog-provided table location and storage_endpoint, prepend the bucket/container name even if the endpoint already contains it. Useful for catalogs that hand back paths without the bucket and expect it to be added at URL construction (Polaris-style paths).", 0) \ + DECLARE(Bool, flat_namespaces, false, "The catalog supports only single-level namespaces, so only top-level namespaces are listed and sub-namespaces are not requested. Set this for Iceberg REST catalogs that reject the `parent` query parameter of the list-namespaces endpoint, such as Apache Polaris federated to AWS Glue, which answers `Glue dataCatalog does not support multipart namespace` with HTTP 400. Some catalogs instead ignore `parent` and echo top-level namespaces back for every parent; without this setting those are listed as fake nested namespaces. Not needed for catalog types that are always flat (`delta_sharing`, BigLake, S3 Tables).", 0) \ #define LIST_OF_DATABASE_ICEBERG_SETTINGS(M, ALIAS) \ DATABASE_ICEBERG_RELATED_SETTINGS(M, ALIAS) \ diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 3b3b86785c5a9..c2a38403f8a0b 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -17,7 +17,6 @@ #include #include #include - #include #include #include @@ -199,6 +198,7 @@ RestCatalog::RestCatalog( const std::string & auth_header_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, DB::ContextPtr context_) : ICatalog(warehouse_) , DB::WithContext(context_) @@ -207,6 +207,7 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , flat_namespaces(flat_namespaces_) { CatalogState initial_state; if (!catalog_credential_.empty()) @@ -233,6 +234,7 @@ RestCatalog::RestCatalog( const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, DB::ContextPtr context_) : ICatalog(warehouse_) , DB::WithContext(context_) @@ -241,6 +243,7 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , flat_namespaces(flat_namespaces_) { } @@ -343,8 +346,9 @@ OneLakeCatalog::OneLakeCatalog( const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, DB::ContextPtr context_) - : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, context_) + : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, flat_namespaces_, context_) { CatalogState initial_state; initial_state.tenant_id = onelake_tenant_id; @@ -629,8 +633,9 @@ HorizonCatalog::HorizonCatalog( const std::string & auth_header_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, DB::ContextPtr context_) - : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, context_) + : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, flat_namespaces_, context_) { CatalogState initial_state; if (!catalog_credential_.empty()) @@ -976,7 +981,7 @@ BigLakeCatalog::BigLakeCatalog( const std::string & google_adc_quota_project_id_, DB::ContextPtr context_, bool allow_server_credentials_in_user_queries_) - : RestCatalog(warehouse_, base_url_, "", "", false, context_) + : RestCatalog(warehouse_, base_url_, "", "", false, /* flat_namespaces */false, context_) , google_project_id(google_project_id_) , google_service_account(google_service_account_) , google_metadata_service(google_metadata_service_) @@ -1321,7 +1326,8 @@ void RestCatalog::getNamespacesRecursive( if (func) func(current_namespace); - getNamespacesRecursive(current_namespace, result, stop_condition, func); + if (!hasFlatNamespaces()) + getNamespacesRecursive(current_namespace, result, stop_condition, func); } } @@ -1343,9 +1349,12 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s bool RestCatalog::hasFlatNamespaces() const { - /// Catalogs whose namespaces are single-level and which ignore the `parent` filter when listing - /// namespaces. For these, sub-namespace listing is skipped (see `parseNamespaces`) so that an echo - /// of the parent is not turned into a fake child, which would otherwise recurse without bound. + /// Catalogs whose namespaces are single-level and which ignore or reject the `parent` filter when + /// listing namespaces. For these, sub-namespace listing is skipped so that an echo of the parent is + /// not turned into a fake child, which would otherwise recurse without bound. + if (flat_namespaces) + return true; + const auto type = getCatalogType(); return type == DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE || type == DB::DatabaseDataLakeCatalogType::ICEBERG_DELTA_SHARING @@ -1416,12 +1425,21 @@ RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & bas "Received error while fetching list of namespaces from iceberg catalog `{}`. ", warehouse); - if (e.code() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND) + if (!base_namespace.empty() && e.getHTTPStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND) message += "Namespace provided in the `parent` query parameter is not found. "; + if (!base_namespace.empty() + && (e.getHTTPStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_BAD_REQUEST + || e.getHTTPStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_IMPLEMENTED)) + message += fmt::format( + "The catalog refused to list sub-namespaces of `{}`. If it supports only single-level " + "namespaces, recreate the database with `SETTINGS flat_namespaces = 1` so that only " + "top-level namespaces are listed. ", + base_namespace); + message += fmt::format( - "Code: {}, status: {}, message: {}", - e.code(), e.getHTTPStatus(), e.displayText()); + "Code: {}, HTTP status: {}, message: {}", + e.code(), static_cast(e.getHTTPStatus()), e.displayText()); throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{}", message); } diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index a21474fd50814..4d0f657719556 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -44,6 +44,7 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & auth_header_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, DB::ContextPtr context_); ~RestCatalog() override = default; @@ -140,6 +141,7 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, DB::ContextPtr context_); void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; @@ -154,6 +156,7 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; + bool flat_namespaces = false; mutable MultiVersion access_token; Poco::Net::HTTPBasicCredentials credentials{}; @@ -180,10 +183,10 @@ class RestCatalog : public ICatalog, public DB::WithContext StopCondition stop_condition, ExecuteFunc func) const; - /// Whether this catalog has flat (single-level) namespaces and ignores the `parent` filter when - /// listing namespaces. Such catalogs (BigLake, Databricks Delta Sharing) echo the same namespaces - /// for any parent; treating those echoes as children would recurse without bound, so sub-namespace - /// listing is skipped for them (see `parseNamespaces`). + /// Whether this catalog has flat (single-level) namespaces, either because its type is always flat + /// (BigLake, Databricks Delta Sharing, S3 Tables) or because of the `flat_namespaces` database + /// setting. Such catalogs are never asked for sub-namespaces (see `getNamespacesRecursive`): they + /// either echo the parent back for any `parent` (which would recurse without bound) or reject it. bool hasFlatNamespaces() const; /// List the immediate child namespaces directly under `base_namespace` @@ -271,6 +274,7 @@ class OneLakeCatalog : public RestCatalog const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, DB::ContextPtr context_); DB::DatabaseDataLakeCatalogType getCatalogType() const override @@ -385,6 +389,7 @@ class HorizonCatalog : public RestCatalog const std::string & auth_header_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, DB::ContextPtr context_); DB::DatabaseDataLakeCatalogType getCatalogType() const override diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index bbf1365250b5a..734b42b95d06c 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -64,7 +64,7 @@ S3TablesCatalog::S3TablesCatalog( const CatalogSettings & catalog_settings_, DB::ContextPtr context_, bool allow_server_credentials_in_user_queries_) - : RestCatalog(warehouse_, base_url_, "", "", false, context_) + : RestCatalog(warehouse_, base_url_, "", "", false, /* flat_namespaces */false, context_) , region(region_) , storage_endpoint(catalog_settings_.storage_endpoint) , signing_service("s3tables") diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index 1b37532ea7d42..b5b4b9b5422ff 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -63,6 +63,11 @@ enum class CatalogShape /// echoes the same top-level namespace for every parent. A REST catalog would recurse on this /// forever (gold -> gold.gold -> ...); a flat-namespace catalog must list the top level only. ParentIgnoringEcho, + /// Flat-namespace catalog (Apache Polaris federated to AWS Glue) that rejects `parent` with + /// HTTP 400 instead of ignoring it, and holds a table in its only top-level namespace. + ParentRejecting, + /// Rejects the client credentials at the OAuth token endpoint. + OAuthTokenRejected, }; void writeJSON(Poco::Net::HTTPServerResponse & response, const std::string & body, Poco::Net::HTTPResponse::HTTPStatus status = Poco::Net::HTTPResponse::HTTP_OK) @@ -136,6 +141,14 @@ class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler { const std::string request_body(std::istreambuf_iterator(request.stream()), {}); ++token_requests; + if (shape == CatalogShape::OAuthTokenRejected) + { + writeError( + response, + Poco::Net::HTTPResponse::HTTP_UNAUTHORIZED, + R"({"error":"invalid_client","error_description":"Client secret does not match"})"); + return; + } /// Horizon secret-only credentials omit client_id; standard REST includes it. if (request_body.contains("client_secret=") && !request_body.contains("client_id=")) { @@ -165,6 +178,12 @@ class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler else if (shape == CatalogShape::ParentIgnoringEcho) /// Ignores `parent` and echoes the top-level namespace back for any parent. writeJSON(response, R"({"namespaces":[["gold"]]})"); + else if (shape == CatalogShape::ParentRejecting) + writeError( + response, + Poco::Net::HTTPResponse::HTTP_BAD_REQUEST, + R"({"error":{"message":"Malformed request: Glue dataCatalog does not support multipart namespace.",)" + R"("type":"BadRequestException","code":400}})"); else writeJSON(response, R"({"namespaces":[]})"); return; @@ -172,7 +191,7 @@ class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler if (path == "/v1/namespaces/namespace/tables") { - if (shape == CatalogShape::TopLevelTable) + if (shape == CatalogShape::TopLevelTable || shape == CatalogShape::ParentRejecting) writeJSON(response, R"({"identifiers":[{"name":"table_a"}]})"); else writeJSON(response, R"({"identifiers":[]})"); @@ -313,7 +332,7 @@ void expectThrowsCode(std::function fn, int expected_code) } } -bool restCatalogEmpty(CatalogShape shape) +bool restCatalogEmpty(CatalogShape shape, bool flat_namespaces = false) { RestCatalogTestServer server(shape); auto context = DB::Context::createCopy(getContext().context); @@ -327,11 +346,32 @@ bool restCatalogEmpty(CatalogShape shape) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + flat_namespaces, context); return catalog.empty(); } +DataLake::ICatalog::Namespaces restCatalogNamespaces(CatalogShape shape, bool flat_namespaces) +{ + RestCatalogTestServer server(shape); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + RestCatalog catalog( + "warehouse", + server.getUrl(), + /* catalog_credential */"", + /* auth_scope */"", + /* auth_header */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + flat_namespaces, + context); + + return catalog.getNamespaces(); +} + bool deltaSharingCatalogEmpty(CatalogShape shape) { RestCatalogTestServer server(shape); @@ -346,6 +386,7 @@ bool deltaSharingCatalogEmpty(CatalogShape shape) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, context); return catalog.empty(); @@ -377,6 +418,86 @@ TEST(RestCatalog, EmptyReturnsTrueWhenNoTablesExist) EXPECT_TRUE(restCatalogEmpty(CatalogShape::Empty)); } +TEST(RestCatalog, RejectedClientCredentialsReportTheOAuthError) +{ + RestCatalogTestServer server(CatalogShape::OAuthTokenRejected); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + try + { + RestCatalog catalog( + "warehouse", + server.getUrl(), + /* catalog_credential */"client-1:wrong-secret", + /* auth_scope */"PRINCIPAL_ROLE:ALL", + /* auth_header */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */true, + /* flat_namespaces */false, + context); + FAIL() << "expected the rejected client credentials to fail the catalog construction"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::DATALAKE_DATABASE_ERROR); + EXPECT_TRUE(e.message().contains("invalid_client")) << e.message(); + EXPECT_TRUE(e.message().contains("Client secret does not match")) << e.message(); + EXPECT_TRUE(e.message().contains("status 401")) << e.message(); + EXPECT_FALSE(e.message().contains("wrong-secret")) << e.message(); + } +} + +TEST(RestCatalog, ParentFilterRejectionReportsFlatNamespacesSetting) +{ + try + { + restCatalogNamespaces(CatalogShape::ParentRejecting, /* flat_namespaces */false); + FAIL() << "expected the rejected `parent` filter to fail the namespace listing"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::DATALAKE_DATABASE_ERROR); + EXPECT_TRUE(e.message().contains("flat_namespaces")) << e.message(); + EXPECT_TRUE(e.message().contains("HTTP status: 400")) << e.message(); + } +} + +TEST(RestCatalog, FlatNamespacesSettingSkipsSubNamespaceListing) +{ + EXPECT_FALSE(restCatalogEmpty(CatalogShape::ParentRejecting, /* flat_namespaces */true)); + EXPECT_EQ(restCatalogNamespaces(CatalogShape::ParentRejecting, /* flat_namespaces */true), ICatalog::Namespaces{"namespace"}); +} + +TEST(RestCatalog, FlatNamespacesSettingIgnoresEchoedParent) +{ + EXPECT_EQ(restCatalogNamespaces(CatalogShape::ParentIgnoringEcho, /* flat_namespaces */true), ICatalog::Namespaces{"gold"}); +} + +TEST(RestCatalog, OneLakeFlatNamespacesSettingSkipsSubNamespaceListing) +{ + RestCatalogTestServer server(CatalogShape::ParentRejecting); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + OneLakeCatalog catalog( + "warehouse", + server.getUrl(), + /* onelake_tenant_id */"tenant-1", + /* onelake_client_id */"", + /* onelake_client_secret */"", + /* bearer_token */"token-1", + /* refresh_token */"", + /* auth_scope */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + /* flat_namespaces */true, + context); + + EXPECT_FALSE(catalog.empty()); + EXPECT_EQ(catalog.getNamespaces(), ICatalog::Namespaces{"namespace"}); +} + TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) { RestCatalogTestServer server(CatalogShape::TopLevelTable); @@ -391,6 +512,7 @@ TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, context); auto existing = TableMetadata().withLocation(); @@ -427,6 +549,7 @@ TEST(RestCatalog, TryGetTableMetadataAuthErrorPropagates) /* auth_scope */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, context); TableMetadata metadata; @@ -458,6 +581,7 @@ TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, context); DB::SettingsChanges changes; @@ -479,6 +603,7 @@ TEST(RestCatalog, ApplySettingsChangesCredentialMode) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, context); EXPECT_EQ(catalog.getStateSnapshot()->client_id, "client-1"); @@ -520,6 +645,7 @@ TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) /* auth_header */"Authorization: Bearer token-1", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, context); DB::SettingsChanges changes; @@ -552,6 +678,7 @@ TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) /* auth_scope */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, context); const auto snapshot_before = catalog.getStateSnapshot(); @@ -612,6 +739,7 @@ TEST(RestCatalog, OneLakeRejectsMalformedBearerToken) /* auth_scope */ "", /* oauth_server_uri */ "", /* oauth_server_use_request_body */ false, + /* flat_namespaces */ false, context); }, DB::ErrorCodes::BAD_ARGUMENTS); @@ -652,6 +780,7 @@ TEST(RestCatalog, OneLakeRefreshTokenTransparentRenewal) /* auth_scope */"https://storage.azure.com/.default", /* oauth_server_uri */server.getUrl() + "/token", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, context); const auto requests_after_construction = server.tokenRequests(); @@ -693,6 +822,7 @@ TEST(RestCatalog, OneLakeRefreshTokenExpiredThrowsWithAlterHint) /* auth_scope */"https://storage.azure.com/.default", /* oauth_server_uri */server.getUrl() + "/token", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, context); /// ADD_FAILURE (rather than FAIL) does not return from the test, so the /// profile event check below is reached on every path; FAIL would make @@ -727,6 +857,7 @@ TEST(RestCatalog, OneLakeApplySettingsChangesRefreshMode) /* auth_scope */"https://storage.azure.com/.default", /* oauth_server_uri */server.getUrl() + "/token", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, context); DB::SettingsChanges changes; @@ -780,6 +911,7 @@ TEST(RestCatalog, HorizonCatalogAuthenticatesWithBarePAT) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, context); EXPECT_EQ(catalog.getCatalogType(), DB::DatabaseDataLakeCatalogType::ICEBERG_HORIZON); @@ -811,6 +943,7 @@ TEST(RestCatalog, HorizonCatalogRequiresCredentialOrAuthHeader) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, context); }, DB::ErrorCodes::BAD_ARGUMENTS); @@ -830,6 +963,7 @@ TEST(RestCatalog, HorizonApplySettingsChangesBarePAT) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, context); DB::SettingsChanges changes; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index b0093bde0cddd..42f4fcaa6d458 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -95,6 +95,11 @@ namespace CurrentMetrics namespace DB { +namespace FailPoints +{ +extern const char object_storage_source_pause_before_virtual_columns[]; +} + namespace ErrorCodes { extern const int CANNOT_COMPILE_REGEXP; @@ -665,6 +670,8 @@ Chunk StorageObjectStorageSource::generate() else if (object_metadata->is_size_known) object_size = object_metadata->size_bytes; + FailPointInjection::pauseFailPoint(FailPoints::object_storage_source_pause_before_virtual_columns); + VirtualColumnUtils::addRequestedFileLikeStorageVirtualsToChunk( chunk, read_from_format_info.requested_virtual_columns, diff --git a/tests/integration/test_storage_iceberg_with_spark/test_lifetime_sources_bug.py b/tests/integration/test_storage_iceberg_with_spark/test_lifetime_sources_bug.py new file mode 100644 index 0000000000000..cd538da005a64 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_lifetime_sources_bug.py @@ -0,0 +1,156 @@ +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + write_iceberg_from_df, + generate_data, + create_iceberg_table, + get_uuid_str, +) + +import threading +import time + + +FAILPOINT = "object_storage_source_pause_before_virtual_columns" + + +def uptime(instance): + # Uptime only ever grows while the server keeps running and drops back to nearly zero when it + # restarts, so comparing two samples tells a survived run from a crashed one. Deriving the start + # time as `now() - uptime()` instead would be off by a second whenever the two functions, both of + # them second-granular, are evaluated on different sides of a second boundary. `system.crash_log` + # is no good either: the table only appears once something has already crashed. + return int(instance.query("SELECT uptime()")) + + +@pytest.mark.parametrize("format_version", ["2"]) +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_source_must_not_outlive_storage(started_cluster_iceberg_with_spark, format_version, storage_type): + """Dropping a table must not pull the storage out from under a reading pipeline. + + `StorageObjectStorageSource::generate` resolves `storage_snapshot->storage.getStorageID()` for every + chunk, and `StorageSnapshot::storage` is a bare `const IStorage &`: the source keeps the snapshot + alive, not the storage behind it. If `DROP TABLE` destroys the storage while a pipeline thread sits + in `generate`, that call locks the mutex of a destroyed `IStorage` and glibc aborts the whole server. + Staging shows 597 crashes sharing exactly this stack, all of them signal 6 through + `IStorage::getStorageID` from `StorageObjectStorageSource::generate`, reported for single node + execution with parallel replicas. + + So the test parks a source inside `generate`, drops the table underneath it, and then lets it wake + up. Whether `DROP` waits for the reading pipeline is the crux: if it goes through while the source + is parked, the table lock is not being held for that path, and the wake-up touches freed memory. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = ( + "test_source_must_not_outlive_storage_" + + format_version + + "_" + + storage_type + + "_" + + get_uuid_str() + ) + + NUM_FILES = 8 + ROWS_PER_FILE = 100 + + write_iceberg_from_df( + spark, generate_data(spark, 0, ROWS_PER_FILE), TABLE_NAME, mode="overwrite", format_version=format_version + ) + for i in range(1, NUM_FILES): + write_iceberg_from_df( + spark, + generate_data(spark, i * 1000, i * 1000 + ROWS_PER_FILE), + TABLE_NAME, + mode="append", + format_version=format_version, + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + uptime_before = uptime(instance) + + # `_path` is what drags the storage into `generate`, and parallel replicas keep the reading pipeline + # on the initiator, which is the shape the crashes were reported for. + select_id = get_uuid_str() + select = ( + f"SELECT _path, count() FROM {TABLE_NAME} GROUP BY _path " + "SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, " + "parallel_replicas_local_plan = 1, cluster_for_parallel_replicas = 'cluster_simple'" + ) + + select_error = [] + drop_error = [] + drop_finished = threading.Event() + + def run_select(): + try: + instance.query(select, query_id=select_id) + except Exception as e: # the query may legitimately die with the table, the server may not + select_error.append(str(e)) + + def run_drop(): + try: + instance.query(f"DROP TABLE {TABLE_NAME} SYNC") + except Exception as e: + drop_error.append(str(e)) + finally: + drop_finished.set() + + select_thread = threading.Thread(target=run_select) + drop_thread = threading.Thread(target=run_drop) + + instance.query(f"SYSTEM ENABLE FAILPOINT {FAILPOINT}") + try: + select_thread.start() + + deadline = time.time() + 60 + running = 0 + while time.time() < deadline: + running = int(instance.query( + f"SELECT count() FROM system.processes WHERE query_id = '{select_id}'" + )) + if running: + break + time.sleep(0.2) + + assert running, "the query never started, so no source is parked in generate()" + + # `system.processes` reports the query from its start, before the pipeline has reached the + # failpoint, so give the source a moment to actually park there. + time.sleep(3) + + drop_thread.start() + # A DROP that returns while the source is parked means nothing held the table for the reading + # pipeline. Recorded rather than asserted on: the outcome that matters is whether the server + # survives the wake-up below. + dropped_under_reader = drop_finished.wait(timeout=10) + finally: + # Releases the parked source, which then reaches `storage_snapshot->storage.getStorageID()`. + instance.query(f"SYSTEM DISABLE FAILPOINT {FAILPOINT}") + + select_thread.join(timeout=180) + drop_thread.join(timeout=180) + assert not select_thread.is_alive(), "the source never woke up after the failpoint was released" + assert not drop_thread.is_alive(), "DROP TABLE never returned" + + assert instance.query("SELECT 1").strip() == "1" + assert uptime(instance) >= uptime_before, ( + "the server restarted during the test: DROP TABLE destroyed the storage while a source was " + "parked in StorageObjectStorageSource::generate, and the wake-up locked the mutex of a " + "destroyed IStorage" + + ( + " (DROP TABLE returned while the source was still parked, so the reading pipeline was not" + " holding the table)" + if dropped_under_reader + else "" + ) + ) diff --git a/tests/integration/test_storage_iceberg_with_spark/test_variant_type.py b/tests/integration/test_storage_iceberg_with_spark/test_variant_type.py new file mode 100644 index 0000000000000..9b85b2cb9dcb6 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_variant_type.py @@ -0,0 +1,75 @@ +import pytest + +from helpers.iceberg_utils import ( + create_iceberg_table, + default_upload_directory, + get_creation_expression, + get_uuid_str, +) + +from helpers.test_tools import TSV + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_variant_type(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + if int(spark.version.split(".")[0]) < 4: + pytest.skip(f"the VARIANT type needs Spark 4.0, the runner has {spark.version}") + TABLE_NAME = "test_variant_type_" + storage_type + "_" + get_uuid_str() + + # `variant` was introduced in Iceberg format version 3. Iceberg writes it as an unshredded + # group of two plain BYTE_ARRAY leaves annotated with the `VARIANT` logical type: + # + # optional group v (Variant(1)) { + # required binary metadata; + # required binary value; + # } + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id INT, v VARIANT) + USING iceberg + TBLPROPERTIES ('format-version'='3') + """ + ) + spark.sql( + f""" + INSERT INTO {TABLE_NAME} VALUES + (1, cast(100000 as variant)), + (2, cast('hello' as variant)) + """ + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_with_spark, + format_version=3, + ) + + table_function_expr = get_creation_expression( + storage_type, + TABLE_NAME, + started_cluster_iceberg_with_spark, + table_function=True, + format_version=3, + ) + + assert instance.query(f"DESCRIBE {table_function_expr} FORMAT TSV") == TSV( + [ + ["id", "Nullable(Int32)"], + ["v", "Dynamic"], + ] + ) + + assert instance.query( + f"SELECT id, dynamicType(v), v FROM {table_function_expr} ORDER BY id FORMAT TSV" + ).strip() == ("1\tInt32\t100000\n" "2\tString\thello") From f5f297764f0a0f94eb9b9a722327150507d2bd33 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 09:05:32 +0000 Subject: [PATCH 012/296] Backport #119999 to 26.8: Mirror the MinIO integration-test images into the clickhouse/ namespace --- .../docker_compose_iceberg_rest_catalog_with_trino.yml | 8 ++++++-- tests/integration/compose/docker_compose_minio.yml | 4 +++- tests/integration/compose/mirror-images.sh | 9 +++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/integration/compose/docker_compose_iceberg_rest_catalog_with_trino.yml b/tests/integration/compose/docker_compose_iceberg_rest_catalog_with_trino.yml index 0cb6d25c3abfe..49f4dea0800bc 100644 --- a/tests/integration/compose/docker_compose_iceberg_rest_catalog_with_trino.yml +++ b/tests/integration/compose/docker_compose_iceberg_rest_catalog_with_trino.yml @@ -20,7 +20,9 @@ services: stop_grace_period: 5s cpus: 3 minio: - image: minio/minio:RELEASE.2024-07-31T05-46-26Z + # Mirror of quay.io/minio/minio:RELEASE.2024-07-31T05-46-26Z (see compose/mirror-images.sh). + # The minio/minio Docker Hub repository was deleted upstream; quay.io is not behind the proxy. + image: clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z environment: - MINIO_ROOT_USER=minio - MINIO_ROOT_PASSWORD=ClickHouse_Minio_P@ssw0rd @@ -38,7 +40,9 @@ services: mc: depends_on: - minio - image: minio/mc:RELEASE.2025-04-16T18-13-26Z + # Mirror of quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z (see compose/mirror-images.sh). + # The minio/mc Docker Hub repository was deleted upstream; quay.io is not behind the proxy. + image: clickhouse/minio-mc:RELEASE.2025-04-16T18-13-26Z environment: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=ClickHouse_Minio_P@ssw0rd diff --git a/tests/integration/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml index fafb45f538393..79d3cd27ade5d 100644 --- a/tests/integration/compose/docker_compose_minio.yml +++ b/tests/integration/compose/docker_compose_minio.yml @@ -1,6 +1,8 @@ services: minio1: - image: minio/minio:RELEASE.2024-09-13T20-26-02Z + # Mirror of quay.io/minio/minio:RELEASE.2024-09-13T20-26-02Z (see compose/mirror-images.sh). + # The minio/minio Docker Hub repository was deleted upstream; quay.io is not behind the proxy. + image: clickhouse/minio-minio:RELEASE.2024-09-13T20-26-02Z volumes: - data1-1:/data1 - ${MINIO_CERTS_DIR:-}:/certs diff --git a/tests/integration/compose/mirror-images.sh b/tests/integration/compose/mirror-images.sh index 839fe27ac603a..cfe83ab43a543 100755 --- a/tests/integration/compose/mirror-images.sh +++ b/tests/integration/compose/mirror-images.sh @@ -4,10 +4,12 @@ # Why: CI runners pull Docker Hub images through the dockerhub-proxy cache # (registry:2 + nginx, backed by S3 — see tests/ci/terraform/dockerhub-proxy.md). # That proxy only fronts Docker Hub. Images hosted on other registries -# (mcr.microsoft.com, ghcr.io) bypass the proxy and are pulled directly, so CI is -# exposed to those registries' anonymous rate limits (e.g. mcr.microsoft.com +# (mcr.microsoft.com, ghcr.io, quay.io) bypass the proxy and are pulled directly, so +# CI is exposed to those registries' anonymous rate limits (e.g. mcr.microsoft.com # returns HTTP 429 "toomanyrequests" under load). Re-hosting them under # clickhouse/ routes the pulls back through the proxy + S3 cache. +# A mirror also outlives the upstream repository: minio/minio and minio/mc were +# deleted from Docker Hub, and quay.io is the only remaining source of those tags. # # Usage: log in to Docker Hub with an account that can push to the clickhouse org, # then run this script. It is idempotent — re-run it to add images or bump versions. @@ -22,6 +24,9 @@ IMAGES=( "ghcr.io/ytsaurus/local:stable-24.2 clickhouse/ytsaurus-local:stable-24.2" "ghcr.io/letsencrypt/pebble:2.9.0 clickhouse/letsencrypt-pebble:2.9.0" "ghcr.io/letsencrypt/pebble-challtestsrv:2.9.0 clickhouse/letsencrypt-pebble-challtestsrv:2.9.0" + "quay.io/minio/minio:RELEASE.2024-09-13T20-26-02Z clickhouse/minio-minio:RELEASE.2024-09-13T20-26-02Z" + "quay.io/minio/minio:RELEASE.2024-07-31T05-46-26Z clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z" + "quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z clickhouse/minio-mc:RELEASE.2025-04-16T18-13-26Z" ) for entry in "${IMAGES[@]}"; do From d530d2513b6e4f7261a6fccc25c7dab23c98de31 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 10:11:29 +0000 Subject: [PATCH 013/296] Backport #120433 to 26.8: Preserve static S3 auth settings on client refresh --- .../ObjectStorages/S3/S3ObjectStorage.cpp | 7 +- .../test_s3_client_refresh/__init__.py | 0 .../test_s3_client_refresh/configs/s3.xml | 21 ++++ .../test_s3_client_refresh/test.py | 95 +++++++++++++++++++ 4 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_s3_client_refresh/__init__.py create mode 100644 tests/integration/test_s3_client_refresh/configs/s3.xml create mode 100644 tests/integration/test_s3_client_refresh/test.py diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 90f374c23aa1a..b441f36783f0c 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -710,18 +710,21 @@ void S3ObjectStorage::applyNewSettings( auto modified_settings = std::make_unique(*s3_settings.get()); + /// Static configurations keep their resolved authentication settings when a session change rebuilds the client. auto apply_endpoint_settings = [&] { if (auto endpoint_settings = context->getStorageS3Settings().getSettings(uri.uri.toString(), context->getUserName())) { - modified_settings->auth_settings.updateIfChanged(endpoint_settings->auth_settings); + if (options.allow_client_change) + modified_settings->auth_settings.updateIfChanged(endpoint_settings->auth_settings); modified_settings->request_settings.updateIfChanged(endpoint_settings->request_settings); } }; auto apply_config_settings = [&] { - modified_settings->auth_settings.updateIfChanged(settings_from_config->auth_settings); + if (options.allow_client_change) + modified_settings->auth_settings.updateIfChanged(settings_from_config->auth_settings); modified_settings->request_settings.updateIfChanged(settings_from_config->request_settings); }; diff --git a/tests/integration/test_s3_client_refresh/__init__.py b/tests/integration/test_s3_client_refresh/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_s3_client_refresh/configs/s3.xml b/tests/integration/test_s3_client_refresh/configs/s3.xml new file mode 100644 index 0000000000000..dec0558850b1b --- /dev/null +++ b/tests/integration/test_s3_client_refresh/configs/s3.xml @@ -0,0 +1,21 @@ + + + 0 + + http://minio1:9001/root/refresh/endpoint/ + 0 + + + http://minio1:9001/root/refresh/keys/ + invalid + invalid + + + + + http://minio1:9001/root/refresh/ + 1 + TSV + + + diff --git a/tests/integration/test_s3_client_refresh/test.py b/tests/integration/test_s3_client_refresh/test.py new file mode 100644 index 0000000000000..b9b6a4dabe53c --- /dev/null +++ b/tests/integration/test_s3_client_refresh/test.py @@ -0,0 +1,95 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_access_key, minio_secret_key + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + with_minio=True, + main_configs=["configs/s3.xml"], + env_variables={ + "AWS_ACCESS_KEY_ID": minio_access_key, + "AWS_SECRET_ACCESS_KEY": minio_secret_key, + "AWS_EC2_METADATA_DISABLED": "true", + }, +) + +TRUSTED = {"s3_allow_server_credentials_in_user_queries": 1} +RESTRICTED = {"s3_allow_server_credentials_in_user_queries": 0} + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield + finally: + cluster.shutdown() + + +@pytest.mark.parametrize("scope", ["global", "endpoint"]) +@pytest.mark.parametrize("partitioned", [False, True]) +def test_named_collection_credentials_survive_restricted_read(scope, partitioned): + # Refresh must preserve the collection's ambient-credential opt-in over server defaults, + # while still refusing restricted reads. Partitioned tables exercise the system-log write path. + table = f"refresh_{scope}_{int(partitioned)}" + filename = f"{scope}/{table}/data" + if partitioned: + filename += "_{_partition_id}" + filename += ".tsv" + partition_by = "PARTITION BY x" if partitioned else "" + node.query( + f"CREATE TABLE {table} (x UInt8) " + f"ENGINE = S3(refresh_env, filename = '{filename}') {partition_by}", + settings=TRUSTED, + ) + try: + for value in (1, 2): + node.query( + f"INSERT INTO {table} VALUES ({value})", + settings={**TRUSTED, "s3_truncate_on_insert": 1}, + ) + # Read back with explicit keys so verification cannot refresh the table's client. + object_name = filename.replace("{_partition_id}", str(value)) + assert ( + node.query( + f"SELECT * FROM s3('http://minio1:9001/root/refresh/{object_name}', " + f"'{minio_access_key}', '{minio_secret_key}', 'TSV', 'x UInt8')", + settings=RESTRICTED, + ) + == f"{value}\n" + ) + + error = node.query_and_get_error( + f"SELECT * FROM {table}", settings=RESTRICTED + ) + if partitioned: + assert "ACCESS_DENIED" in error or "NOT_IMPLEMENTED" in error, error + else: + assert "ACCESS_DENIED" in error, error + finally: + node.query(f"DROP TABLE {table} SYNC") + + +@pytest.mark.parametrize("named_collection", [False, True]) +def test_explicit_keys_survive_client_refresh(named_collection): + # Static keys must also survive a session change, even with conflicting endpoint credentials. + table = f"refresh_keys_{int(named_collection)}" + url = f"http://minio1:9001/root/refresh/keys/{table}.tsv" + if named_collection: + arguments = ( + f"refresh_env, url = '{url}', access_key_id = '{minio_access_key}', " + f"secret_access_key = '{minio_secret_key}'" + ) + else: + arguments = f"'{url}', '{minio_access_key}', '{minio_secret_key}', 'TSV'" + node.query( + f"CREATE TABLE {table} (x UInt8) ENGINE = S3({arguments})", settings=TRUSTED + ) + try: + node.query(f"INSERT INTO {table} VALUES (7)", settings=TRUSTED) + for settings in (RESTRICTED, TRUSTED, RESTRICTED): + assert node.query(f"SELECT * FROM {table}", settings=settings) == "7\n" + finally: + node.query(f"DROP TABLE {table} SYNC") From 78b528c998fd621d0a83f67ad8da42e859ac5c62 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 10:13:42 +0000 Subject: [PATCH 014/296] Backport #116647 to 26.8: Check access to the source table when resolving the structure of `mergeTreeCodecBlockCounts` --- .../StorageMergeTreeCodecBlockCounts.cpp | 55 +++-- .../StorageMergeTreeCodecBlockCounts.h | 21 +- ...TableFunctionMergeTreeCodecBlockCounts.cpp | 42 ++-- ...mergeTreeCodecBlockCounts_access.reference | 52 +++++ .../05059_mergeTreeCodecBlockCounts_access.sh | 205 ++++++++++++++++++ 5 files changed, 341 insertions(+), 34 deletions(-) create mode 100644 tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.reference create mode 100755 tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.sh diff --git a/src/Storages/StorageMergeTreeCodecBlockCounts.cpp b/src/Storages/StorageMergeTreeCodecBlockCounts.cpp index b15f13cf0a240..a02ea2d0473f3 100644 --- a/src/Storages/StorageMergeTreeCodecBlockCounts.cpp +++ b/src/Storages/StorageMergeTreeCodecBlockCounts.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -296,25 +298,39 @@ void ReadFromMergeTreeCodecBlockCounts::initializePipeline(QueryPipelineBuilder } StorageMergeTreeCodecBlockCounts::StorageMergeTreeCodecBlockCounts( - const StorageID & table_id_, StoragePtr source_table_, const ColumnsDescription & columns_) + const StorageID & table_id_, StorageID source_table_id_, const ColumnsDescription & columns_) : IStorage(table_id_) - , source_table(std::move(source_table_)) + , source_table_id(std::move(source_table_id_)) { - const auto * merge_tree = dynamic_cast(source_table.get()); - if (!merge_tree) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeCodecBlockCounts expected MergeTree table, got: {}", source_table->getName()); - - /// `system.parts_columns` lists patch parts, so this function does too. - data_parts = merge_tree->getDataPartsVectorForInternalUsage( - {MergeTreeData::DataPartState::Active}, {MergeTreeData::DataPartKind::Regular, MergeTreeData::DataPartKind::Patch}); - std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); - StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); setInMemoryMetadata(storage_metadata); } +void StorageMergeTreeCodecBlockCounts::checkSourceTableAccess(const StoragePtr & source_table, const ContextPtr & context) +{ + const auto source_metadata = source_table->getInMemoryMetadataPtr(context, false); + context->checkAccess(AccessType::SELECT, source_table->getStorageID(), source_metadata->getColumns().getNamesOfPhysical()); +} + +StoragePtr StorageMergeTreeCodecBlockCounts::resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context) +{ + /// `SHOW TABLES` is the privilege that governs whether the table's existence may be learned, and it is implied + /// by a grant on any single column of it, so this only adds a tier below the `SELECT` check on every column + /// that follows the resolution. `SHOW COLUMNS`, which `DESCRIBE` of the source table requires, is not implied + /// by column-level grants, so it would reject a user who holds `SELECT` on every column separately. + context->checkAccess(AccessType::SHOW_TABLES, source_table_id); + + auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + checkSourceTableAccess(source_table, context); + + if (!dynamic_cast(source_table.get())) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Table function mergeTreeCodecBlockCounts expected MergeTree table, got: {}", source_table->getName()); + + return source_table; +} + void StorageMergeTreeCodecBlockCounts::read( QueryPlan & query_plan, const Names & column_names, @@ -327,9 +343,18 @@ void StorageMergeTreeCodecBlockCounts::read( { storage_snapshot->check(column_names); - const auto source_metadata = source_table->getInMemoryMetadataPtr(context, false); + /// Under the reader's context, see the constructor. + auto source_table = resolveSourceTable(source_table_id, context); const auto source_storage_id = source_table->getStorageID(); - context->checkAccess(AccessType::SELECT, source_storage_id, source_metadata->getColumns().getNamesOfPhysical()); + + /// A cast to a base class, so not `assert_cast`, which asserts the exact type and would reject every + /// `MergeTree` table. `resolveSourceTable` has already rejected a source table that is not a `MergeTree`. + const auto & merge_tree = dynamic_cast(*source_table); + + /// `system.parts_columns` lists patch parts, so this function does too. + auto data_parts = merge_tree.getDataPartsVectorForInternalUsage( + {MergeTreeData::DataPartState::Active}, {MergeTreeData::DataPartKind::Regular, MergeTreeData::DataPartKind::Patch}); + std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); auto sample_block = std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names)); @@ -353,7 +378,7 @@ void StorageMergeTreeCodecBlockCounts::read( query_plan.addStep( std::make_unique( - column_names, query_info, storage_snapshot, std::move(context), std::move(sample_block), data_parts, source_storage_id)); + column_names, query_info, storage_snapshot, std::move(context), std::move(sample_block), std::move(data_parts), source_storage_id)); } } diff --git a/src/Storages/StorageMergeTreeCodecBlockCounts.h b/src/Storages/StorageMergeTreeCodecBlockCounts.h index 0a07928f9cc2a..e8852112a0aef 100644 --- a/src/Storages/StorageMergeTreeCodecBlockCounts.h +++ b/src/Storages/StorageMergeTreeCodecBlockCounts.h @@ -1,7 +1,7 @@ #pragma once +#include #include -#include namespace DB { @@ -11,10 +11,24 @@ namespace DB class StorageMergeTreeCodecBlockCounts final : public IStorage { public: - StorageMergeTreeCodecBlockCounts(const StorageID & table_id_, StoragePtr source_table_, const ColumnsDescription & columns_); + /// Holds only the name of the source table. It is resolved and checked on every read, under the context of the + /// user who reads, so that the checks run where the data is disclosed rather than where the storage is built, + /// and so that the storage keeps no reference to the source table between reads. + StorageMergeTreeCodecBlockCounts(const StorageID & table_id_, StorageID source_table_id_, const ColumnsDescription & columns_); std::string getName() const override { return "MergeTreeCodecBlockCounts"; } + /// Every column of this function is derived from the source table's data, so reading any of them requires + /// `SELECT` on all of the source table's columns. Called both when the function's structure is resolved and + /// when it is read, so that resolving the structure cannot reveal anything about a table the user cannot select from. + static void checkSourceTableAccess(const StoragePtr & source_table, const ContextPtr & context); + + /// Resolves the source table for `context`'s user, in the order that discloses nothing the user may not learn: + /// `SHOW TABLES` on the name, before the catalog is consulted, so that an inaccessible table and a missing one + /// answer alike; then `SELECT` on every column, before the engine is examined, so that a user without it cannot + /// learn the engine from the `BAD_ARGUMENTS` that rejects a table that is not a `MergeTree`. + static StoragePtr resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context); + void read( QueryPlan & query_plan, const Names & column_names, @@ -26,8 +40,7 @@ class StorageMergeTreeCodecBlockCounts final : public IStorage size_t num_streams) override; private: - StoragePtr source_table; - MergeTreeData::DataPartsVector data_parts; + StorageID source_table_id; }; } diff --git a/src/TableFunctions/TableFunctionMergeTreeCodecBlockCounts.cpp b/src/TableFunctions/TableFunctionMergeTreeCodecBlockCounts.cpp index 854115b42309d..5d87fb3ebac8d 100644 --- a/src/TableFunctions/TableFunctionMergeTreeCodecBlockCounts.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeCodecBlockCounts.cpp @@ -4,9 +4,8 @@ #include #include #include -#include +#include #include -#include #include #include #include @@ -18,7 +17,6 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; } @@ -28,10 +26,19 @@ class TableFunctionMergeTreeCodecBlockCounts : public ITableFunction static constexpr auto name = "mergeTreeCodecBlockCounts"; std::string getName() const override { return name; } + /// Refused in line with the other `MergeTree` introspection table functions. This storage holds only the source + /// table's name and resolves it on every read, so it pins nothing, but the persisted form has no uses and is + /// refused for all of them alike, so that none of them has to stay correct while outliving its query. + bool canBeUsedToCreateTable() const override { return false; } + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; private: + /// The fixed structure. It is not derived from the source table, but the table function does not declare it as + /// static: resolving it is a read of the source table and has to check access to it, see `getActualTableStructure`. + static ColumnsDescription getColumns(); + StoragePtr executeImpl( const ASTPtr & ast_function, ContextPtr context, @@ -69,13 +76,15 @@ void TableFunctionMergeTreeCodecBlockCounts::parseArguments(const ASTPtr & ast_f ColumnsDescription TableFunctionMergeTreeCodecBlockCounts::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); - - const auto * merge_tree = dynamic_cast(source_table.get()); - if (!merge_tree) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Table function {} expected MergeTree table, got: {}", getName(), source_table->getName()); + /// The structure is fixed, so nothing in it depends on the source table. The table is still resolved here, because + /// resolving the structure is a read of it and needs the same access as reading it: this is what `DESCRIBE` + /// goes through, under the context of the user who asks. + StorageMergeTreeCodecBlockCounts::resolveSourceTable(source_table_id, context); + return getColumns(); +} +ColumnsDescription TableFunctionMergeTreeCodecBlockCounts::getColumns() +{ auto codec_map = std::make_shared(std::make_shared(), std::make_shared()); ColumnsDescription columns; @@ -99,16 +108,17 @@ ColumnsDescription TableFunctionMergeTreeCodecBlockCounts::getActualTableStructu StoragePtr TableFunctionMergeTreeCodecBlockCounts::executeImpl( const ASTPtr & /*ast_function*/, - ContextPtr context, + ContextPtr /*context*/, const std::string & table_name, ColumnsDescription /*cached_columns*/, - bool is_insert_query) const + bool /*is_insert_query*/) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); - auto columns = getActualTableStructure(context, is_insert_query); - + /// Deliberately does not resolve the source table. Building the storage discloses nothing, because its structure + /// is fixed, and the storage may be built and read under different contexts (`EXPLAIN` builds it without reading). + /// The source is resolved and checked by `StorageMergeTreeCodecBlockCounts::read`, on the path that reads its data, + /// under the context of the user who reads. StorageID storage_id(getDatabaseName(), table_name); - auto res = std::make_shared(std::move(storage_id), std::move(source_table), std::move(columns)); + auto res = std::make_shared(std::move(storage_id), source_table_id, getColumns()); res->startup(); return res; @@ -125,6 +135,8 @@ Selecting `codec_block_counts` reads `.bin` data files, not just metadata. The o Parts that do not record their substreams in `columns_substreams.txt` are not listed. +Every reported value is derived from the table's data, so reading any column of the result requires the `SELECT` privilege on all columns of the table. A grant that covers only some of the columns is not enough. The privilege is also required to resolve the structure of the function, e.g. by `DESCRIBE`. A user who is not allowed to see the table at all, that is, one without the `SHOW TABLES` privilege on it, gets `ACCESS_DENIED` whether or not it exists, so the function does not tell such a user which tables exist. + If a row policy applies to the table for the current user, reading `codec_block_counts` throws `ACCESS_DENIED`, because the counts would cover rows the policy hides. The other columns stay readable, `system.parts_columns` reports them regardless of row policies. ## Syntax {#syntax} diff --git a/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.reference b/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.reference new file mode 100644 index 0000000000000..b0fe7054c2e51 --- /dev/null +++ b/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.reference @@ -0,0 +1,52 @@ +Without SELECT on the source table +ACCESS_DENIED +Without SELECT on the source table, structure only +ACCESS_DENIED +With SELECT on a single column of the source table +ACCESS_DENIED +ACCESS_DENIED +With SELECT on every column of the source table +a ['LZ4'] +b ['LZ4'] +part_name +column +substream +data_compressed_bytes +data_uncompressed_bytes +codec_block_counts +Non-MergeTree source table, without SELECT on it +ACCESS_DENIED +Non-MergeTree source table, with SELECT on it +BAD_ARGUMENTS +Hidden source table, before it is resolved +ACCESS_DENIED +Missing source table +ACCESS_DENIED +Missing source table, for a user who can see the database +UNKNOWN_TABLE +Hidden source table, before it is resolved, on the read path +ACCESS_DENIED +Missing source table, on the read path +ACCESS_DENIED +Missing source table, on the read path, for a user who can see the database +UNKNOWN_TABLE +EXPLAIN QUERY TREE is the same for a readable, a hidden and a missing source table +EXPLAIN SYNTAX is the same for a readable, a hidden and a missing source table +EXPLAIN PLAN of a hidden source table +ACCESS_DENIED +EXPLAIN PIPELINE of a non-MergeTree source table +BAD_ARGUMENTS +Nested in remote(...), with SELECT on a single column of the source table +grant SELECT +ACCESS_DENIED +Nested in a Remote table engine, with SELECT on a single column of the source table +grant SELECT +ACCESS_DENIED +Nested in remote(...), over a hidden source table +grant SHOW TABLES +ACCESS_DENIED +Nested in a Remote table engine, over a hidden source table +grant SHOW TABLES +ACCESS_DENIED +Number of tables the refused carriers left behind +0 diff --git a/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.sh b/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.sh new file mode 100755 index 0000000000000..fcebc1a3d35e1 --- /dev/null +++ b/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every column of `mergeTreeCodecBlockCounts` is derived from the source table's data, so reading any of +# them requires `SELECT` on all of its columns. Resolving the structure of the function requires the same. + +username="user_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +${CLICKHOUSE_CLIENT} -m --query " + DROP USER IF EXISTS ${username}; + DROP TABLE IF EXISTS t_codec_access; + DROP TABLE IF EXISTS t_codec_access_log; + DROP TABLE IF EXISTS t_codec_access_hidden; + DROP TABLE IF EXISTS t_codec_access_partial; + DROP TABLE IF EXISTS t_codec_access_dst; + + -- Explicit codecs, CI randomises the server-level default compression codec. + CREATE TABLE t_codec_access (a UInt64 CODEC(LZ4), b UInt64 CODEC(LZ4)) + ENGINE = MergeTree ORDER BY tuple() + SETTINGS min_bytes_for_wide_part = 0; + + INSERT INTO t_codec_access SELECT number, number FROM numbers(1000); + + CREATE TABLE t_codec_access_log (a UInt64) ENGINE = Log; + + -- Never granted to the test user, so it stays invisible to it. + CREATE TABLE t_codec_access_hidden (a UInt64) ENGINE = MergeTree ORDER BY tuple(); + + CREATE USER ${username} NOT IDENTIFIED; +" + +echo "Without SELECT on the source table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access);" 2>&1 | + grep -o "ACCESS_DENIED" | uniq + +echo "Without SELECT on the source table, structure only" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access);" 2>&1 | + grep -o "ACCESS_DENIED" | uniq + +echo "With SELECT on a single column of the source table" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT(a) ON t_codec_access TO ${username};" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access);" 2>&1 | + grep -o "ACCESS_DENIED" | uniq +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access);" 2>&1 | + grep -o "ACCESS_DENIED" | uniq + +echo "With SELECT on every column of the source table" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT(b) ON t_codec_access TO ${username};" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT DISTINCT column, mapKeys(codec_block_counts) FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access) ORDER BY column;" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access) FORMAT TSV" | cut -f 1 + +# The engine of the source table is disclosed by `SHOW CREATE TABLE`, which requires `SHOW COLUMNS` on it, so +# it is not something the user below is allowed to learn: at this point it holds neither that privilege nor +# `SELECT` on the table. The access check therefore has to run before the check that rejects a table of another +# engine with `BAD_ARGUMENTS`. + +echo "Non-MergeTree source table, without SELECT on it" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_log);" 2>&1 | + grep -o "ACCESS_DENIED\|BAD_ARGUMENTS" | uniq + +echo "Non-MergeTree source table, with SELECT on it" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON t_codec_access_log TO ${username};" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_log);" 2>&1 | + grep -o "ACCESS_DENIED\|BAD_ARGUMENTS" | uniq + +# Which tables exist is not something a user without any privilege on them is allowed to learn, so the check +# on the name has to run before the source table is resolved: an inaccessible table and a missing one answer alike. + +echo "Hidden source table, before it is resolved" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "Missing source table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_missing);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "Missing source table, for a user who can see the database" +${CLICKHOUSE_CLIENT} --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_missing);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +# The same for an ordinary read: the check on the name runs in `StorageMergeTreeCodecBlockCounts::read` as +# well, not only when the structure is resolved, so a plain `SELECT` is not an existence oracle either. + +echo "Hidden source table, before it is resolved, on the read path" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "Missing source table, on the read path" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_missing);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "Missing source table, on the read path, for a user who can see the database" +${CLICKHOUSE_CLIENT} --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_missing);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +# Analysis-only entrypoints (`EXPLAIN QUERY TREE`, `EXPLAIN SYNTAX`) resolve the table function without reading +# it, so the source table is not resolved on that path and no check runs there. That discloses nothing: the +# structure of this function is a fixed constant and nothing on that path consults the catalog, so the answer is +# the same whether the source table is readable, hidden, or missing. Pinned below, one arm per entrypoint. + +# `EXPLAIN QUERY TREE` exists only with the analyzer, and a configuration that turns it off rejects the +# statement with a message that names the source table - which is what the arm below compares. The setting +# is pinned so that the comparison stays about the disclosure and not about which rejection arrived. +explain_as_user() { + ${CLICKHOUSE_CLIENT} --user="${username}" --enable_analyzer 1 --query \ + "EXPLAIN $1 SELECT * FROM mergeTreeCodecBlockCounts(currentDatabase(), $2);" 2>&1 | sed "s/$2/SOURCE/g" +} + +for kind in "QUERY TREE" "SYNTAX"; do + readable=$(explain_as_user "${kind}" t_codec_access) + hidden=$(explain_as_user "${kind}" t_codec_access_hidden) + missing=$(explain_as_user "${kind}" t_codec_access_missing) + if [ "${readable}" = "${hidden}" ] && [ "${hidden}" = "${missing}" ]; then + echo "EXPLAIN ${kind} is the same for a readable, a hidden and a missing source table" + else + echo "EXPLAIN ${kind} tells a readable, a hidden and a missing source table apart" + fi +done + +# `EXPLAIN PLAN` and `EXPLAIN PIPELINE` do build the read plan, so they go through the checks. + +echo "EXPLAIN PLAN of a hidden source table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "EXPLAIN PLAN SELECT * FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "EXPLAIN PIPELINE of a non-MergeTree source table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "EXPLAIN PIPELINE SELECT * FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_log);" 2>&1 | + grep -o "ACCESS_DENIED\|BAD_ARGUMENTS" | uniq + +# A table function that `canBeUsedToCreateTable` refuses for `CREATE TABLE ... AS f(...)` can still reach a +# persisted definition nested in an argument of another one: both `remote(..., f(...))` and +# `ENGINE = Remote(..., f(...))` keep it in `remote_table_function_ptr`. When the cluster has a local shard, +# both forms resolve the nested function under the creating user's context, through +# `getStructureOfRemoteTableInShard` -> `getActualTableStructureWithAccess`, which is the seam this change adds +# the check to. So the carrier is closed by the check on the source table rather than by the veto, and the arms +# below pin which privilege refuses it, one per tier and one per form. Everything the carrier itself needs is +# granted first, so that what refuses is the source table and not the carrier; `CREATE TABLE` is granted on the +# destination name alone, because a grant on the database would imply `SHOW TABLES` on the hidden table too. + +${CLICKHOUSE_CLIENT} -m --query " + CREATE TABLE t_codec_access_partial (a UInt64 CODEC(LZ4), b UInt64 CODEC(LZ4)) + ENGINE = MergeTree ORDER BY tuple() + SETTINGS min_bytes_for_wide_part = 0; + + INSERT INTO t_codec_access_partial SELECT number, number FROM numbers(1000); + + GRANT CREATE TABLE, DROP TABLE ON t_codec_access_dst TO ${username}; + GRANT READ, WRITE ON REMOTE TO ${username}; + GRANT TABLE ENGINE ON Remote TO ${username}; + GRANT TABLE ENGINE ON Distributed TO ${username}; + GRANT SELECT(a) ON t_codec_access_partial TO ${username}; +" + +# Reports the privilege the carrier demanded and the error code, one line each, from a single attempt. +carrier_as_user() { + local out + out=$(${CLICKHOUSE_CLIENT} --user="${username}" --query "$1" 2>&1) + echo "${out}" | grep -o "grant SHOW TABLES\|grant SELECT" | uniq + echo "${out}" | grep -o "ACCESS_DENIED\|BAD_ARGUMENTS" | uniq +} + +echo "Nested in remote(...), with SELECT on a single column of the source table" +carrier_as_user "CREATE TABLE t_codec_access_dst AS remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_partial));" + +echo "Nested in a Remote table engine, with SELECT on a single column of the source table" +carrier_as_user "CREATE TABLE t_codec_access_dst ENGINE = Remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_partial));" + +# The check on the name runs on this path too, so a carrier is not an existence oracle either. + +echo "Nested in remote(...), over a hidden source table" +carrier_as_user "CREATE TABLE t_codec_access_dst AS remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden));" + +echo "Nested in a Remote table engine, over a hidden source table" +carrier_as_user "CREATE TABLE t_codec_access_dst ENGINE = Remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden));" + +echo "Number of tables the refused carriers left behind" +${CLICKHOUSE_CLIENT} --query \ + "SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 't_codec_access_dst';" + +${CLICKHOUSE_CLIENT} -m --query " + DROP USER ${username}; + DROP TABLE t_codec_access; + DROP TABLE t_codec_access_log; + DROP TABLE t_codec_access_hidden; + DROP TABLE t_codec_access_partial; +" From 5f9d2b72f11e7072efeac8f14c7e2893a9ec8d34 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 10:41:24 +0000 Subject: [PATCH 015/296] Backport #120356 to 26.8: Reject path traversal in file names from part checksums --- .../MergeTree/MergeTreeDataPartChecksum.cpp | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 33b16fa8d1a1c..e8f417f8f9dde 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -112,6 +112,34 @@ UInt64 MergeTreeDataPartChecksums::getTotalSizeUncompressedOnDisk() const return res; } +namespace +{ + +/// File names in checksums.txt are used verbatim to build paths inside the part directory +/// (e.g. when checking sizes or removing the part), so they must not escape it. +void assertFileNameIsRelativeAndContained(const String & name) +{ + if (name.empty()) + throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, "Empty file name in checksums of data part"); + + if (name.starts_with('/')) + throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, "Absolute file name '{}' in checksums of data part", name); + + /// A NUL byte would be kept inside a single path component here, but the local disk layer + /// passes the joined path to C APIs which truncate at the first NUL, so "..\0/x" would act as "..". + if (name.contains('\0')) + throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, "File name '{}' in checksums of data part contains a NUL byte", name); + + for (const auto & component : std::filesystem::path(name)) + { + if (component == "." || component == "..") + throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, + "File name '{}' in checksums of data part contains '{}' path component", name, component.string()); + } +} + +} + bool MergeTreeDataPartChecksums::read(ReadBuffer & in, size_t format_version) { switch (format_version) @@ -155,6 +183,7 @@ bool MergeTreeDataPartChecksums::readV2(ReadBuffer & in) Checksum sum; readString(name, in); + assertFileNameIsRelativeAndContained(name); assertString("\n\tsize: ", in); readText(sum.file_size, in); assertString("\n\thash: ", in); @@ -192,6 +221,7 @@ bool MergeTreeDataPartChecksums::readV3(ReadBuffer & in) Checksum sum; readStringBinary(name, in); + assertFileNameIsRelativeAndContained(name); readVarUInt(sum.file_size, in); readBinaryLittleEndian(sum.file_hash, in); readBinaryLittleEndian(sum.is_compressed, in); From 5c19509d4664670da9d1a5a5ec9a306d61e4694e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 14:06:16 +0000 Subject: [PATCH 016/296] Backport #120252 to 26.8: Fix out-of-bounds read on truncated compact `Variant` discriminators --- .../Serializations/SerializationVariant.cpp | 15 ++++++---- .../SerializationVariantElement.cpp | 15 ++++++---- ...truncated_compact_discriminators.reference | 30 +++++++++++++++++++ ...riant_truncated_compact_discriminators.sql | 18 +++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference create mode 100644 tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql diff --git a/src/DataTypes/Serializations/SerializationVariant.cpp b/src/DataTypes/Serializations/SerializationVariant.cpp index 8628d0c5373b9..9e9728a4d44fe 100644 --- a/src/DataTypes/Serializations/SerializationVariant.cpp +++ b/src/DataTypes/Serializations/SerializationVariant.cpp @@ -32,6 +32,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; extern const int INCORRECT_DATA; + extern const int CANNOT_READ_ALL_DATA; } /// Validate that a discriminator value is within bounds (< num_variants) or is NULL_DISCRIMINATOR. @@ -685,8 +686,7 @@ std::vector SerializationVariant::deserializeCompactDiscriminators( DeserializeBinaryBulkStateVariantDiscriminators & state, const DeserializeBinaryBulkSettings & settings) const { - auto & discriminators = assert_cast(discriminators_column); - auto & discriminators_data = discriminators.getData(); + auto & discriminators_data = assert_cast(discriminators_column).getData(); /// Reset state if we are reading from the start of the granule and not from the previous position in the file. if (!continuous_reading) @@ -709,16 +709,19 @@ std::vector SerializationVariant::deserializeCompactDiscriminators( size_t limit_in_granule = std::min(limit, state.remaining_rows_in_granule); if (state.granule_format == CompactDiscriminatorsGranuleFormat::COMPACT) { - auto & data = discriminators.getData(); - data.resize_fill(data.size() + limit_in_granule, state.compact_discr); + discriminators_data.resize_fill(discriminators_data.size() + limit_in_granule, state.compact_discr); if (state.compact_discr != ColumnVariant::NULL_DISCRIMINATOR) variant_limits[state.compact_discr] += limit_in_granule; } else { - SerializationNumber::create()->deserializeBinaryBulk(discriminators, *stream, limit_in_granule, 0); - size_t start = discriminators_data.size() - limit_in_granule; + size_t start = discriminators_data.size(); + SerializationNumber::deserializeBinaryBulk(discriminators_data, *stream, limit_in_granule); + size_t num_read = discriminators_data.size() - start; + if (num_read != limit_in_granule) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read all discriminators in Variant granule. Expected: {}, got: {}", limit_in_granule, num_read); for (size_t i = start; i != discriminators_data.size(); ++i) { diff --git a/src/DataTypes/Serializations/SerializationVariantElement.cpp b/src/DataTypes/Serializations/SerializationVariantElement.cpp index 9526595dcdfe6..48af83d42da9e 100644 --- a/src/DataTypes/Serializations/SerializationVariantElement.cpp +++ b/src/DataTypes/Serializations/SerializationVariantElement.cpp @@ -14,6 +14,7 @@ namespace ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; + extern const int CANNOT_READ_ALL_DATA; } UInt128 SerializationVariantElement::getHash(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_, size_t num_variants_, bool nullable_added_by_extraction_) @@ -290,8 +291,7 @@ size_t SerializationVariantElement::deserializeCompactDiscriminators( const ISerialization * serialization) { auto * discriminators_state = checkAndGetState(discriminators_state_, serialization); - auto & discriminators = assert_cast(discriminators_column); - auto & discriminators_data = discriminators.getData(); + auto & discriminators_data = assert_cast(discriminators_column).getData(); /// Reset state if we are reading from the start of the granule and not from the previous position in the file. if (!continuous_reading) @@ -315,16 +315,19 @@ size_t SerializationVariantElement::deserializeCompactDiscriminators( size_t limit_in_granule = std::min(limit, discriminators_state->remaining_rows_in_granule); if (discriminators_state->granule_format == SerializationVariant::CompactDiscriminatorsGranuleFormat::COMPACT) { - auto & data = discriminators.getData(); - data.resize_fill(data.size() + limit_in_granule, discriminators_state->compact_discr); + discriminators_data.resize_fill(discriminators_data.size() + limit_in_granule, discriminators_state->compact_discr); if (discriminators_state->compact_discr == variant_discriminator) variant_limit += limit_in_granule; } else { - SerializationNumber::create()->deserializeBinaryBulk(discriminators, *stream, limit_in_granule, 0); - size_t start = discriminators_data.size() - limit_in_granule; + size_t start = discriminators_data.size(); + SerializationNumber::deserializeBinaryBulk(discriminators_data, *stream, limit_in_granule); + size_t num_read = discriminators_data.size() - start; + if (num_read != limit_in_granule) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read all discriminators in Variant granule. Expected: {}, got: {}", limit_in_granule, num_read); for (size_t i = start; i != discriminators_data.size(); ++i) variant_limit += (discriminators_data[i] == variant_discriminator); diff --git a/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference new file mode 100644 index 0000000000000..91425ed846bbb --- /dev/null +++ b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference @@ -0,0 +1,30 @@ +42 +\N +\N +\N +42 +\N +0 +\N +2 +\N +4 +\N +6 +\N +0 +\N +2 +\N +4 +\N +6 +\N +0 +1 +0 +1 +0 +1 +0 +1 diff --git a/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql new file mode 100644 index 0000000000000..a42982dfe5797 --- /dev/null +++ b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql @@ -0,0 +1,18 @@ +-- Truncated `COMPACT` `Variant` discriminator streams must be rejected, not read past the buffer. +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x64\x01v\x0eVariant(UInt8)', unhex('01000000000000006400'))); -- { serverError CANNOT_READ_ALL_DATA } +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x02\x01v\x0eVariant(UInt8)', unhex('01000000000000000200ff'))); -- { serverError CANNOT_READ_ALL_DATA } +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x6c\x01v\x0eVariant(UInt8)', unhex('01000000000000000801ff6400ff'))); -- { serverError CANNOT_READ_ALL_DATA } + +-- Complete streams still decode correctly. +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x02\x01v\x0eVariant(UInt8)', unhex('0100000000000000020000ff2a'))); +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x04\x01v\x0eVariant(UInt8)', unhex('01000000000000000201ff020000ff2a'))); + +CREATE TABLE variant_compact_granules (id UInt8, v Variant(UInt8, String)) +ENGINE = MergeTree ORDER BY id +SETTINGS use_compact_variant_discriminators_serialization = 1, index_granularity = 8, + min_rows_for_wide_part = 0, min_bytes_for_wide_part = 0; +INSERT INTO variant_compact_granules SELECT number, if(number % 2 = 0, number::UInt8, NULL) FROM numbers(8); +SELECT v FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +SELECT v.UInt8 FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +SELECT v.UInt8.null FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +DROP TABLE variant_compact_granules; From 674b9945b7ccea091bad2efd87111944c1ad547a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 15:25:15 +0000 Subject: [PATCH 017/296] Backport #119848 to 26.8: Do not skip access checks of a user query executed as a nested query --- .../ClickHouseDictionarySource.cpp | 30 +++++-- .../Access/InterpreterExecuteAsQuery.cpp | 8 +- .../InterpreterParallelWithQuery.cpp | 6 +- src/Interpreters/QueryFlags.h | 5 ++ src/Interpreters/executeQuery.cpp | 5 +- src/Storages/StorageQueryRunner.cpp | 7 +- ...5212_parallel_with_access_checks.reference | 12 +++ .../05212_parallel_with_access_checks.sh | 63 ++++++++++++++ .../05213_execute_as_access_checks.reference | 11 +++ .../05213_execute_as_access_checks.sh | 61 ++++++++++++++ ...05214_query_runner_access_checks.reference | 15 ++++ .../05214_query_runner_access_checks.sh | 84 +++++++++++++++++++ ...ccess_checks_replicated_database.reference | 9 ++ ..._with_access_checks_replicated_database.sh | 61 ++++++++++++++ ...y_invalidate_query_access_checks.reference | 9 ++ ...ctionary_invalidate_query_access_checks.sh | 64 ++++++++++++++ 16 files changed, 439 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/05212_parallel_with_access_checks.reference create mode 100755 tests/queries/0_stateless/05212_parallel_with_access_checks.sh create mode 100644 tests/queries/0_stateless/05213_execute_as_access_checks.reference create mode 100755 tests/queries/0_stateless/05213_execute_as_access_checks.sh create mode 100644 tests/queries/0_stateless/05214_query_runner_access_checks.reference create mode 100755 tests/queries/0_stateless/05214_query_runner_access_checks.sh create mode 100644 tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference create mode 100755 tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh create mode 100644 tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference create mode 100755 tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 8a8fcf27c1ec6..250a3a504f3de 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -162,6 +162,26 @@ std::string ClickHouseDictionarySource::toString() const return "ClickHouse: " + configuration.db + '.' + configuration.table + (where.empty() ? "" : ", where: " + where); } +namespace +{ + +/// The query text comes from the dictionary definition (possibly a `CREATE DICTIONARY` written by a user +/// who has no other privileges), and for a local source it is executed as an `internal` query on behalf +/// of the configured user. So only a `SELECT` is allowed: any other statement (`CREATE TABLE`, ...) +/// would run with the access checks of `internal` queries skipped. +void checkQueryIsSelect(const String & query, const char * description, const char * error_message) +{ + const char * query_begin = query.data(); + const char * query_end = query.data() + query.size(); + ParserQuery parser(query_end); + ASTPtr ast = parseQuery(parser, query_begin, query_end, description, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + + if (!ast || ast->getQueryKind() != IAST::QueryKind::Select) + throw Exception(ErrorCodes::INCORRECT_QUERY, "{}", error_message); +} + +} + BlockIO ClickHouseDictionarySource::createStreamForQuery(const String & query) { BlockIO io; @@ -173,13 +193,7 @@ BlockIO ClickHouseDictionarySource::createStreamForQuery(const String & query) auto context_copy = Context::createCopy(context); context_copy->makeQueryContext(); - const char * query_begin = query.data(); - const char * query_end = query.data() + query.size(); - ParserQuery parser(query_end); - ASTPtr ast = parseQuery(parser, query_begin, query_end, "Query for ClickHouse dictionary", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); - - if (!ast || ast->getQueryKind() != IAST::QueryKind::Select) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Only SELECT query can be used as a dictionary source"); + checkQueryIsSelect(query, "Query for ClickHouse dictionary", "Only SELECT query can be used as a dictionary source"); if (configuration.is_local) { @@ -205,6 +219,8 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re { LOG_TRACE(log, "Performing invalidate query"); + checkQueryIsSelect(request, "Invalidate query for ClickHouse dictionary", "Only SELECT query can be used as a dictionary invalidate query"); + /// Copy context because results of scalar subqueries potentially could be cached auto context_copy = Context::createCopy(context); context_copy->makeQueryContext(); diff --git a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp index eeb5faae0d1b9..feb284043a18c 100644 --- a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp +++ b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp @@ -100,7 +100,13 @@ BlockIO InterpreterExecuteAsQuery::execute() { /// EXECUTE AS auto subquery_context = impersonateQueryContext(getContext(), target_user_name); - return executeQuery(query.subquery->formatWithSecretsOneLine(), subquery_context, QueryFlags{ .internal = true }).second; + /// The subquery is nested, hence `internal`, but its text comes from the user, hence `user_initiated`: + /// without it the access checks of `CREATE` subqueries would be skipped, so the impersonated statement + /// would not be limited to the privileges of the target user. + return executeQuery( + query.subquery->formatWithSecretsOneLine(), subquery_context, + QueryFlags{ .internal = true, .user_initiated = true }) + .second; } else { diff --git a/src/Interpreters/InterpreterParallelWithQuery.cpp b/src/Interpreters/InterpreterParallelWithQuery.cpp index eb6f14dcc54d9..ad7466083bc31 100644 --- a/src/Interpreters/InterpreterParallelWithQuery.cpp +++ b/src/Interpreters/InterpreterParallelWithQuery.cpp @@ -122,7 +122,11 @@ void InterpreterParallelWithQuery::executeSubqueries(const ASTs & subqueries) void InterpreterParallelWithQuery::executeSubquery(ASTPtr subquery, ContextMutablePtr subquery_context) { - auto query_io = executeQuery(subquery->formatWithSecretsOneLine(), subquery_context, QueryFlags{ .internal = true }).second; + /// The subqueries are nested, hence `internal`, but their text comes from the user, hence `user_initiated`: + /// without it the access checks of `CREATE` subqueries would be skipped. + auto query_io = executeQuery( + subquery->formatWithSecretsOneLine(), subquery_context, QueryFlags{ .internal = true, .user_initiated = true }) + .second; auto & pipeline = query_io.pipeline; diff --git a/src/Interpreters/QueryFlags.h b/src/Interpreters/QueryFlags.h index 24273e5ca0311..3e8eb3707f71e 100644 --- a/src/Interpreters/QueryFlags.h +++ b/src/Interpreters/QueryFlags.h @@ -6,6 +6,11 @@ namespace DB struct QueryFlags { bool internal = false; /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. + /// If true, the query was written by the user even though it is executed as an `internal` query, i.e. it is not + /// initiated by the server itself. Subqueries of `PARALLEL WITH` are like that: they are re-executed as nested + /// queries, but their text comes from the user. Such queries must be subject to all the restrictions of a regular + /// user query, in particular to access checks - `internal` alone must never be treated as a permission to skip them. + bool user_initiated = false; bool distributed_backup_restore = false; /// If true, this query is a part of backup restore. bool parse_query_from_initial_buffer = false; /// If true, do not read more data while parsing the query. The remaining input can be streaming insert data. bool background = false; /// If true, this query is the background run scheduled by executeQueryInBackground. diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 3f3ce1dc2661a..8615932970d47 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -3107,7 +3107,10 @@ static BlockIO executeQueryImpl( if (auto * create_interpreter = typeid_cast(interpreter.get())) { create_interpreter->setIsRestoreFromBackup(flags.distributed_backup_restore); - create_interpreter->setInternal(internal); + /// `InterpreterCreateQuery` uses `internal` to mean "initiated by the server itself, so all + /// the restrictions for user queries (access checks among them) can be skipped". A query + /// written by the user is never that, even when it is executed as a nested `internal` query. + create_interpreter->setInternal(internal && !flags.user_initiated); } std::unique_ptr span; diff --git a/src/Storages/StorageQueryRunner.cpp b/src/Storages/StorageQueryRunner.cpp index 845b2bd814bbe..e10d6f5d681db 100644 --- a/src/Storages/StorageQueryRunner.cpp +++ b/src/Storages/StorageQueryRunner.cpp @@ -442,7 +442,12 @@ class QueryRunnerDispatcher : WithContext void executeLocally(const QueryRunnerJob & job, ContextMutablePtr job_context) const { - auto io = executeQuery(job.query, job_context, QueryFlags{ .internal = true }).second; + /// The job is nested, hence `internal` - which is also what marks these queries with + /// `is_internal = 1` in `system.query_log`. Its text comes from the user who inserted it, + /// hence `user_initiated`: without it the access checks of `CREATE` jobs would be skipped, so + /// a job would not be limited to the privileges of the principal it runs as. + auto io + = executeQuery(job.query, job_context, QueryFlags{ .internal = true, .user_initiated = true }).second; try { if (io.pipeline.initialized()) diff --git a/tests/queries/0_stateless/05212_parallel_with_access_checks.reference b/tests/queries/0_stateless/05212_parallel_with_access_checks.reference new file mode 100644 index 0000000000000..10f1739ece85f --- /dev/null +++ b/tests/queries/0_stateless/05212_parallel_with_access_checks.reference @@ -0,0 +1,12 @@ +-- without the CREATE TABLE privilege, directly +ACCESS_DENIED +-- without the CREATE TABLE privilege, inside PARALLEL WITH +ACCESS_DENIED +-- CREATE DATABASE inside PARALLEL WITH is checked too +ACCESS_DENIED +-- nothing was created +0 +0 +-- with the CREATE TABLE privilege, PARALLEL WITH still works +t_parallel_1 +t_parallel_2 diff --git a/tests/queries/0_stateless/05212_parallel_with_access_checks.sh b/tests/queries/0_stateless/05212_parallel_with_access_checks.sh new file mode 100755 index 0000000000000..d66dceb563cf2 --- /dev/null +++ b/tests/queries/0_stateless/05212_parallel_with_access_checks.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `PARALLEL WITH` must not bypass access checks of its subqueries. + +user="user_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${user}" +# `TABLE ENGINE` is granted up front so that the only privilege missing below is `CREATE TABLE`. +# `access_control_improvements.table_engines_require_grant` is enabled in the test configuration, and +# the `TABLE ENGINE` check in `getTablePropertiesAndNormalizeCreateQuery` does not consult `internal` - +# without this grant a `CREATE TABLE` would be refused by that check even when its own access check is +# skipped, which would hide the very bypass this test is about. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password" + +# The test database may be reused across runs, so only the objects of this test are counted, and they +# are dropped both before and after the run. +tables="'t_direct', 't_parallel_1', 't_parallel_2'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_direct, ${CLICKHOUSE_DATABASE}.t_parallel_1, ${CLICKHOUSE_DATABASE}.t_parallel_2" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db_1" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db_2" + +echo "-- without the CREATE TABLE privilege, directly" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_direct (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- without the CREATE TABLE privilege, inside PARALLEL WITH" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- CREATE DATABASE inside PARALLEL WITH is checked too" +${CLIENT_AS_USER} --query " + CREATE DATABASE ${CLICKHOUSE_DATABASE}_db_1 + PARALLEL WITH + CREATE DATABASE ${CLICKHOUSE_DATABASE}_db_2 +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables})" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name IN ('${CLICKHOUSE_DATABASE}_db_1', '${CLICKHOUSE_DATABASE}_db_2')" + +echo "-- with the CREATE TABLE privilege, PARALLEL WITH still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.* TO ${user}" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_2 (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query "SELECT name FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables}) ORDER BY name" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1, ${CLICKHOUSE_DATABASE}.t_parallel_2" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05213_execute_as_access_checks.reference b/tests/queries/0_stateless/05213_execute_as_access_checks.reference new file mode 100644 index 0000000000000..6454eeb3e1620 --- /dev/null +++ b/tests/queries/0_stateless/05213_execute_as_access_checks.reference @@ -0,0 +1,11 @@ +-- target has no CREATE TABLE privilege +ACCESS_DENIED +-- target has no CREATE DATABASE privilege +ACCESS_DENIED +-- two layers of nesting: PARALLEL WITH inside EXECUTE AS is checked too +ACCESS_DENIED +-- nothing was created +0 +0 +-- once the target has the privilege, EXECUTE AS still works +t_execute_as diff --git a/tests/queries/0_stateless/05213_execute_as_access_checks.sh b/tests/queries/0_stateless/05213_execute_as_access_checks.sh new file mode 100755 index 0000000000000..8c36ae883e7a5 --- /dev/null +++ b/tests/queries/0_stateless/05213_execute_as_access_checks.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `EXECUTE AS` must not bypass access checks of the impersonated statement: the statement runs with +# the privileges of the target user, so a target without the `CREATE TABLE` privilege must be denied. + +caller="caller_${CLICKHOUSE_DATABASE}" +target="target_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${caller}, ${target}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${caller} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${target}" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${caller}, ${target}" +${CLICKHOUSE_CLIENT} --query "GRANT IMPERSONATE ON ${target} TO ${caller}" +# See the note in `05212_parallel_with_access_checks.sh`: `TABLE ENGINE` is granted up front so that +# the only privilege missing below is `CREATE TABLE`, otherwise the `TABLE ENGINE` check - which does +# not consult `internal` - would hide the bypass this test is about. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${target}" + +CLIENT_AS_CALLER="${CLICKHOUSE_CLIENT} --user ${caller} --password password" + +# The test database may be reused across runs, so only the objects of this test are counted, and they +# are dropped both before and after the run. +tables="'t_execute_as', 't_nested_1', 't_nested_2'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_execute_as, ${CLICKHOUSE_DATABASE}.t_nested_1, ${CLICKHOUSE_DATABASE}.t_nested_2" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db" + +echo "-- target has no CREATE TABLE privilege" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE TABLE ${CLICKHOUSE_DATABASE}.t_execute_as (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- target has no CREATE DATABASE privilege" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE DATABASE ${CLICKHOUSE_DATABASE}_db +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- two layers of nesting: PARALLEL WITH inside EXECUTE AS is checked too" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_nested_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_nested_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables})" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name = '${CLICKHOUSE_DATABASE}_db'" + +echo "-- once the target has the privilege, EXECUTE AS still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.* TO ${target}" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE TABLE ${CLICKHOUSE_DATABASE}.t_execute_as (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query "SELECT name FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables}) ORDER BY name" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.t_execute_as" +${CLICKHOUSE_CLIENT} --query "DROP USER ${caller}, ${target}" diff --git a/tests/queries/0_stateless/05214_query_runner_access_checks.reference b/tests/queries/0_stateless/05214_query_runner_access_checks.reference new file mode 100644 index 0000000000000..50d1f2894c738 --- /dev/null +++ b/tests/queries/0_stateless/05214_query_runner_access_checks.reference @@ -0,0 +1,15 @@ +-- without the CREATE DATABASE privilege, directly +ACCESS_DENIED +-- without the CREATE DATABASE privilege, queued into the QueryRunner table: the job is denied +ExceptionBeforeStart ACCESS_DENIED +-- queued through two layers of QueryRunner tables: the inner job is denied as well +QueryFinish OK +ExceptionBeforeStart ACCESS_DENIED +ExceptionBeforeStart ACCESS_DENIED +-- the database was not created +0 +-- once the privilege is granted, the queued job runs +ExceptionBeforeStart ACCESS_DENIED +ExceptionBeforeStart ACCESS_DENIED +QueryFinish OK +1 diff --git a/tests/queries/0_stateless/05214_query_runner_access_checks.sh b/tests/queries/0_stateless/05214_query_runner_access_checks.sh new file mode 100755 index 0000000000000..646a4c82d8cb6 --- /dev/null +++ b/tests/queries/0_stateless/05214_query_runner_access_checks.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A job queued into a `QueryRunner` table must not bypass the access checks of the principal it runs +# as. With `SQL SECURITY INVOKER` that principal is the user who inserted the job, so a user without +# the `CREATE DATABASE` privilege must not be able to create a database by queueing it. +# +# The `INSERT` into the table succeeds no matter how the job ends - a failed job is only logged - and +# the mere absence of the database would also be observed if the job never ran at all. Hence the +# outcome of every job is asserted through `system.query_log`, where the jobs are recorded as +# internal queries of the `ClickHouse QueryRunner` client running on behalf of the user. + +# The user and the database carry a random suffix so that the rows this run reads from the shared +# `system.query_log` are its own: a previous run on the same server may have used the same test +# database, and its jobs would otherwise be printed again. +run_id=$(random_str 8) +user="user_${CLICKHOUSE_DATABASE}_${run_id}" +db="${CLICKHOUSE_DATABASE}_db_${run_id}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.runner" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.outer_runner" +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.runner (query String, database String) ENGINE = QueryRunner + SETTINGS mode = 'synchronous', threads = 1 SQL SECURITY INVOKER +" +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.outer_runner (query String, database String) ENGINE = QueryRunner + SETTINGS mode = 'synchronous', threads = 1 SQL SECURITY INVOKER +" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT, INSERT ON ${CLICKHOUSE_DATABASE}.* TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password" + +# Prints how the jobs whose query text starts with $1 ended: the log entry type and the error code. +# Only the jobs of this test's user run by the `QueryRunner` client are considered, so the direct +# queries of the user and the `INSERT` queries that queued the jobs are not mixed in, and the user's +# name is unique to this run. Every job below names the database of this test, which is what puts it +# in `current_database`. +function jobs_outcome() +{ + ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" + ${CLICKHOUSE_CLIENT} --query " + SELECT type, errorCodeToName(exception_code) + FROM system.query_log + WHERE event_date >= yesterday() AND is_internal AND client_name = 'ClickHouse QueryRunner' + AND current_database = currentDatabase() + AND user = '${user}' AND startsWith(query, '${1}') AND type != 'QueryStart' + ORDER BY event_time_microseconds + " +} + +echo "-- without the CREATE DATABASE privilege, directly" +${CLIENT_AS_USER} --query "CREATE DATABASE ${db}" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- without the CREATE DATABASE privilege, queued into the QueryRunner table: the job is denied" +${CLIENT_AS_USER} --query "INSERT INTO ${CLICKHOUSE_DATABASE}.runner VALUES ('CREATE DATABASE ${db}', '${CLICKHOUSE_DATABASE}')" +jobs_outcome "CREATE DATABASE ${db}" + +echo "-- queued through two layers of QueryRunner tables: the inner job is denied as well" +${CLIENT_AS_USER} --query " + INSERT INTO ${CLICKHOUSE_DATABASE}.outer_runner + VALUES ('INSERT INTO ${CLICKHOUSE_DATABASE}.runner VALUES (''CREATE DATABASE ${db}'', ''${CLICKHOUSE_DATABASE}'')', '${CLICKHOUSE_DATABASE}') +" +jobs_outcome "INSERT INTO ${CLICKHOUSE_DATABASE}.runner" +jobs_outcome "CREATE DATABASE ${db}" + +echo "-- the database was not created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name = '${db}'" + +echo "-- once the privilege is granted, the queued job runs" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE DATABASE ON ${db}.* TO ${user}" +${CLIENT_AS_USER} --query "INSERT INTO ${CLICKHOUSE_DATABASE}.runner VALUES ('CREATE DATABASE ${db}', '${CLICKHOUSE_DATABASE}')" +jobs_outcome "CREATE DATABASE ${db}" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name = '${db}'" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${db}" +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.outer_runner" +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.runner" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference new file mode 100644 index 0000000000000..8e3a9dbe8fd45 --- /dev/null +++ b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference @@ -0,0 +1,9 @@ +-- without the CREATE TABLE privilege, directly +ACCESS_DENIED +-- without the CREATE TABLE privilege, inside PARALLEL WITH +ACCESS_DENIED +-- nothing was created +0 +-- with the CREATE TABLE privilege, PARALLEL WITH still works +t_parallel_1 +t_parallel_2 diff --git a/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh new file mode 100755 index 0000000000000..dff7b96ce9cb5 --- /dev/null +++ b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: zookeeper + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `PARALLEL WITH` must not bypass the access checks of its subqueries in a `Replicated` database either. +# There a `CREATE TABLE` is additionally replayed through the replicated DDL log, and the replay of a +# statement written by the user is subject to the same rules as the replay of a direct statement. + +# The user, the database and its ZooKeeper path carry a random suffix, and the leftovers of a previous +# run that died halfway are dropped up front, so that the test can be rerun on the same server. +run_id=$(random_str 8) +user="user_${CLICKHOUSE_DATABASE}_${run_id}" +db="rdb_${CLICKHOUSE_DATABASE}_${run_id}" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${db} SYNC" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE ${db} ENGINE = Replicated('/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/rdb_${run_id}', '1', '1')" +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${db}.* TO ${user}" +# `TABLE ENGINE` is granted up front so that the only privilege missing below is `CREATE TABLE`, +# see `05212_parallel_with_access_checks.sh`. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password --distributed_ddl_output_mode none" + +echo "-- without the CREATE TABLE privilege, directly" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_direct (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- without the CREATE TABLE privilege, inside PARALLEL WITH" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${db}.t_parallel_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM system.tables + WHERE database = '${db}' AND name IN ('t_direct', 't_parallel_1', 't_parallel_2') +" + +echo "-- with the CREATE TABLE privilege, PARALLEL WITH still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${db}.* TO ${user}" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${db}.t_parallel_2 (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query " + SELECT name FROM system.tables + WHERE database = '${db}' AND name IN ('t_direct', 't_parallel_1', 't_parallel_2') + ORDER BY name +" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${db} SYNC" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference new file mode 100644 index 0000000000000..97c9358c2bd09 --- /dev/null +++ b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference @@ -0,0 +1,9 @@ +-- a non-SELECT invalidate query of a local source is rejected, so it does not run as internal +1 +-- reloaded after the invalidate query was evaluated +1 +1 +-- nothing was created +0 +-- a SELECT invalidate query keeps working +2 diff --git a/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh new file mode 100755 index 0000000000000..c3b179a57cbb0 --- /dev/null +++ b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The `invalidate_query` of a local `ClickHouse` dictionary source is executed as an `internal` query +# on behalf of the source user. Its text comes from `CREATE DICTIONARY`, so a user who may create a +# dictionary but not a table must not be able to smuggle a `CREATE TABLE` through it: only a `SELECT` +# is accepted, the same way as for the main dictionary query. + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT, CREATE DICTIONARY, DROP DICTIONARY, dictGet ON ${db}.* TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password" + +# The test database may be reused across runs, so the objects of this test are dropped both before +# and after the run. +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY IF EXISTS ${db}.dict_invalidate_ddl, ${db}.dict_invalidate_select" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${db}.t_smuggled" + +echo "-- a non-SELECT invalidate query of a local source is rejected, so it does not run as internal" +${CLIENT_AS_USER} --query " + CREATE DICTIONARY ${db}.dict_invalidate_ddl (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(QUERY 'SELECT 1 AS id, 1 AS value' INVALIDATE_QUERY 'CREATE TABLE ${db}.t_smuggled (x UInt8) ENGINE = Memory')) + LAYOUT(FLAT()) LIFETIME(MIN 1 MAX 1) +" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_ddl, 'value', toUInt64(1))" + +# The first load does not run the invalidate query; it is run by the periodic check (every 5 seconds) +# once the lifetime has passed, and it decides whether the dictionary is reloaded. Wait for that +# check to happen: the rejected invalidate query counts as "modified", so the dictionary is reloaded +# and `last_successful_update_time` advances. +first_update=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'") +for _ in $(seq 1 120) +do + update=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'") + if [[ "${update}" -gt "${first_update}" ]] + then + break + fi + sleep 0.5 +done +echo "-- reloaded after the invalidate query was evaluated" +${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) > ${first_update} FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_ddl, 'value', toUInt64(1))" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${db}' AND name = 't_smuggled'" + +echo "-- a SELECT invalidate query keeps working" +${CLIENT_AS_USER} --query " + CREATE DICTIONARY ${db}.dict_invalidate_select (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(QUERY 'SELECT 1 AS id, 2 AS value' INVALIDATE_QUERY 'SELECT 1')) + LAYOUT(FLAT()) LIFETIME(MIN 1 MAX 1) +" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_select, 'value', toUInt64(1))" + +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY ${db}.dict_invalidate_ddl, ${db}.dict_invalidate_select" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" From a60b43cb9515ac4e17faada5588a2aa8c8943ab5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 15:53:28 +0000 Subject: [PATCH 018/296] Backport #118791 to 26.8: Allow to change fs cache boundary alignment per query --- ci/jobs/scripts/check_style/various_checks.sh | 5 ++ src/Core/Settings.cpp | 2 +- src/Interpreters/Context.cpp | 4 ++ ...boundary_alignment_for_disk_read.reference | 2 + ..._cache_boundary_alignment_for_disk_read.sh | 51 +++++++++++++++++++ 5 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.reference create mode 100755 tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.sh diff --git a/ci/jobs/scripts/check_style/various_checks.sh b/ci/jobs/scripts/check_style/various_checks.sh index c0453293f99f7..9f2367174a27b 100755 --- a/ci/jobs/scripts/check_style/various_checks.sh +++ b/ci/jobs/scripts/check_style/various_checks.sh @@ -210,6 +210,11 @@ tests_with_system_drop=( $( sort -u ) ) for test_case in "${tests_with_system_drop[@]}"; do + # `SYSTEM DROP FILESYSTEM CACHE ''` affects only the named cache, not the + # server-wide state, so a test which drops the cache it created itself can run in + # parallel. Skip a test if every `SYSTEM DROP` in it drops a cache by name. + grep -oiP "system\s+drop(\s+filesystem\s+cache\s+'[^']+')?" "$test_case" | + grep -qivP "filesystem\s+cache\s+'" || continue grep -qP '(--|#)\s*[Tt]ags:.*no-parallel' "$test_case" || echo "Test with SYSTEM DROP should have no-parallel tag: $test_case" done diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 064ae514eb49b..63a92d32a3bba 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7245,7 +7245,7 @@ Maximum time to wait for a file segment which is being downloaded to the filesys Prefer bigger buffer size if filesystem cache is enabled to avoid writing small file segments which deteriorate cache performance. On the other hand, enabling this setting might increase memory usage. )", 0) \ DECLARE(UInt64, filesystem_cache_boundary_alignment, 0, R"( -Filesystem cache boundary alignment. This setting is applied only for non-disk read (e.g. for cache of remote table engines / table functions, but not for storage configuration of MergeTree tables). Value 0 means no alignment. +Filesystem cache boundary alignment. For non-disk read (e.g. for cache of remote table engines / table functions) value 0 means no alignment. For disk read (e.g. for MergeTree tables on a disk with cache) value 0 means that `boundary_alignment` from the cache configuration is used. )", 0) \ DECLARE(UInt64, temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds, (10 * 60 * 1000), R"( Wait time to lock cache for space reservation for temporary data in filesystem cache diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e07a32080a5be..0db1f0c3577ef 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -317,6 +317,7 @@ namespace Setting extern const SettingsBool enable_filesystem_read_prefetches_log; extern const SettingsBool enable_blob_storage_log; extern const SettingsBool enable_blob_storage_log_for_read_operations; + extern const SettingsUInt64 filesystem_cache_boundary_alignment; extern const SettingsUInt64 filesystem_cache_max_download_size; extern const SettingsUInt64 filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; extern const SettingsUInt64 filesystem_cache_wait_for_concurrent_download_timeout_milliseconds; @@ -8647,6 +8648,9 @@ ReadSettings Context::getReadSettings() const = settings_ref[Setting::filesystem_cache_enable_background_download_during_fetch]; res.filesystem_cache_settings.prefer_bigger_buffer_size = settings_ref[Setting::filesystem_cache_prefer_bigger_buffer_size]; + if (settings_ref[Setting::filesystem_cache_boundary_alignment]) + res.filesystem_cache_settings.boundary_alignment = settings_ref[Setting::filesystem_cache_boundary_alignment]; + res.filesystem_cache_settings.max_download_size_per_query = settings_ref[Setting::filesystem_cache_max_download_size]; res.filesystem_cache_settings.skip_download_if_exceeds_per_query_cache_write_limit = settings_ref[Setting::filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit]; diff --git a/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.reference b/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.reference new file mode 100644 index 0000000000000..0d66ea1aee952 --- /dev/null +++ b/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.reference @@ -0,0 +1,2 @@ +0 +1 diff --git a/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.sh b/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.sh new file mode 100755 index 0000000000000..e31d6a228c900 --- /dev/null +++ b/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-random-settings, no-random-merge-tree-settings, no-distributed-cache + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Query level setting `filesystem_cache_boundary_alignment` must override +# `boundary_alignment` of the cache configuration also for disk read. +ALIGNMENT=$((20 * 1024 * 1024)) + +$CLICKHOUSE_CLIENT -m -q " +DROP TABLE IF EXISTS test; +CREATE TABLE test (a Int32, b String) +ENGINE = MergeTree() +ORDER BY a +SETTINGS disk = disk(type = cache, + max_size = '1Gi', + max_file_segment_size = '40Mi', + boundary_alignment = '1Mi', + cache_on_write_operations = 0, + path = '$CLICKHOUSE_TEST_UNIQUE_NAME', + name = '$CLICKHOUSE_TEST_UNIQUE_NAME', + disk = 's3_disk'); + +INSERT INTO test SELECT number, randomString(100) FROM numbers(500000); + +SYSTEM DROP FILESYSTEM CACHE '$CLICKHOUSE_TEST_UNIQUE_NAME'; + +SET read_through_distributed_cache = 0; +SET filesystem_cache_boundary_alignment = $ALIGNMENT; + +-- Read a granule from the middle of the table, so that the file segments +-- of the big column are neither at the beginning nor at the end of the file. +SELECT * FROM test WHERE a = 250000 FORMAT Null; +" + +# File segments must start at a boundary of the requested alignment +# (and there must be a file segment which does not start at the beginning of the file, +# otherwise the check above is trivial). +$CLICKHOUSE_CLIENT -m -q " +SELECT count() FROM system.filesystem_cache +WHERE cache_name = '$CLICKHOUSE_TEST_UNIQUE_NAME' +AND file_segment_range_begin % $ALIGNMENT != 0; + +SELECT count() > 0 FROM system.filesystem_cache +WHERE cache_name = '$CLICKHOUSE_TEST_UNIQUE_NAME' +AND file_segment_range_begin > 0; +" + +$CLICKHOUSE_CLIENT -q "DROP TABLE test" From 8049b49823b7489a9ff296a9809c53234c695474 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 16:15:56 +0000 Subject: [PATCH 019/296] Backport #116446 to 26.8: Do not use the distributed-plan Shuffle aggregation when bucket order is promised --- src/Core/Settings.cpp | 1 + .../Optimizations/makeDistributed.cpp | 13 ++ ...ed_plan_merge_scatter_multishard.reference | 16 +- ...tributed_plan_merge_scatter_multishard.sql | 15 +- ...lan_shuffle_bucket_order_promise.reference | 14 ++ ...uted_plan_shuffle_bucket_order_promise.sql | 167 ++++++++++++++++++ 6 files changed, 213 insertions(+), 13 deletions(-) create mode 100644 tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.reference create mode 100644 tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.sql diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 064ae514eb49b..e91f3465602f7 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -9075,6 +9075,7 @@ Experimental dictionary source for integration with YTsaurus. )", EXPERIMENTAL) \ DECLARE(Bool, distributed_plan_force_shuffle_aggregation, false, R"( Use Shuffle aggregation strategy instead of PartialAggregation + Merge in distributed query plan. +Ignored where the Shuffle strategy cannot produce a correct result, for example for `GROUPING SETS` or when the aggregation must produce results in bucket order. )", EXPERIMENTAL) \ DECLARE(Bool, enable_cascades_optimizer, false, R"( Enable the Cascades cost-based optimizer for distributed query plans. diff --git a/src/Processors/QueryPlan/Optimizations/makeDistributed.cpp b/src/Processors/QueryPlan/Optimizations/makeDistributed.cpp index 3353325b95940..52ce73ca539c8 100644 --- a/src/Processors/QueryPlan/Optimizations/makeDistributed.cpp +++ b/src/Processors/QueryPlan/Optimizations/makeDistributed.cpp @@ -587,6 +587,19 @@ void tryMakeDistributedAggregation(QueryPlan::Node & node, QueryPlan::Nodes & no if (optimization_settings.distributed_plan_force_shuffle_aggregation && !aggregation_keys.empty()) strategy = Shuffle; + /// Shuffle moves the aggregation step unchanged, so each of the `bucket_count` instances keeps the + /// promise of bucket order while ordering only its own share, and the gather cannot restore a global + /// order: it merges by a sort description, and the bucket number is chunk metadata, not a column. + /// Shuffle is therefore impossible here, so `distributed_plan_force_shuffle_aggregation` cannot + /// apply either, as with `GROUPING SETS` below. + if (aggregating_step->shouldProduceResultsInBucketOrder()) + { + if (!can_use_partial_aggregation) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "make_distributed_plan does not support aggregation in order which must produce results in bucket order"); + strategy = PartialAggregation; + } + /// Shuffle scatters by the full key set, so GROUPING SETS subtotals (over key subsets) would be /// produced in several buckets and duplicated. Partial aggregation has no such problem: every /// worker produces partial states for every grouping set over its share of the data, tagged with diff --git a/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.reference b/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.reference index 1e2b514dd4ceb..6007533519d57 100644 --- a/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.reference +++ b/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.reference @@ -1,17 +1,17 @@ Expression ((Project names + Projection)) MergingAggregated ReadFromMerge - GatherExchange - Expression - Aggregating - ShuffleExchange + Expression + MergingAggregated (merge) + GatherExchange + Aggregating (partial) Expression (Before GROUP BY) Filter ReadFromMergeTree (default.base107946_1) - GatherExchange - Expression - Aggregating - ShuffleExchange + Expression + MergingAggregated (merge) + GatherExchange + Aggregating (partial) Expression (Before GROUP BY) Filter ReadFromMergeTree (default.base107946_4) diff --git a/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.sql b/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.sql index d756bd43797ba..ae1d43a893ccc 100644 --- a/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.sql +++ b/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.sql @@ -26,18 +26,23 @@ CREATE TABLE m107946 ENGINE = Merge(currentDatabase(), '^d107946_(1|4)$'); -- aggregation with a non-zero limit (Code 344). -- prefer_localhost_replica must be 1: with 0 the Distributed child ships its plan to the localhost -- replica over the classic protocol, which cannot deserialize distributed-plan steps (Code 47). --- distributed_plan_max_rows_to_broadcast = 0 forces shuffle aggregation and bucketed reads, so the --- child plan deterministically contains exchanges. +-- distributed_plan_max_rows_to_broadcast = 0 makes the size-based strategy choice pick shuffle +-- aggregation, and forces bucketed reads, so the child plan deterministically contains exchanges. -- distributed_plan_default_shuffle_join_bucket_count is pinned > 1: with 1 bucket the broken path -- is not exercised. +-- distributed_aggregation_memory_efficient must be 1: the plan below depends on the bucket-order +-- promise, which holds when either it or enable_memory_bound_merging_of_aggregation_results is on, and +-- the runner randomizes both. SET make_distributed_plan = 1, enable_parallel_replicas = 0, distributed_plan_execute_locally = 1, use_statistics = 1, distributed_plan_optimize_exchanges = 1, enable_join_runtime_filters = 0, max_rows_to_group_by = 0, prefer_localhost_replica = 1, distributed_plan_max_rows_to_broadcast = 0, - distributed_plan_default_shuffle_join_bucket_count = 8, explain_query_plan_default = 'legacy'; + distributed_plan_default_shuffle_join_bucket_count = 8, explain_query_plan_default = 'legacy', + distributed_aggregation_memory_efficient = 1; -- The outer plan stays single-stage: the children aggregate up to the mergeable state themselves, --- so there are no exchanges above ReadFromMerge. Each child plan under it carries a single layer --- of exchanges: a gather over the aggregation over the shuffle. +-- so there are no exchanges above ReadFromMerge. Aggregating to the mergeable state promises results +-- in bucket order, and the shuffle strategy cannot keep that promise, so each child plan carries a +-- partial aggregation and its merge around a gather rather than an aggregation over a shuffle. EXPLAIN SELECT count(_table) FROM m107946 WHERE _table = 'base107946_1' GROUP BY _table; -- The reproducer from the issue. It must not throw; no rows match because _table exposes the diff --git a/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.reference b/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.reference new file mode 100644 index 0000000000000..6523a0f5d10f5 --- /dev/null +++ b/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.reference @@ -0,0 +1,14 @@ +1 +1 +1 +1 +1 +1 +7 600056 +7 600056 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.sql b/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.sql new file mode 100644 index 0000000000000..4a609c2334145 --- /dev/null +++ b/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.sql @@ -0,0 +1,167 @@ +-- Tags: shard, no-old-analyzer + +SET enable_parallel_replicas = 0; +SET automatic_parallel_replicas_mode = 0; +SET explain_query_plan_default = 'legacy'; +-- Distributed aggregation cannot enforce a global `max_rows_to_group_by`, so pin it to 0. +SET max_rows_to_group_by = 0; + +DROP TABLE IF EXISTS t_shuffle_bucket_order; +CREATE TABLE t_shuffle_bucket_order (k UInt64, v UInt64) ENGINE = MergeTree ORDER BY tuple() + SETTINGS index_granularity = 256, auto_statistics_types = ''; +INSERT INTO t_shuffle_bucket_order SELECT number % 50000, number FROM numbers(200000); + +SET make_distributed_plan = 1; +SET distributed_plan_execute_locally = 1; +SET distributed_plan_default_shuffle_join_bucket_count = 2; +-- No statistics, so the strategy choice does not depend on an estimated group count. +SET use_statistics = 0; +SET distributed_plan_max_rows_to_broadcast = 0; +-- A shard plan otherwise carries `BlocksMarshallingStep`, which cannot run on a worker, and a plan +-- holding it is executed with its exchanges turned into no-ops instead of being distributed. +SET enable_parallel_blocks_marshalling = 0; +-- Two-level aggregation states in every producer, so the merge consumes several buckets per input. +SET group_by_two_level_threshold = 10000; +SET group_by_two_level_threshold_bytes = 1; +SET max_threads = 16; +SET distributed_aggregation_memory_efficient = 1; +-- Pinned because the runner randomizes it and the promise below is set from either setting, so an arm +-- that leaves this one open does not say which of the two it exercised. +SET enable_memory_bound_merging_of_aggregation_results = 1; +-- `EXPLAIN PLAN distributed = 1` prints a shipped shard plan as it was shipped, and a shard plan is +-- shipped before the distributed rewrite runs on it, so the rows below read the rewrite only when the +-- shard receives the query as text. +SET serialize_query_plan = 0; + +-- Arming, asserted separately from the results below, on the same rewritten plan the guard acts on: +-- `distributed = 1` shows the per-shard plans, and the settings sit on the inner query because the +-- wrapper's own `SETTINGS` clause, which keeps the wrapper itself out of the rewrite, would otherwise +-- apply to the plan being explained as well. The initiator merges bucket by bucket, it does so over +-- more than one producer, and both are required for a duplicated bucket to be observable. +SELECT count() > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1) + WHERE explain ILIKE '%memory-efficient%' + SETTINGS make_distributed_plan = 0; +SELECT count() > 1 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1) + WHERE explain ILIKE '%ReadFromMergeTree%' + SETTINGS make_distributed_plan = 0; + +-- The guard itself, observed at the site it acts on: the shuffle scatter is gone from the shard plan, +-- so the strategy is partial aggregation plus merge. The second row pins that +-- `distributed_plan_force_shuffle_aggregation` loses to the guard, as it loses to GROUPING SETS. +SELECT count() = 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; +SELECT count() = 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; + +-- And the strategy it demotes to is the partial aggregation plus its merge, each named at the site +-- that builds it, so a plan left undistributed rather than demoted does not satisfy these two rows. +SELECT count() > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%MergingAggregated (merge)%' + SETTINGS make_distributed_plan = 0; +SELECT count() > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%Aggregating (partial)%' + SETTINGS make_distributed_plan = 0; + +-- The aggregation must complete. The shuffle strategy keeps the promise to produce results in bucket +-- order while each of its instances orders only its own share, so the merge receives a bucket it has +-- already merged and rejects the whole query. Two shapes, because the merge reaches the duplicate +-- from both its ordered and its delayed-bucket push. The force setting pins the strategy, so neither +-- arm depends on the statistics-free default choice. +SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k FORMAT Null + SETTINGS distributed_plan_force_shuffle_aggregation = 1; +SELECT k FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY ALL FORMAT Null + SETTINGS distributed_plan_force_shuffle_aggregation = 1; +-- A shipped shard plan is rewritten by the shard that receives it rather than by the initiator. No +-- explain reaches a rewrite made there, so this row pins the outcome rather than the strategy: it does +-- not separate the demotion from an aggregation left unrewritten above the gather, which the rows above +-- separate whenever the plan is not shipped. +SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k FORMAT Null + SETTINGS distributed_plan_force_shuffle_aggregation = 1, serialize_query_plan = 1; + +-- The keys and the aggregate values must match the plain plan, not merely avoid the rejection. A +-- single group keeps the output deterministic without an ORDER BY, which this plan cannot distribute. +SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k HAVING k = 7; +SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k HAVING k = 7 + SETTINGS make_distributed_plan = 0; + +-- The shuffle strategy stays available, including under its force setting, when nothing downstream +-- requires bucket order. The first row is the shard plan of the query the guard acts on, with both +-- settings that create the promise turned off: the same plan loses the shuffle above when they are on, +-- so a guard demoting every non-final aggregation fails here. +SELECT count() > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 0, + enable_memory_bound_merging_of_aggregation_results = 0, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; + +-- The next two rows keep the `Complete` stage, with both settings pinned on, so what keeps the shuffle +-- there is the stage alone: a guard reading those settings instead of the promise fails them. That is +-- also the default configuration, since both settings default to 1. +SELECT count() > 0 FROM (EXPLAIN PLAN actions = 1 SELECT k, sum(v) FROM t_shuffle_bucket_order GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 1, + enable_memory_bound_merging_of_aggregation_results = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; +SELECT count() > 0 FROM (EXPLAIN PLAN actions = 1 SELECT k, sum(v) FROM t_shuffle_bucket_order GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 1, + enable_memory_bound_merging_of_aggregation_results = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; + +-- The promise is set from either setting, so each of the two demotes on its own. Both rows pin both +-- settings, and each asserts the whole shape at once: no shuffle, and the partial aggregation plus its +-- merge, so a plan left undistributed does not satisfy them either. +SELECT countIf(explain ILIKE '%by hash(%') = 0 + AND countIf(explain ILIKE '%Aggregating (partial)%') > 0 + AND countIf(explain ILIKE '%MergingAggregated (merge)%') > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 1, + enable_memory_bound_merging_of_aggregation_results = 0, + distributed_plan_force_shuffle_aggregation = 1) + SETTINGS make_distributed_plan = 0; +SELECT countIf(explain ILIKE '%by hash(%') = 0 + AND countIf(explain ILIKE '%Aggregating (partial)%') > 0 + AND countIf(explain ILIKE '%MergingAggregated (merge)%') > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 0, + enable_memory_bound_merging_of_aggregation_results = 1, + distributed_plan_force_shuffle_aggregation = 1) + SETTINGS make_distributed_plan = 0; + +-- The boundary of the row above: with only memory-bound merging on, the merge over the shard output is +-- not the memory-efficient one, so nothing there reads the bucket order the demotion preserves. +SELECT count() = 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 0, + enable_memory_bound_merging_of_aggregation_results = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%memory-efficient%' + SETTINGS make_distributed_plan = 0; + +DROP TABLE t_shuffle_bucket_order; From aee16e2cb1e951f6341ddf8caa2c08336f0057a4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 18:51:49 +0000 Subject: [PATCH 020/296] Backport #119847 to 26.8: Handle a ZooKeeper `Multi` request without subrequests in Keeper --- src/Coordination/KeeperStorageImpl.cpp | 59 ++++++++++++++++--- .../test_keeper_empty_multi/__init__.py | 0 .../configs/keeper.xml | 28 +++++++++ .../test_keeper_empty_multi/test.py | 55 +++++++++++++++++ 4 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_keeper_empty_multi/__init__.py create mode 100644 tests/integration/test_keeper_empty_multi/configs/keeper.xml create mode 100644 tests/integration/test_keeper_empty_multi/test.py diff --git a/src/Coordination/KeeperStorageImpl.cpp b/src/Coordination/KeeperStorageImpl.cpp index 2c007d237e582..92b5405c561c2 100644 --- a/src/Coordination/KeeperStorageImpl.cpp +++ b/src/Coordination/KeeperStorageImpl.cpp @@ -1532,15 +1532,26 @@ static Coordination::Error preprocess( return Coordination::Error::ZOK; } +/// Cuts the deltas of the next subrequest, up to its `SubDeltaEnd` marker, off the front of `deltas` +/// and drops the marker. `preprocess` appends the marker after every subrequest, so a range without +/// it does not match the request that is being processed: the markers were lost, and stepping past +/// the end of the range to look for them is undefined behavior. This runs on the raft commit and +/// replay threads, so treat it like every other mismatch between a request and its deltas. +/// +/// `FailedMultiDelta` is the other marker `preprocess` emits, and the callers handle it before they +/// get here: it is the sole delta of a failed multi request. Inside a subrequest slice it is out of +/// place, and `commit` would ignore it and report the subrequest as successful, so the walk stops on +/// both markers and rejects the failure marker instead of passing it on as an ordinary delta. static KeeperStorage::DeltaRange extractSubdeltas(KeeperStorage::DeltaRange & deltas) { - auto it = deltas.begin(); - - for (; it != deltas.end(); ++it) - { - if (std::holds_alternative(it->operation)) - break; - } + auto it = std::ranges::find_if( + deltas, + [](const auto & delta) + { return std::holds_alternative(delta.operation) || std::holds_alternative(delta.operation); }); + if (it == deltas.end()) + onStorageInconsistency("Missing SubDeltaEnd marker for a Multi subrequest"); + if (std::holds_alternative(it->operation)) + onStorageInconsistency("Unexpected failure marker inside a subrequest of a Multi request"); KeeperStorage::DeltaRange result{deltas.begin(), it}; ++it; @@ -1564,10 +1575,34 @@ process(const Coordination::ZooKeeperMultiRequest & zk_request, Storage & storag const auto & subrequests = zk_request.requests; - // the deltas will have at least SubDeltaEnd or FailedMultiDelta - chassert(!deltas.empty()); + /// `preprocess` appends at least `SubDeltaEnd` or `FailedMultiDelta` for every subrequest, so the + /// range is empty only for a multi request that has no subrequests. Such a request is accepted - + /// ZooKeeper answers it with an empty successful response, and a client that builds a transaction + /// from a list that turns out to be empty sends exactly that - so answer it the same way here. + /// `processWatches` below already handles the empty range, and this runs on the raft commit + /// thread, where an exception terminates the process. + /// + /// The success return is reserved for the true zero-subrequest case: a multi request with + /// subrequests but without deltas means that the markers of the preprocessing were lost, and + /// answering it with an empty success would silently drop every suboperation, so it goes through + /// the storage inconsistency path like every other request whose deltas do not match. + if (deltas.empty()) + { + if (!subrequests.empty()) + onStorageInconsistency("Unexpected empty deltas for Multi request with subrequests"); + + response->error = Coordination::Error::ZOK; + return response; + } + if (const auto * failed_multi = std::get_if(&deltas.front().operation)) { + /// `preprocess` puts the failure marker last and the caller rolls back everything before it, + /// so the marker is the only delta of a failed multi request. Anything else in the range is + /// a delta that no subrequest response would account for, so it cannot be dropped silently. + if (std::next(deltas.begin()) != deltas.end()) + onStorageInconsistency("Unexpected deltas after the failure marker of a Multi request"); + const size_t subrequests_count = subrequests.size(); for (size_t i = 0; i < subrequests_count; ++i) @@ -1591,6 +1626,12 @@ process(const Coordination::ZooKeeperMultiRequest & zk_request, Storage & storag *multi_subrequest, [&](const auto & subrequest) { return process(subrequest, storage, std::move(subdeltas), session_id); })); } + /// Every delta of the transaction belongs to one of the subrequests above. Deltas left after the + /// last marker belong to no subrequest: they are already applied to the storage, and no response + /// would account for them, so they cannot be silently ignored either. + if (!deltas.empty()) + onStorageInconsistency("Unexpected deltas after the last subrequest of a Multi request"); + response->error = Coordination::Error::ZOK; return response; } diff --git a/tests/integration/test_keeper_empty_multi/__init__.py b/tests/integration/test_keeper_empty_multi/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_keeper_empty_multi/configs/keeper.xml b/tests/integration/test_keeper_empty_multi/configs/keeper.xml new file mode 100644 index 0000000000000..33d1f784a67c9 --- /dev/null +++ b/tests/integration/test_keeper_empty_multi/configs/keeper.xml @@ -0,0 +1,28 @@ + + + + node + 9181 + + + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + + + + + 1 + node + 9234 + + + + diff --git a/tests/integration/test_keeper_empty_multi/test.py b/tests/integration/test_keeper_empty_multi/test.py new file mode 100644 index 0000000000000..fe73c9f635cfa --- /dev/null +++ b/tests/integration/test_keeper_empty_multi/test.py @@ -0,0 +1,55 @@ +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/keeper.xml"], + stay_alive=True, + with_zookeeper=False, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + keeper_utils.wait_nodes(cluster, [node]) + yield cluster + finally: + cluster.shutdown() + + +def test_empty_multi_request(started_cluster): + # A `Multi` request whose subrequest list holds nothing but the terminator record - what a client + # sends when it builds a transaction from a list that turns out to be empty; kazoo serializes it + # as exactly that for a transaction without operations. It parses, and it is preprocessed into + # no deltas at all, so it used to be written to the changelog and then read `deltas.front()` on + # an empty range on the raft commit thread, terminating the process - and terminating it again + # on every restart that replayed the entry. + zk = keeper_utils.get_fake_zk(cluster, node.name) + try: + # ZooKeeper answers with an empty successful multi response, which kazoo turns into an + # empty list of results. + assert zk.transaction().commit() == [] + + assert keeper_utils.send_4lw_cmd(cluster, node, "ruok") == "imok" + + # The session and the server keep working. + zk.create("/after_empty_multi", b"1") + assert zk.get("/after_empty_multi")[0] == b"1" + # A multi transaction with subrequests keeps working. + transaction = zk.transaction() + transaction.create("/after_empty_multi/child", b"2") + assert transaction.commit() == ["/after_empty_multi/child"] + assert zk.get("/after_empty_multi/child")[0] == b"2" + finally: + zk.stop() + zk.close() + + # The entry of the empty multi is in the changelog, and the restart replays it. + node.restart_clickhouse() + keeper_utils.wait_nodes(cluster, [node]) + assert keeper_utils.send_4lw_cmd(cluster, node, "ruok") == "imok" From dab3b91861557feba08dce567c9e7f2f2e1f4a81 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 17 Sep 2026 22:03:29 +0000 Subject: [PATCH 021/296] Backport #119495 to 26.8: Fix parameterized view insert select legacy interpreter --- src/Interpreters/Context.cpp | 4 +++- ...ew_insert_select_parallel_distributed.reference | 1 + ...zed_view_insert_select_parallel_distributed.sql | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.reference create mode 100644 tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.sql diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 0db1f0c3577ef..756c5ee47bcdd 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -3074,7 +3074,9 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression, const create.set(create.sql_security, sql_security); auto view_context = view_metadata->getSQLSecurityOverriddenContext(shared_from_this()); - auto sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(query, view_context); + auto sample_block = getSettingsRef()[Setting::allow_experimental_analyzer] + ? InterpreterSelectQueryAnalyzer::getSampleBlock(query, view_context) + : InterpreterSelectWithUnionQuery::getSampleBlock(query, view_context); auto res = std::make_shared(StorageID(database_name, table_name), create, ColumnsDescription(sample_block->getNamesAndTypesList()), diff --git a/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.reference b/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.reference new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.sql b/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.sql new file mode 100644 index 0000000000000..9561e1a60a5c4 --- /dev/null +++ b/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.sql @@ -0,0 +1,14 @@ +-- INSERT SELECT from a parameterized view resolved the view through the legacy interpreter when +-- parallel_distributed_insert_select was enabled, so views only the analyzer can resolve failed. +SET enable_analyzer = 1; +SET parallel_distributed_insert_select = 2; + +CREATE TABLE t (id UInt8, p String) ENGINE = MergeTree ORDER BY id; +-- `SELECT i.*` over two joins yields columns named `i.p` in the legacy interpreter. +CREATE VIEW v AS SELECT count() AS c FROM (SELECT i.* FROM t AS i LEFT JOIN t AS a ON i.p = a.p LEFT JOIN t AS b ON i.p = b.p WHERE i.id = {id:UInt8}) AS i WHERE i.p = 'a'; +CREATE TABLE d (c UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/d', '1') ORDER BY c; + +INSERT INTO d SELECT * FROM v(id = 1); +SELECT * FROM d; + +DROP TABLE d SYNC; From 4cafeecb049ecb2848e0f39b89e00c3961e5a108 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 18 Sep 2026 11:23:49 +0000 Subject: [PATCH 022/296] Backport #119523 to 26.8: Check the read source grant and the host filter when a `HDFS` database serves a table from its cache --- src/Databases/DatabaseHDFS.cpp | 33 +++--- .../test_database_hdfs_read_grant/__init__.py | 0 .../configs/allowlist.xml | 5 + .../test_database_hdfs_read_grant/test.py | 101 ++++++++++++++++++ 4 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_database_hdfs_read_grant/__init__.py create mode 100644 tests/integration/test_database_hdfs_read_grant/configs/allowlist.xml create mode 100644 tests/integration/test_database_hdfs_read_grant/test.py diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp index 525a562a8f43b..726aa1282b76c 100644 --- a/src/Databases/DatabaseHDFS.cpp +++ b/src/Databases/DatabaseHDFS.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -105,15 +106,29 @@ bool DatabaseHDFS::checkUrl(const std::string & url, ContextPtr context_, bool t bool DatabaseHDFS::isTableExist(const String & name, ContextPtr context_) const { - std::lock_guard lock(mutex); - if (loaded_tables.contains(name)) - return true; + /// A name exists when it forms a URL this database may use, which needs no HDFS request. The cache + /// must not answer it: that reports which names other callers resolved, past any filter tightening. + if (source.empty() && !name.starts_with("hdfs://")) + return false; - return checkUrl(name, context_, false); + return checkUrl(getTablePath(name), context_, false); } StoragePtr DatabaseHDFS::getTableImpl(const String & name, ContextPtr context_) const { + auto url = getTablePath(name); + auto args = makeASTFunction("hdfs", make_intrusive(url)); + + auto table_function = TableFunctionFactory::instance().get(args, context_); + if (!table_function) + return nullptr; + + /// The cache is keyed on the name alone, so what authorizes a resolution is checked above it. The + /// grant is the table function's to check: a filtered grant matches the URI it reports, not the path. + table_function->checkSourceAccess(context_, /* is_insert_query */ false); + + checkUrl(url, context_, true); + /// Check if the table exists in the loaded tables map. { std::lock_guard lock(mutex); @@ -122,16 +137,6 @@ StoragePtr DatabaseHDFS::getTableImpl(const String & name, ContextPtr context_) return it->second; } - auto url = getTablePath(name); - - checkUrl(url, context_, true); - - auto args = makeASTFunction("hdfs", make_intrusive(url)); - - auto table_function = TableFunctionFactory::instance().get(args, context_); - if (!table_function) - return nullptr; - /// TableFunctionHDFS throws exceptions, if table cannot be created. auto table_storage = table_function->execute(args, context_, name); if (table_storage) diff --git a/tests/integration/test_database_hdfs_read_grant/__init__.py b/tests/integration/test_database_hdfs_read_grant/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_database_hdfs_read_grant/configs/allowlist.xml b/tests/integration/test_database_hdfs_read_grant/configs/allowlist.xml new file mode 100644 index 0000000000000..b6ed5de3a79a8 --- /dev/null +++ b/tests/integration/test_database_hdfs_read_grant/configs/allowlist.xml @@ -0,0 +1,5 @@ + + + allowed.example.com:9000 + + diff --git a/tests/integration/test_database_hdfs_read_grant/test.py b/tests/integration/test_database_hdfs_read_grant/test.py new file mode 100644 index 0000000000000..2a6f1b0d531e4 --- /dev/null +++ b/tests/integration/test_database_hdfs_read_grant/test.py @@ -0,0 +1,101 @@ +"""A `HDFS` database must not serve a table from its cache unchecked. + +The cache is keyed on the table name alone, so an entry one user resolved is handed to every later +caller. Both the read source grant and the remote host filter therefore have to be checked above it. +The same shape was fixed for the `Filesystem` database in +https://github.com/ClickHouse/ClickHouse/issues/118042. +""" + +import pytest + +from helpers.cluster import ClickHouseCluster, is_arm +from helpers.config_manager import ConfigManager + +if is_arm(): + pytestmark = pytest.mark.skip + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", with_hdfs=True, stay_alive=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_hdfs_database_cache_is_checked(started_cluster): + hdfs_api = started_cluster.hdfs_api + for name in ("warm.tsv", "cold.tsv", "cold2.tsv"): + hdfs_api.write_data("/" + name, "7\n") + + # The engine rejects a source that carries a path, so table names here are relative, which is the + # form the `EXISTS TABLE` arms below need. + node.query("CREATE DATABASE hdfs_db ENGINE = HDFS('hdfs://hdfs1:9000')") + # The source grant is the variable under test, so the table grant is deliberately broad and cannot + # confound the result. It also shows that it does not confer the source grant by itself. + node.query("CREATE USER u") + node.query("GRANT SELECT ON *.* TO u") + + # A user holding the grant resolves the name, which is what caches the storage for it. + assert node.query("SELECT * FROM hdfs_db.`warm.tsv`") == "7\n" + + # `tryGetTable` reports a denied resolution as `UNKNOWN_TABLE` (code 60). What this asserts is that + # the cached name is refused exactly like the name that was never resolved. + error = node.query_and_get_error("SELECT * FROM hdfs_db.`warm.tsv`", user="u") + assert "Code: 60." in error, error + error = node.query_and_get_error("SELECT * FROM hdfs_db.`cold.tsv`", user="u") + assert "Code: 60." in error, error + + # `EXISTS TABLE` needs only `SHOW TABLES`, so it must answer alike for a cached name and for one + # that was never resolved, or it reports which names other users have read. + assert node.query("EXISTS TABLE hdfs_db.`warm.tsv`", user="u") == "1\n" + assert node.query("EXISTS TABLE hdfs_db.`never.tsv`", user="u") == "1\n" + # A name that forms no usable URL still answers 0, and answers it to everyone: the two lines above + # are a measurement, and this probe does not depend on the grant. + assert node.query("EXISTS TABLE hdfs_db.`hdfs://hdfs1:9000`", user="u") == "0\n" + assert node.query("EXISTS TABLE hdfs_db.`hdfs://hdfs1:9000`") == "0\n" + + node.query("GRANT READ ON HDFS TO u") + + # With the grant the cached name is served. `TableFunctionExecute` counts calls of the table + # function, which serving from the cache does not make, so a counter that does not move is what + # shows the read was answered from the cache. The resolve below is the control that moves it. + calls = "SELECT sum(value) FROM system.events WHERE event = 'TableFunctionExecute'" + before = int(node.query(calls)) + assert node.query("SELECT * FROM hdfs_db.`warm.tsv`", user="u") == "7\n" + assert int(node.query(calls)) == before + assert node.query("SELECT * FROM hdfs_db.`cold2.tsv`") == "7\n" + assert int(node.query(calls)) > before + + # A grant restricted by URL must keep working: the filter is matched against the URI the table + # function reports, which for HDFS is the host of the table and not its path. + node.query("CREATE USER f") + node.query("GRANT SELECT ON *.* TO f") + node.query("GRANT READ ON HDFS('hdfs://hdfs1:9000') TO f") + assert node.query("SELECT * FROM hdfs_db.`warm.tsv`", user="f") == "7\n" + + # The host filter is re-read on `SYSTEM RELOAD CONFIG`, which keeps the cache, so a cached table + # must stop being served once its host is no longer allowed. These run as the fully granted user, + # so the grant cannot be the cause. + with ConfigManager() as cm: + cm.add_main_config(node, "configs/allowlist.xml") + error = node.query_and_get_error("SELECT * FROM hdfs_db.`warm.tsv`") + assert "Code: 60." in error, error + assert node.query("EXISTS TABLE hdfs_db.`warm.tsv`") == "0\n" + # A path that reports the refusal instead of masking it names the URL that was rejected. + error = node.query_and_get_error("INSERT INTO hdfs_db.`warm.tsv` VALUES (1)") + assert "UNACCEPTABLE_URL" in error, error + # The name that is not in the cache is refused the same way, as it already was. + error = node.query_and_get_error("SELECT * FROM hdfs_db.`cold.tsv`") + assert "Code: 60." in error, error + + # Removing the file and reloading again restores the answer, so the refusals above came from the + # filter and not from an evicted or poisoned cache entry. + assert node.query("SELECT * FROM hdfs_db.`warm.tsv`") == "7\n" + + node.query("DROP DATABASE hdfs_db SYNC") + node.query("DROP USER u, f") From 703dd38d53e051b5407f05b2e5e32ee2ebc5bea3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 18 Sep 2026 12:50:21 +0000 Subject: [PATCH 023/296] Backport #117981 to 26.8: Do not terminate the server on retryable errors while loading outdated parts --- src/Common/FailPoint.cpp | 2 + src/Storages/MergeTree/MergeTreeData.cpp | 107 ++++++++++++++++-- ...d_outdated_parts_retryable_error.reference | 4 + ...27_load_outdated_parts_retryable_error.sql | 33 ++++++ ..._outdated_parts_cancel_on_detach.reference | 3 + ...28_load_outdated_parts_cancel_on_detach.sh | 76 +++++++++++++ 6 files changed, 217 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.reference create mode 100644 tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.sql create mode 100644 tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.reference create mode 100755 tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.sh diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 25428b51a6307..5df9d5d8f3b17 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -282,6 +282,8 @@ static struct InitFiu REGULAR(rmt_merge_selecting_task_no_free_threads) \ REGULAR(rmt_merge_selecting_task_max_part_size) \ REGULAR(merge_tree_load_statistics_throw) \ + REGULAR(merge_tree_load_outdated_parts_retryable_error) \ + PAUSEABLE(merge_tree_load_outdated_parts_pause) \ PAUSEABLE(smt_mutate_task_pause_in_prepare) \ PAUSEABLE(smt_merge_selecting_task_pause_when_scheduled) \ REGULAR(smt_merge_selecting_task_reach_memory_limit) \ diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 54a70a4442208..0fe87551a1a4d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -430,6 +430,17 @@ namespace FailPoints /// transient error (e.g. temporary disk unavailability). Used to test that the refresh task /// reschedules itself after such an error instead of stopping permanently. extern const char merge_tree_refresh_parts_throw_once[]; + /// Throws a retryable error (`MEMORY_LIMIT_EXCEEDED`) while loading every outdated part in the + /// background. Used to test that the loading is retried later instead of terminating the server. + extern const char merge_tree_load_outdated_parts_retryable_error[]; + /// Pauses every worker that loads an outdated part in the background until the failpoint is disabled. + /// Used to cancel the loading (e.g. with `DETACH TABLE`) while the workers are in flight. + extern const char merge_tree_load_outdated_parts_pause[]; +} + +namespace ErrorCodes +{ + extern const int MEMORY_LIMIT_EXCEEDED; } static String getPartNameFromAST(const ASTPtr & partition) @@ -3498,10 +3509,42 @@ try /// Acquire shared lock because 'relative_data_path' is used while loading parts. TableLockHolder shared_lock; if (is_async) - shared_lock = lockForShare(RWLockImpl::NO_QUERY, (*getSettings())[MergeTreeSetting::lock_acquire_timeout_for_background_operations]); + { + shared_lock = tryLockForShare(RWLockImpl::NO_QUERY, (*getSettings())[MergeTreeSetting::lock_acquire_timeout_for_background_operations]); + if (!shared_lock) + { + /// The table is being dropped, detached or restarted (e.g. by `SYSTEM RESTART REPLICA`). + /// It is not an inconsistency of the set of parts, so retry later instead of terminating: + /// if the table is going away, the task is deactivated in `shutdown` anyway. + LOG_DEBUG(log, "Cannot lock the table to load outdated data parts because it is being dropped or restarted, will retry later"); + outdated_data_parts_loading_task->scheduleAfter(loading_parts_max_backoff_ms); + return; + } + } std::atomic_size_t num_loaded_parts = 0; + /// A retryable error (e.g. not enough memory or a network error) is not a sign of an inconsistent + /// set of parts, so the parts that failed with it are put back to the queue and loaded later. + /// + /// The workers collect them under a dedicated mutex and must not touch `outdated_data_parts_mutex`: + /// the cancellation branch below waits for all the workers while holding that mutex, + /// so a worker that takes it before its future becomes ready would deadlock + /// `DETACH`, `DROP`, shutdown and `SYSTEM RESTART REPLICA`. The failed parts are returned + /// to `outdated_unloaded_data_parts` only after all the workers have finished. + std::mutex failed_parts_mutex; + PartLoadingTreeNodes failed_parts; + std::exception_ptr retryable_exception; + std::atomic_bool has_retryable_exception = false; + + /// Must be called after `runner.waitForAllToFinishAndRethrowFirstError()` and under `outdated_data_parts_mutex`. + auto requeue_failed_parts = [&]() TSA_REQUIRES(outdated_data_parts_mutex) + { + std::lock_guard lock(failed_parts_mutex); + outdated_unloaded_data_parts.insert(outdated_unloaded_data_parts.end(), failed_parts.begin(), failed_parts.end()); + failed_parts.clear(); + }; + auto blocker = CannotAllocateThreadFaultInjector::blockFaultInjections(); ThreadPoolCallbackRunnerLocal runner(getOutdatedPartsLoadingThreadPool().get(), ThreadName::MERGETREE_LOAD_OUTDATED_PARTS); @@ -3520,6 +3563,7 @@ try /// Wait for every scheduled task /// In case of any exception it will be re-thrown and server will be terminated. runner.waitForAllToFinishAndRethrowFirstError(); + requeue_failed_parts(); LOG_DEBUG(log, "Stopped loading outdated data parts because task was canceled. " @@ -3527,22 +3571,47 @@ try return; } - if (outdated_unloaded_data_parts.empty()) + /// Do not start loading the remaining parts if the loading is going to be retried later anyway. + if (outdated_unloaded_data_parts.empty() || has_retryable_exception) break; part = outdated_unloaded_data_parts.back(); outdated_unloaded_data_parts.pop_back(); } - /// num_loaded_parts will outlive runner, so capturing by reference is ok - runner.enqueueAndKeepTrack([this, my_part = part, &num_loaded_parts, replicated]() + /// The captured locals will outlive runner, so capturing by reference is ok + runner.enqueueAndKeepTrack([this, my_part = part, &num_loaded_parts, &failed_parts_mutex, &failed_parts, &retryable_exception, &has_retryable_exception, replicated]() { auto blocker_for_runner_thread = CannotAllocateThreadFaultInjector::blockFaultInjections(); - auto res = loadDataPartWithRetries( - my_part->info, my_part->name, my_part->disk, - DataPartState::Outdated, data_parts_mutex, loading_parts_initial_backoff_ms, - loading_parts_max_backoff_ms, loading_parts_max_tries); + LoadPartResult res; + try + { + FailPointInjection::pauseFailPoint(FailPoints::merge_tree_load_outdated_parts_pause); + + fiu_do_on(FailPoints::merge_tree_load_outdated_parts_retryable_error, + { + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected retryable error while loading outdated part {}", my_part->name); + }); + + res = loadDataPartWithRetries( + my_part->info, my_part->name, my_part->disk, + DataPartState::Outdated, data_parts_mutex, loading_parts_initial_backoff_ms, + loading_parts_max_backoff_ms, loading_parts_max_tries); + } + catch (...) + { + if (!isRetryableException(std::current_exception())) + throw; + + /// The part is not added to the set of parts if the loading failed, so it can be loaded again from scratch. + std::lock_guard lock(failed_parts_mutex); + failed_parts.push_back(my_part); + if (!retryable_exception) + retryable_exception = std::current_exception(); + has_retryable_exception = true; + return; + } ++num_loaded_parts; if (res.is_broken) @@ -3559,6 +3628,28 @@ try runner.waitForAllToFinishAndRethrowFirstError(); + /// All the workers have finished, so no synchronization is needed to read `retryable_exception`. + if (has_retryable_exception) + { + size_t num_unloaded_parts = 0; + { + std::lock_guard lock(outdated_data_parts_mutex); + requeue_failed_parts(); + num_unloaded_parts = outdated_unloaded_data_parts.size(); + } + + /// Synchronous loading (on table drop) has no task to retry with, so it fails fast as before. + if (!is_async) + std::rethrow_exception(retryable_exception); + + LOG_WARNING(log, "Loading of outdated data parts was interrupted by a retryable error, will retry later. " + "Loaded {} parts, {} left unloaded. Error: {}", + num_loaded_parts.load(), num_unloaded_parts, getExceptionMessage(retryable_exception, /*with_stacktrace=*/ false)); + + outdated_data_parts_loading_task->scheduleAfter(loading_parts_max_backoff_ms); + return; + } + LOG_DEBUG(log, "Loaded {} outdated data parts {}", num_loaded_parts.load(), is_async ? "asynchronously" : "synchronously"); diff --git a/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.reference b/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.reference new file mode 100644 index 0000000000000..1d7c2d9d3d3d5 --- /dev/null +++ b/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.reference @@ -0,0 +1,4 @@ +3 +3 +0 +3 diff --git a/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.sql b/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.sql new file mode 100644 index 0000000000000..4ff781a873d42 --- /dev/null +++ b/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.sql @@ -0,0 +1,33 @@ +-- Tags: no-shared-merge-tree, no-parallel +-- no-shared-merge-tree: SharedMergeTree doesn't load inactive parts to memory after restart +-- no-parallel: SYSTEM ENABLE FAILPOINT is process-wide, and another copy of this test +-- toggling the failpoint would change the number of parts loaded in this one. + +DROP TABLE IF EXISTS t_load_outdated_parts; + +-- Outdated parts must survive DETACH/ATTACH to be loaded in the background after ATTACH. +CREATE TABLE t_load_outdated_parts (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS old_parts_lifetime = 600; + +INSERT INTO t_load_outdated_parts VALUES (1); +INSERT INTO t_load_outdated_parts VALUES (2); +INSERT INTO t_load_outdated_parts VALUES (3); +OPTIMIZE TABLE t_load_outdated_parts FINAL; + +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts' AND NOT active; + +DETACH TABLE t_load_outdated_parts; + +-- Every attempt to load an outdated part fails with a retryable error. +-- The server must not terminate, the loading must be retried later. +SYSTEM ENABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error; +ATTACH TABLE t_load_outdated_parts; + +SELECT count() FROM t_load_outdated_parts; +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts' AND NOT active; + +SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error; +SYSTEM WAIT LOADING PARTS t_load_outdated_parts; + +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts' AND NOT active; + +DROP TABLE t_load_outdated_parts; diff --git a/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.reference b/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.reference new file mode 100644 index 0000000000000..74452ce6831b8 --- /dev/null +++ b/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.reference @@ -0,0 +1,3 @@ +DETACH finished +All outdated parts are loaded +100 diff --git a/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.sh b/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.sh new file mode 100755 index 0000000000000..5f4e63145a3ed --- /dev/null +++ b/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Tags: no-shared-merge-tree, no-parallel +# no-shared-merge-tree: SharedMergeTree doesn't load inactive parts to memory after restart +# no-parallel: SYSTEM ENABLE FAILPOINT is process-wide, the pause failpoint would block the +# loading of outdated parts of the tables of other tests. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +function cleanup() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_pause" 2>/dev/null + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error" 2>/dev/null +} +trap cleanup EXIT + +# The loading of outdated parts is cancelled by DETACH TABLE while the loading workers are in flight, +# and then the workers fail with a retryable error. The DETACH must not hang: the workers must not +# take the mutex the cancellation branch waits under, and the failed parts must be requeued only +# after all the workers finished. +# +# The number of parts is greater than the size of the loading thread pool plus its queue +# (`max_outdated_parts_loading_thread_pool_size`, 32 each), so the dispatcher still has parts +# to hand out when the cancellation arrives and the cancellation branch is really exercised. +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS t_load_outdated_parts_cancel; + CREATE TABLE t_load_outdated_parts_cancel (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS old_parts_lifetime = 600; + SYSTEM STOP MERGES t_load_outdated_parts_cancel; + INSERT INTO t_load_outdated_parts_cancel SELECT number FROM numbers(100) + SETTINGS max_block_size = 1, min_insert_block_size_rows = 1, min_insert_block_size_bytes = 1, max_insert_threads = 1; + SYSTEM START MERGES t_load_outdated_parts_cancel; + OPTIMIZE TABLE t_load_outdated_parts_cancel FINAL; +" + +outdated_before=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts_cancel' AND NOT active") +[ "$outdated_before" -ge 100 ] || echo "Unexpected number of outdated parts: $outdated_before" + +$CLICKHOUSE_CLIENT --query " + DETACH TABLE t_load_outdated_parts_cancel; + SYSTEM ENABLE FAILPOINT merge_tree_load_outdated_parts_pause; + SYSTEM ENABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error; + ATTACH TABLE t_load_outdated_parts_cancel; + SYSTEM WAIT FAILPOINT merge_tree_load_outdated_parts_pause PAUSE; +" + +# Cancel the loading while the workers are paused. DETACH waits for the loading task to stop. +$CLICKHOUSE_CLIENT --query "DETACH TABLE t_load_outdated_parts_cancel" & +detach_pid=$! + +for _ in $(seq 1 300); do + detach_running=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.processes WHERE current_database = currentDatabase() AND query LIKE 'DETACH TABLE t_load_outdated_parts_cancel%'") + [ "$detach_running" = "1" ] && break + sleep 0.1 +done +# Give the DETACH a moment to reach the loading task and cancel it. +sleep 0.5 + +# Let the workers fail with the retryable error. +$CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_pause" + +wait $detach_pid +echo "DETACH finished" + +# Without the failpoints, all the outdated parts are loaded after ATTACH. +$CLICKHOUSE_CLIENT --query " + SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error; + ATTACH TABLE t_load_outdated_parts_cancel; + SYSTEM WAIT LOADING PARTS t_load_outdated_parts_cancel; +" + +outdated_after=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts_cancel' AND NOT active") +[ "$outdated_after" = "$outdated_before" ] && echo "All outdated parts are loaded" || echo "Outdated parts before: $outdated_before, after: $outdated_after" + +$CLICKHOUSE_CLIENT --query "SELECT count() FROM t_load_outdated_parts_cancel" +$CLICKHOUSE_CLIENT --query "DROP TABLE t_load_outdated_parts_cancel" From 360e534cd1f11020043983240d80442a1ea386be Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 18 Sep 2026 13:10:18 +0000 Subject: [PATCH 024/296] Backport #118496 to 26.8: Fix a text index direct read silently returning no rows while an update is pending --- .../optimizeDirectReadFromTextIndex.cpp | 17 ++- .../02346_text_index_bug106460.reference | 21 ++++ .../02346_text_index_bug106460.sql | 109 ++++++++++++++++++ 3 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02346_text_index_bug106460.reference create mode 100644 tests/queries/0_stateless/02346_text_index_bug106460.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp index ad24c8857abfd..a1f99567b6565 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp @@ -64,6 +64,7 @@ struct TextIndexReadInfo MergeTreeIndexPtr index_helper = nullptr; bool is_materialized = false; bool is_fully_materialized = false; + bool has_patched_parts = false; }; using TextIndexReadInfos = absl::flat_hash_map; @@ -213,6 +214,7 @@ void collectTextIndexReadInfos(const ReadFromMergeTree * read_from_merge_tree_st /// other partitions/parts not in `parts_with_ranges`, disabling direct text index reads even when /// the queried parts have no on-the-fly updates for the index columns. NameSet all_updated_columns; + bool has_patched_parts = false; for (const auto & part : unique_parts) { auto alter_conversions = MergeTreeData::getAlterConversionsForPart(part, mutations_snapshot, context @@ -222,8 +224,12 @@ void collectTextIndexReadInfos(const ReadFromMergeTree * read_from_merge_tree_st ); const auto & part_updated_columns = alter_conversions->getAllUpdatedColumns(); all_updated_columns.insert(part_updated_columns.begin(), part_updated_columns.end()); + has_patched_parts |= alter_conversions->hasPatches(); } + if (has_patched_parts) + LOG_TRACE(logger, "Cannot use direct reading from text index. Reason: a part has a pending patch"); + for (const auto & index : indexes->skip_indexes.useful_indices) { if (!index.index->isTextIndex()) @@ -246,7 +252,8 @@ void collectTextIndexReadInfos(const ReadFromMergeTree * read_from_merge_tree_st .condition = index.condition_template->generateUnsubstituted(), .index = &index, .is_materialized = num_materialized_parts > 0, - .is_fully_materialized = num_materialized_parts == unique_parts.size() + .is_fully_materialized = num_materialized_parts == unique_parts.size(), + .has_patched_parts = has_patched_parts }; } } @@ -586,9 +593,11 @@ class TextIndexDAGReplacer const bool is_index_analyzed = !require_index_analyzed_predicate || isIndexAnalyzedPredicate(index_name, info, canonical_node); - /// Use direct read only when enabled and the entry is direct-read-eligible (has `index`). Otherwise - /// just inject the tokenizer/preprocessor/postprocessor (no virtual column), same as None mode. - if (!direct_read_from_text_index || !info.index || search_query->getDirectReadMode() == TextIndexDirectReadMode::None) + /// Use direct read only when enabled and the entry is direct-read-eligible (has `index`) and has no + /// patched parts. Otherwise just inject the tokenizer/preprocessor/postprocessor (no virtual column), + /// same as None mode. + if (!direct_read_from_text_index || !info.index || info.has_patched_parts + || search_query->getDirectReadMode() == TextIndexDirectReadMode::None) { selected_conditions.emplace_back(search_query, index_name, String{}, &info, is_index_analyzed); used_index_columns.insert(index_header.begin()->name); diff --git a/tests/queries/0_stateless/02346_text_index_bug106460.reference b/tests/queries/0_stateless/02346_text_index_bug106460.reference new file mode 100644 index 0000000000000..d31e718aeb3c4 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_bug106460.reference @@ -0,0 +1,21 @@ +100 1 +100 1 +100 1 +1 1 +11 0 +21 0 +-- materialized +100 1 +1 +-- With preprocessor +0 +10 +0 +10 +patched indexed column +99 +99 +99 +1 +1 +1 diff --git a/tests/queries/0_stateless/02346_text_index_bug106460.sql b/tests/queries/0_stateless/02346_text_index_bug106460.sql new file mode 100644 index 0000000000000..04459c383bfe2 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_bug106460.sql @@ -0,0 +1,109 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: the distributed plan forces `use_skip_indexes_on_data_read = 0`, which turns +-- the direct read off, so those runs would only ever exercise the fallback path. + +-- Checks that a text index search returns the same rows while a lightweight update is pending on the +-- table as it does without, and as it does with direct read turned off. + +SET query_plan_direct_read_from_text_index = 1; +SET enable_lightweight_update = 1; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab ( + id UInt64, + c UInt64, + s String, + INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) +ENGINE = MergeTree ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 0; -- keep the patch pending + +INSERT INTO tab SELECT number, 0, concat('tok', toString(number % 10), ' word') FROM numbers(1000); + +SYSTEM STOP MERGES tab; +UPDATE tab SET c = 1 WHERE id < 10; +-- the update is now pending + +-- 100 rows match `tok1`, of which only `id = 1` was updated. +SELECT count(), sum(c) FROM tab WHERE hasToken(s, 'tok1') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT count(), sum(c) FROM tab WHERE hasToken(s, 'tok1') SETTINGS query_plan_direct_read_from_text_index = 0; +SELECT count(), sum(c) FROM tab WHERE hasToken(s, 'tok1') SETTINGS use_skip_indexes = 0; + +-- A second predicate alongside the search: the rows disappeared here too. +SELECT id, c FROM tab WHERE hasToken(s, 'tok1') AND id < 25 ORDER BY id; + +SELECT '-- materialized'; +SYSTEM START MERGES tab; +ALTER TABLE tab MODIFY SETTING apply_patches_on_merge = 1; +OPTIMIZE TABLE tab FINAL; + +SELECT count(), sum(c) FROM tab WHERE hasToken(s, 'tok1'); + +-- Direct read must be used +SELECT count() > 0 FROM +( + EXPLAIN actions = 1 SELECT count() FROM tab WHERE hasToken(s, 'tok1') +) WHERE explain LIKE '%\_\_text_index%'; + +DROP TABLE tab; + +SELECT '-- With preprocessor'; + +CREATE TABLE tab +( + id UInt64, + c UInt64, + s String, + INDEX idx_s s TYPE text(tokenizer = splitByNonAlpha, preprocessor = lower(s)) +) +ENGINE = MergeTree ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 0; + +INSERT INTO tab SELECT number, 0, if(number < 10, 'Hello World', 'foo bar') FROM numbers(1000); + +-- The preprocessor applies on the index path only, so these two legitimately differ. +SELECT count() FROM tab WHERE hasAnyTokens(s, 'hello') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE hasAnyTokens(s, 'hello') SETTINGS use_skip_indexes = 1; + +SYSTEM STOP MERGES tab; +UPDATE tab SET c = 1 WHERE id = 500; + +-- Same two answers with an update pending on `c`, which the index does not cover. +SELECT count() FROM tab WHERE hasAnyTokens(s, 'hello') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE hasAnyTokens(s, 'hello') SETTINGS use_skip_indexes = 1; + +DROP TABLE tab; + +-- Updating the indexed column itself. The search must see the new value. +SELECT 'patched indexed column'; + +CREATE TABLE tab ( + id UInt64, + s String, + INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) +ENGINE = MergeTree ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 0; + +INSERT INTO tab SELECT number, concat('tok', toString(number % 10), ' word') FROM numbers(1000); + +SYSTEM STOP MERGES tab; +UPDATE tab SET s = 'zebra word' WHERE id = 1; + +SELECT count() FROM tab WHERE hasToken(s, 'tok1') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT count() FROM tab WHERE hasToken(s, 'tok1') SETTINGS query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab WHERE hasToken(s, 'tok1') SETTINGS use_skip_indexes = 0; + +SELECT count() FROM tab WHERE hasToken(s, 'zebra') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT count() FROM tab WHERE hasToken(s, 'zebra') SETTINGS query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab WHERE hasToken(s, 'zebra') SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; From acb067c51729869ac8d90ec5e095cbef0d0047e5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 18 Sep 2026 14:08:30 +0000 Subject: [PATCH 025/296] Backport #120214 to 26.8: Keep the LIMIT block size shrink for queries with arrayJoin --- src/Planner/PlannerJoinTree.cpp | 60 ++++++++++++------- .../QueryPlan/ReadFromMergeTree.cpp | 3 +- src/Storages/SelectQueryInfo.h | 2 + ...rinks_block_size_with_array_join.reference | 7 +++ ...mit_shrinks_block_size_with_array_join.sql | 32 ++++++++++ 5 files changed, 80 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.reference create mode 100644 tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 2c27d8acef429..e4a6018359a4d 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1042,16 +1042,6 @@ UInt64 mainQueryNodeBlockSizeByLimit(const SelectQueryInfo & select_query_info) limit_offset = offset_uint->getUInt(0); } - /// `arrayJoin` in the projection expands one input row into several output rows after the - /// source has run. Capping the source to `limit + offset` rows would truncate input BEFORE - /// expansion, so hard consumers of `trivial_limit` (StorageLoop, system.zeros, generateRandom) - /// could drop output rows that the LIMIT should keep. See issue #82279 and the sibling guard - /// in `numbersLikeUtils::shouldPushdownLimit`. (The `ARRAY JOIN` clause is lowered to a - /// separate table expression in the analyzer, so it is not a single-table read and never - /// reaches this optimization.) - if (hasFunctionNode(main_query_node.getProjectionNode(), "arrayJoin")) - return 0; - /** If not specified DISTINCT, WHERE, GROUP BY, HAVING, ORDER BY, JOIN, LIMIT BY, LIMIT WITH TIES * but LIMIT is specified with UInt64 value, and limit + offset < max_block_size, * then as the block size we will use limit + offset (not to read more from the table than requested), @@ -1280,7 +1270,7 @@ void pushOrderByIntoView( /// source rows before the expansion runs, so if the top ordered rows have /// empty arrays the rewritten query would return too few rows instead of /// continuing to lower ordered rows to fill the `LIMIT`. Mirror the existing - /// guard in `mainQueryNodeBlockSizeByLimit`. + /// `trivial_limit` guard in `buildQueryPlanForTableExpression`. if (hasFunctionNode(outer->getProjectionNode(), "arrayJoin")) return; @@ -1676,6 +1666,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_ UInt64 max_block_size = settings[Setting::max_block_size]; UInt64 max_block_size_limited = 0; + /// LIMIT + OFFSET as the most rows the source has to produce, when that holds. + UInt64 max_source_rows = 0; if (is_single_table_expression && !select_query_options.only_analyze) { /** If not specified DISTINCT, WHERE, GROUP BY, HAVING, ORDER BY, JOIN, LIMIT BY, LIMIT WITH TIES @@ -1692,24 +1684,46 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_ max_block_size_limited = mainQueryNodeBlockSizeByLimit(select_query_info); if (max_block_size_limited) { - if (max_block_size_limited < max_block_size) + const bool has_array_join = hasFunctionNode(select_query_info.query_tree->as().getProjectionNode(), "arrayJoin"); + const bool shrink_block = max_block_size_limited < max_block_size; + if (shrink_block) { - max_block_size = std::max(1, max_block_size_limited); - max_streams = 1; - max_threads_execute_query = 1; + /// With `arrayJoin` the source cannot stop at the LIMIT, so over a long run of empty arrays it streams + /// every row anyway, and one-row blocks make that hundreds of times slower than the default block. + /// Keep a few hundred rows per block: still a small read, and the empty prefix stays cheap. + constexpr UInt64 min_block_size_above_array_join = 256; + if (has_array_join) + max_block_size = std::min(max_block_size, std::max(max_block_size_limited, min_block_size_above_array_join)); + else + max_block_size = std::max(1, max_block_size_limited); } - if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows != 0) + /// With `arrayJoin` the LIMIT does not bound the source rows, so only the block size shrinks (#82279). + if (!has_array_join) { - if (max_block_size_limited < select_query_info.local_storage_limits.local_limits.size_limits.max_rows) + max_source_rows = max_block_size_limited; + if (shrink_block) + { + max_streams = 1; + max_threads_execute_query = 1; + } + + if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows != 0) + { + if (max_block_size_limited < select_query_info.local_storage_limits.local_limits.size_limits.max_rows) + table_expression_query_info.trivial_limit = max_block_size_limited; + /// Ask to read just enough rows to make the max_rows limit effective (so it has a chance to be triggered). + else if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows < std::numeric_limits::max()) + table_expression_query_info.trivial_limit = 1 + select_query_info.local_storage_limits.local_limits.size_limits.max_rows; + } + else + { table_expression_query_info.trivial_limit = max_block_size_limited; - /// Ask to read just enough rows to make the max_rows limit effective (so it has a chance to be triggered). - else if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows < std::numeric_limits::max()) - table_expression_query_info.trivial_limit = 1 + select_query_info.local_storage_limits.local_limits.size_limits.max_rows; + } } else { - table_expression_query_info.trivial_limit = max_block_size_limited; + table_expression_query_info.small_limit_above_array_join = shrink_block; } } @@ -2590,8 +2604,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_ if (table_expression_query_info.trivial_limit > 0 && table_expression_query_info.trivial_limit < rows_to_read) rows_to_read = table_expression_query_info.trivial_limit; - if (max_block_size_limited && (max_block_size_limited < rows_to_read)) - rows_to_read = max_block_size_limited; + if (max_source_rows && (max_source_rows < rows_to_read)) + rows_to_read = max_source_rows; const size_t number_of_replicas_to_use = rows_to_read / settings[Setting::parallel_replicas_min_number_of_rows_per_replica]; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 81a5ea1c95e56..db565127c1525 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -707,7 +707,8 @@ Pipe ReadFromMergeTree::readFromPool( * Because time spend during filling per thread tasks can be greater than whole query * execution for big tables with small limit. */ - bool use_prefetched_read_pool = query_info.trivial_limit == 0 && (allow_prefetched_remote || allow_prefetched_local); + bool use_prefetched_read_pool = query_info.trivial_limit == 0 && !query_info.small_limit_above_array_join + && (allow_prefetched_remote || allow_prefetched_local); if (use_prefetched_read_pool) { diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 150e34dee225c..a86ca8b23ce9c 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -212,6 +212,8 @@ struct SelectQueryInfo // If not 0, that means it's a trivial limit query. UInt64 trivial_limit = 0; + /// A trivial limit query whose rows `arrayJoin` expands: the source must not stop at the limit, but should read small. + bool small_limit_above_array_join = false; /// For IStorageSystemOneBlock std::vector columns_mask; diff --git a/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.reference b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.reference new file mode 100644 index 0000000000000..e38f471d7a816 --- /dev/null +++ b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.reference @@ -0,0 +1,7 @@ +1 +256 +0 +9000 +9001 +9002 +1 diff --git a/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql new file mode 100644 index 0000000000000..a4d19a379fddf --- /dev/null +++ b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql @@ -0,0 +1,32 @@ +-- LIMIT still shrinks the block size with arrayJoin, only the source-side limit stays off (#82279) +DROP TABLE IF EXISTS t_aj_limit; +CREATE TABLE t_aj_limit (k UInt64, a Array(UInt64)) ENGINE = MergeTree ORDER BY k SETTINGS index_granularity = 8; +INSERT INTO t_aj_limit SELECT number, [number] FROM numbers(200000); + +SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 FORMAT Null SETTINGS max_threads = 8, log_comment = '05182_limit'; + +SYSTEM FLUSH LOGS query_log; +SELECT argMax(read_rows, event_time_microseconds) < 5000 FROM system.query_log +WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '05182_limit'; + +-- the block does not shrink below a few hundred rows, so a long run of empty arrays is not streamed one row at a time +SELECT DISTINCT bs FROM (SELECT arrayJoin(a), blockSize() AS bs FROM t_aj_limit LIMIT 3 SETTINGS max_threads = 1, enable_parallel_replicas = 0); + +-- the prefetched pool sizes its reads by marks, not by the block size, so a small LIMIT keeps it off +SELECT countIf(explain LIKE '%PrefetchedReadPool%') FROM (EXPLAIN PIPELINE SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 SETTINGS allow_prefetched_read_pool_for_local_filesystem = 1, local_filesystem_read_method = 'pread_threadpool', max_threads = 8, merge_tree_min_rows_for_concurrent_read = 1, merge_tree_min_bytes_for_concurrent_read = 1); + +DROP TABLE t_aj_limit; + +-- a long empty-array prefix: the LIMIT is tiny but the source has to get past the prefix +DROP TABLE IF EXISTS t_aj_sparse; +CREATE TABLE t_aj_sparse (k UInt64, a Array(UInt64)) ENGINE = MergeTree ORDER BY k SETTINGS index_granularity = 8; +INSERT INTO t_aj_sparse SELECT number, if(number < 9000, [], [number]) FROM numbers(10000); + +-- one local stream reads in key order; with parallel replicas the order and the initiator's read_rows are not deterministic +SELECT arrayJoin(a) FROM t_aj_sparse LIMIT 3 SETTINGS max_threads = 1, enable_parallel_replicas = 0, log_comment = '05182_sparse'; + +SYSTEM FLUSH LOGS query_log; +SELECT argMax(read_rows, event_time_microseconds) >= 9000 FROM system.query_log +WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '05182_sparse'; + +DROP TABLE t_aj_sparse; From f26359bdc3b042025a6c469ede3040c13de5ae73 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 18 Sep 2026 14:48:43 +0000 Subject: [PATCH 026/296] Backport #119846 to 26.8: Reject paths with an embedded NUL byte in filesystem containment checks --- src/Common/filesystemHelpers.cpp | 17 ++++ .../gtest_path_starts_with_embedded_nul.cpp | 78 +++++++++++++++++++ .../Local/LocalObjectStorage.cpp | 8 ++ ...e_path_relatively_to_base_embedded_nul.cpp | 76 ++++++++++++++++++ src/Storages/StorageFile.cpp | 16 ++++ ...object_storage_path_embedded_nul.reference | 3 + ..._local_object_storage_path_embedded_nul.sh | 32 ++++++++ ...12_path_containment_embedded_nul.reference | 4 + .../05212_path_containment_embedded_nul.sh | 28 +++++++ .../05213_storage_file_embedded_nul.reference | 5 ++ .../05213_storage_file_embedded_nul.sh | 32 ++++++++ 11 files changed, 299 insertions(+) create mode 100644 src/Common/tests/gtest_path_starts_with_embedded_nul.cpp create mode 100644 src/Disks/tests/gtest_resolve_path_relatively_to_base_embedded_nul.cpp create mode 100644 tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference create mode 100755 tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh create mode 100644 tests/queries/0_stateless/05212_path_containment_embedded_nul.reference create mode 100755 tests/queries/0_stateless/05212_path_containment_embedded_nul.sh create mode 100644 tests/queries/0_stateless/05213_storage_file_embedded_nul.reference create mode 100755 tests/queries/0_stateless/05213_storage_file_embedded_nul.sh diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 94c132db1b677..fb0624911e94a 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -217,8 +217,22 @@ String getFilesystemName([[maybe_unused]] const String & mount_point) #endif } +/// A path with an embedded NUL is malformed, and, more importantly, it cannot be validated: the +/// comparisons below see the whole value, while every syscall the path is later passed to (`open`, +/// `mkdir`, `stat`) stops at the first NUL. A path shaped as `\0/` would therefore be reported as contained in the prefix while it addresses ``, +/// anywhere on the filesystem. Report such a path as not contained, so that every containment check +/// fails closed. +static bool containsEmbeddedNul(const std::filesystem::path & path) +{ + return path.native().contains('\0'); +} + bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path) { + if (containsEmbeddedNul(path) || containsEmbeddedNul(prefix_path)) + return false; + auto rel = fs::relative(path, prefix_path); if (rel.empty() || rel == "..") return false; @@ -240,6 +254,9 @@ static bool fileOrSymlinkPathStartsWith(const std::filesystem::path & path, cons /// `.` and `..` and extra `/`. Path is not canonized because otherwise path will /// not be a path of a symlink itself. + if (containsEmbeddedNul(path) || containsEmbeddedNul(prefix_path)) + return false; + auto rel = fs::absolute(path).lexically_normal().lexically_relative(fs::absolute(prefix_path).lexically_normal()); if (rel.empty() || rel == "..") diff --git a/src/Common/tests/gtest_path_starts_with_embedded_nul.cpp b/src/Common/tests/gtest_path_starts_with_embedded_nul.cpp new file mode 100644 index 0000000000000..3a9373ab3f46f --- /dev/null +++ b/src/Common/tests/gtest_path_starts_with_embedded_nul.cpp @@ -0,0 +1,78 @@ +#include + +#include + +#include +#include + +#include /// for ::getpid + +namespace fs = std::filesystem; + +namespace +{ + +/// The containment helpers compare the whole string, while every syscall a path is later passed to stops at +/// the first NUL byte. The tests below build the shape that abuses the difference: as a whole the path +/// normalizes back into the prefix, truncated at the NUL it addresses `probe` next to the prefix. +struct EmbeddedNulPaths +{ + fs::path prefix; + std::string contained; + std::string escaping_through_nul; + + EmbeddedNulPaths() + : prefix(fs::temp_directory_path() / ("path_starts_with_embedded_nul_" + std::to_string(::getpid()))) + { + fs::create_directories(prefix); + std::string back_into_prefix = "/../" + prefix.filename().string() + "/file"; + contained = (prefix / ".." / "probe").string() + back_into_prefix; + escaping_through_nul = (prefix / ".." / "probe").string() + std::string(1, '\0') + back_into_prefix; + } + + ~EmbeddedNulPaths() + { + std::error_code ec; + fs::remove_all(prefix, ec); + } +}; + +std::string withEmbeddedNul(const fs::path & path) +{ + return path.string() + std::string(1, '\0') + "suffix"; +} + +} + +TEST(PathStartsWithEmbeddedNul, PathStartsWith) +{ + EmbeddedNulPaths paths; + const std::string prefix = paths.prefix.string(); + + /// Positive control: the same traversal without a NUL is contained, so it is the NUL that flips the result. + EXPECT_TRUE(DB::pathStartsWith(paths.contained, prefix)); + EXPECT_TRUE(DB::pathStartsWith(fs::path(paths.contained), paths.prefix)); + EXPECT_TRUE(DB::pathStartsWith((paths.prefix / "file").string(), prefix)); + + EXPECT_FALSE(DB::pathStartsWith(paths.escaping_through_nul, prefix)); + EXPECT_FALSE(DB::pathStartsWith(fs::path(paths.escaping_through_nul), paths.prefix)); + + /// A NUL anywhere in the path, even where it cannot escape, and a NUL in the prefix are rejected too. + EXPECT_FALSE(DB::pathStartsWith(withEmbeddedNul(paths.prefix / "file"), prefix)); + EXPECT_FALSE(DB::pathStartsWith((paths.prefix / "file").string(), withEmbeddedNul(paths.prefix))); + EXPECT_FALSE(DB::pathStartsWith(fs::path((paths.prefix / "file").string()), fs::path(withEmbeddedNul(paths.prefix)))); +} + +TEST(PathStartsWithEmbeddedNul, FileOrSymlinkPathStartsWith) +{ + EmbeddedNulPaths paths; + const std::string prefix = paths.prefix.string(); + + EXPECT_TRUE(DB::fileOrSymlinkPathStartsWith(paths.contained, prefix)); + EXPECT_TRUE(DB::fileOrSymlinkPathStartsWith((paths.prefix / "file").string(), prefix)); + + EXPECT_FALSE(DB::fileOrSymlinkPathStartsWith(paths.escaping_through_nul, prefix)); + + EXPECT_FALSE(DB::fileOrSymlinkPathStartsWith(withEmbeddedNul(paths.prefix / "file"), prefix)); + EXPECT_FALSE(DB::fileOrSymlinkPathStartsWith((paths.prefix / "file").string(), withEmbeddedNul(paths.prefix))); +} diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp index 8d29f6a9a85d3..41ffed349e10d 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp @@ -138,6 +138,14 @@ LocalObjectStorage::LocalObjectStorage(LocalObjectStorageSettings settings_) String resolvePathRelativelyToBase(const String & path, const String & base_path) { + /// A path with an embedded NUL cannot be validated: `std::string` and `fs::path` compare the whole + /// value, while every syscall the resolved path is later passed to (`open`, `mkdir`, `stat`) stops at + /// the NUL. A path shaped as `\0/` would therefore + /// pass the containment check below and still make the kernel operate on ``, anywhere on the + /// filesystem. `listObjects` rejects such a path for its own reason - keep both checks. + if (path.contains('\0')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path contains an embedded NUL byte"); + auto configured_base = fs::path(base_path).lexically_normal(); auto is_inside = [&](const String & candidate) diff --git a/src/Disks/tests/gtest_resolve_path_relatively_to_base_embedded_nul.cpp b/src/Disks/tests/gtest_resolve_path_relatively_to_base_embedded_nul.cpp new file mode 100644 index 0000000000000..4786b1ef090f1 --- /dev/null +++ b/src/Disks/tests/gtest_resolve_path_relatively_to_base_embedded_nul.cpp @@ -0,0 +1,76 @@ +#include + +#include +#include + +#include +#include + +#include /// for ::getpid + +namespace fs = std::filesystem; + +namespace DB::ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int PATH_ACCESS_DENIED; +} + +namespace +{ + +struct ScopedBaseDir +{ + fs::path path; + + ScopedBaseDir() + : path(fs::temp_directory_path() / ("resolve_path_relatively_to_base_embedded_nul_" + std::to_string(::getpid()))) + { + fs::create_directories(path); + } + + ~ScopedBaseDir() + { + std::error_code ec; + fs::remove_all(path, ec); + } +}; + +int errorCodeOf(const std::string & path, const std::string & base) +{ + try + { + DB::resolvePathRelativelyToBase(path, base); + } + catch (const DB::Exception & e) + { + return e.code(); + } + return 0; +} + +} + +TEST(ResolvePathRelativelyToBaseEmbeddedNul, RejectsEmbeddedNul) +{ + ScopedBaseDir base; + const std::string base_path = base.path.string(); + const std::string back_into_base = "/../" + base.path.filename().string() + "/file"; + + /// Positive controls: the resolver still accepts a relative path and an absolute traversal that ends up inside. + EXPECT_EQ(DB::resolvePathRelativelyToBase("file", base_path), (base.path / "file").string()); + const std::string contained = (base.path / ".." / "probe").string() + back_into_base; + EXPECT_EQ(DB::resolvePathRelativelyToBase(contained, base_path), contained); + + /// Negative control: a plain traversal outside is denied by the containment check. + EXPECT_EQ(errorCodeOf((base.path / ".." / "probe").string(), base_path), DB::ErrorCodes::PATH_ACCESS_DENIED); + + /// As a whole string, this path normalizes into the base directory; truncated at the NUL, as every syscall + /// would see it, it addresses `probe` next to it. It must be rejected before any containment comparison. + const std::string escaping_through_nul = (base.path / ".." / "probe").string() + std::string(1, '\0') + back_into_base; + EXPECT_EQ(errorCodeOf(escaping_through_nul, base_path), DB::ErrorCodes::BAD_ARGUMENTS); + + /// A NUL that cannot escape is rejected all the same, in a relative and in an absolute path. + EXPECT_EQ(errorCodeOf(std::string("file\0suffix", 11), base_path), DB::ErrorCodes::BAD_ARGUMENTS); + EXPECT_EQ(errorCodeOf((base.path / "file").string() + std::string(1, '\0') + "suffix", base_path), DB::ErrorCodes::BAD_ARGUMENTS); +} diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 79e97e33900f9..64565be6ad13d 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -345,6 +345,16 @@ std::string getTablePath(const std::string & table_dir_path, const std::string & return table_dir_path + "/data." + escapeForFileName(format_name); } +/// Every syscall a path is passed to (`stat`, `open`, `opendir`) stops at the first NUL byte, while the +/// containment checks see the whole value. A path with an embedded NUL therefore addresses a location the +/// checks never look at, so it is rejected before the first filesystem probe. The path is not echoed in the +/// message: it would put the NUL byte into the logs. +void throwIfPathContainsEmbeddedNul(const std::string & path) +{ + if (path.contains('\0')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "File path contains an embedded NUL byte"); +} + /// Both db_dir_path and table_path must be converted to absolute paths (in particular, path cannot contain '..'). void checkCreationIsAllowed( ContextPtr context_global, @@ -352,6 +362,8 @@ void checkCreationIsAllowed( const std::string & table_path, bool can_be_directory) { + throwIfPathContainsEmbeddedNul(table_path); + if (context_global->getApplicationType() != Context::ApplicationType::SERVER) return; @@ -399,6 +411,10 @@ std::pair splitToArchivePathAndPathInArchive(const String & sour /// Finds files matching a specified pattern with globs. Strings getPathsList(const String & path_with_globs, const String & user_files_path, const ContextPtr & context, size_t & total_bytes_to_read) { + /// The listing below stats the path and expands its globs on the filesystem before the matched paths + /// are checked for containment in `user_files_path`, so the path must be rejected here, not there. + throwIfPathContainsEmbeddedNul(path_with_globs); + fs::path user_files_absolute_path = fs::weakly_canonical(user_files_path); fs::path fs_pattern(path_with_globs); if (fs_pattern.is_relative()) diff --git a/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference new file mode 100644 index 0000000000000..9f845b0366429 --- /dev/null +++ b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference @@ -0,0 +1,3 @@ +-- a path with an embedded NUL is rejected +-- the same for icebergLocal +-- a path inside user_files is still accepted (and fails for its own reason) diff --git a/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh new file mode 100755 index 0000000000000..492ed2d68abdb --- /dev/null +++ b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: Requires DeltaLake +# Tag no-msan: DeltaKernel is not compiled with msan + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A path with an embedded NUL byte must not pass the `user_files` containment check: the check sees the +# whole string, while every syscall the path is later passed to stops at the NUL. The path below is +# normalized into `user_files` as a whole string, but addresses `/../probe_$CLICKHOUSE_DATABASE` +# once truncated at the NUL. + +TAIL="../$(basename "${USER_FILES_PATH}")/nul_${CLICKHOUSE_DATABASE}" + +echo "-- a path with an embedded NUL is rejected" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM deltaLakeLocal('${USER_FILES_PATH}/../probe_${CLICKHOUSE_DATABASE}\0/${TAIL}') -- { serverError PATH_ACCESS_DENIED } +" + +echo "-- the same for icebergLocal" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM icebergLocal('${USER_FILES_PATH}/../probe_${CLICKHOUSE_DATABASE}\0/${TAIL}') -- { serverError PATH_ACCESS_DENIED } +" + +echo "-- a path inside user_files is still accepted (and fails for its own reason)" +mkdir -p "${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM deltaLakeLocal('${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}') -- { serverError DELTA_KERNEL_ERROR } +" +rmdir "${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" diff --git a/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference b/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference new file mode 100644 index 0000000000000..056f0839d8ecf --- /dev/null +++ b/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference @@ -0,0 +1,4 @@ +-- the file function checks with fileOrSymlinkPathStartsWith only +data +-- the Filesystem database engine checks with pathStartsWith only +data diff --git a/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh b/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh new file mode 100755 index 0000000000000..978a90a4ccba7 --- /dev/null +++ b/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A path with an embedded NUL byte must be rejected by every filesystem containment check: the check sees +# the whole string, while every syscall the path is later passed to stops at the NUL. Each case below names +# an existing entry inside `user_files` followed by a NUL and a suffix, so that a check which ignored the +# NUL would let the truncated path through and the query would succeed. + +DIR="${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +mkdir -p "${DIR}" +printf 'data' > "${DIR}/file.csv" + +echo "-- the file function checks with fileOrSymlinkPathStartsWith only" +${CLICKHOUSE_CLIENT} --query "SELECT file('nul_${CLICKHOUSE_DATABASE}/file.csv')" +${CLICKHOUSE_CLIENT} --query "SELECT file('nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix') -- { serverError DATABASE_ACCESS_DENIED }" +${CLICKHOUSE_CLIENT} --query "SELECT file('${DIR}/file.csv\0suffix') -- { serverError DATABASE_ACCESS_DENIED }" + +echo "-- the Filesystem database engine checks with pathStartsWith only" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('${DIR}\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('nul_${CLICKHOUSE_DATABASE}\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('${DIR}')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM nul_${CLICKHOUSE_DATABASE}.\`file.csv\`" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE nul_${CLICKHOUSE_DATABASE}" + +rm -r "${DIR}" diff --git a/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference b/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference new file mode 100644 index 0000000000000..3fab3d2e8c89f --- /dev/null +++ b/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference @@ -0,0 +1,5 @@ +-- the file table function +data +-- the File engine +data +data diff --git a/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh b/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh new file mode 100755 index 0000000000000..79adfea697447 --- /dev/null +++ b/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The `file` table function and the `File` engine stat the path and expand its globs on the filesystem before +# the matched paths are checked for containment in `user_files`, and every syscall stops at the first NUL byte. +# A path with an embedded NUL must therefore be rejected up front, before anything touches the filesystem. +# Each case names an existing entry followed by a NUL and a suffix, so that the truncated path would succeed. + +DIR="${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +mkdir -p "${DIR}" +printf 'data\n' > "${DIR}/file.csv" + +echo "-- the file table function" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('nul_${CLICKHOUSE_DATABASE}/file.csv', CSV, 'x String')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/file.csv\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/*\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/file.csv\0suffix.zip :: file.csv', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION file('${DIR}/file.csv\0suffix', CSV, 'x String') VALUES ('more') -- { serverError BAD_ARGUMENTS }" + +echo "-- the File engine" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, 'nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, '${DIR}/file.csv\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, 'nul_${CLICKHOUSE_DATABASE}/file.csv')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM nul_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} --query "DROP TABLE nul_${CLICKHOUSE_DATABASE}" + +cat "${DIR}/file.csv" +rm -r "${DIR}" From 69d3b91de655e9310f92e2e647e9273d4013ae2a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 18 Sep 2026 15:17:47 +0000 Subject: [PATCH 027/296] Update autogenerated version to 26.8.7.19 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 99f0c1d87ca71..202244fd2c8db 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54519) +SET(VERSION_REVISION 54520) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 7) -SET(VERSION_GITHASH 4c31daf60f098627e7f9a867bfa2ab5889aafd74) -SET(VERSION_DESCRIBE v26.8.7.1-lts) -SET(VERSION_STRING 26.8.7.1) +SET(VERSION_PATCH 8) +SET(VERSION_GITHASH f29d27def7ee0adb10e7e2fff080ba204054dd4b) +SET(VERSION_DESCRIBE v26.8.8.1-lts) +SET(VERSION_STRING 26.8.8.1) # end of autochange From a704aed4741facb2a3958f7f3a7fb5e1b14ab3e2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 18 Sep 2026 18:24:43 +0000 Subject: [PATCH 028/296] Backport #116539 to 26.8: Bump `cctz` to tzdata 2026c --- contrib/cctz | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/cctz b/contrib/cctz index c2ba12b73531a..8e694da054a9a 160000 --- a/contrib/cctz +++ b/contrib/cctz @@ -1 +1 @@ -Subproject commit c2ba12b73531a7dbc3ac45b5007649e35760f6d8 +Subproject commit 8e694da054a9a31d98392bf03ee188b04f810d0a From d740654028446da2574b61d9f4a92cc961d5185a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 10:17:16 +0000 Subject: [PATCH 029/296] Backport #120935 to 26.8: Do not dereference the optional copy properties of an asynchronous Azure copy --- .../copyAzureBlobStorageFile.cpp | 12 +- ...st_azure_copy_blob_optional_properties.cpp | 155 ++++++++++++++++++ 2 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 src/IO/tests/gtest_azure_copy_blob_optional_properties.cpp diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 7c036bf4041b7..a43b91dc86e9a 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -436,16 +436,22 @@ void copyAzureBlobStorageFile( auto copy_status = properties_model.CopyStatus; auto copy_status_description = properties_model.CopyStatusDescription; - + /// `CopySource` and `CopyStatusDescription` are optional in the properties of a blob: + /// the SDK models them as `Nullable`, and `Nullable::Value()` of an empty one aborts the + /// process in a release build (`AZURE_ASSERT_MSG` expands to a bare `std::abort` under + /// `NDEBUG`). The properties polled here come from the remote endpoint, which is under no + /// obligation to send either header, so nothing below dereferences them unchecked. if (copy_status.HasValue() && copy_status.Value() == Azure::Storage::Blobs::Models::CopyStatus::Success) { - LOG_TRACE(log, "Copy of {} to {} finished", properties_model.CopySource.Value(), dest_blob); + LOG_TRACE(log, "Copy of {} to {} finished", src_blob, dest_blob); } else { if (copy_status.HasValue()) throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Copy from {} to {} failed with status {} description {} (operation is done {})", - src_blob, dest_blob, copy_status.Value().ToString(), copy_status_description.Value(), operation.IsDone()); + src_blob, dest_blob, copy_status.Value().ToString(), + copy_status_description.HasValue() ? copy_status_description.Value() : String(""), + operation.IsDone()); throw Exception( ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Copy from {} to {} didn't complete with success status (operation is done {})", diff --git a/src/IO/tests/gtest_azure_copy_blob_optional_properties.cpp b/src/IO/tests/gtest_azure_copy_blob_optional_properties.cpp new file mode 100644 index 0000000000000..b0c8e706645ea --- /dev/null +++ b/src/IO/tests/gtest_azure_copy_blob_optional_properties.cpp @@ -0,0 +1,155 @@ +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace +{ + +constexpr size_t blob_size = 100; + +/// A body stream that owns what it serves. Every response needs one: the transport policy of the +/// SDK buffers the body by calling `ReadToEnd` on it unconditionally, so a response without a body +/// stream dereferences a null pointer - including the answer to a HEAD request, which has no body. +class OwningBodyStream : public Azure::Core::IO::BodyStream +{ +public: + explicit OwningBodyStream(std::string data_) : data(std::move(data_)) { } + + int64_t Length() const override { return static_cast(data.size()); } + + void Rewind() override { position = 0; } + +private: + size_t OnRead(uint8_t * buffer, size_t count, const Azure::Core::Context &) override + { + const size_t to_read = std::min(count, data.size() - position); + if (to_read != 0) + memcpy(buffer, data.data() + position, to_read); + position += to_read; + return to_read; + } + + std::string data; + size_t position = 0; +}; + +/// An endpoint that accepts an asynchronous `Copy Blob` (`StartCopyFromUri`) and reports it as +/// completed in the polled properties of the destination, without the optional `x-ms-copy-source` +/// header. The SDK models `BlobProperties::CopySource` as `Nullable`, and `Nullable::Value()` of an +/// empty one aborts the process in a release build (`AZURE_ASSERT_MSG` expands to a bare +/// `std::abort` under `NDEBUG`), so an endpoint behaving this way must not be able to take the +/// server down. +class CopyWithoutCopySourceTransport : public Azure::Core::Http::HttpTransport +{ +public: + std::unique_ptr Send(Azure::Core::Http::Request & request, const Azure::Core::Context &) override + { + /// `Copy Blob`: a `PUT` of the destination with `x-ms-copy-source`. Accepted, still pending. + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Put) + { + ++copies_started; + auto response = std::make_unique(1, 1, Azure::Core::Http::HttpStatusCode::Accepted, "Accepted"); + response->SetHeader("Content-Length", "0"); + response->SetHeader("ETag", "\"0x8DA000000000001\""); + response->SetHeader("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("x-ms-copy-id", "copy-id"); + response->SetHeader("x-ms-copy-status", "pending"); + response->SetBodyStream(std::make_unique("")); + return response; + } + + /// The properties of the destination, polled until the copy completes. These are the headers + /// the SDK reads from such a response; `x-ms-copy-source` is left out on purpose. + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Head) + { + ++properties_polled; + auto response = std::make_unique(1, 1, Azure::Core::Http::HttpStatusCode::Ok, "OK"); + response->SetHeader("Content-Length", std::to_string(blob_size)); + response->SetHeader("ETag", "\"0x8DA000000000001\""); + response->SetHeader("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("x-ms-creation-time", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-lease-state", "available"); + response->SetHeader("x-ms-lease-status", "unlocked"); + response->SetHeader("x-ms-server-encrypted", "true"); + response->SetHeader("x-ms-copy-id", "copy-id"); + response->SetHeader("x-ms-copy-status", "success"); + response->SetHeader("x-ms-copy-progress", std::to_string(blob_size) + "/" + std::to_string(blob_size)); + response->SetHeader("x-ms-copy-completion-time", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetBodyStream(std::make_unique("")); + return response; + } + + /// Anything else would be the read-and-write fallback, which must not be reached: the native + /// copy completed. + ++unexpected_requests; + auto response = std::make_unique(1, 1, Azure::Core::Http::HttpStatusCode::NotFound, "Not Found"); + response->SetHeader("Content-Length", "0"); + response->SetBodyStream(std::make_unique("")); + return response; + } + + size_t copies_started = 0; + size_t properties_polled = 0; + size_t unexpected_requests = 0; +}; + +} + +TEST(AzureNativeCopy, CompletionWithoutTheCopySourceInTheProperties) +{ + /// The copy source is logged at the trace level, and a `LOG_TRACE` evaluates its arguments only + /// when the logger is at that level - which is the default level of the server, so this is the + /// configuration in which the property is dereferenced. + getLogger("copyAzureBlobStorageFile")->setLevel("trace"); + + auto transport = std::make_shared(); + + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = transport; + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + auto settings = std::make_shared(); + settings->use_native_copy = true; + /// A blob of at least `max_single_part_copy_size` bytes is copied with the asynchronous `Copy Blob`, + /// whose completion is read from the polled properties. + settings->max_single_part_copy_size = blob_size; + + ASSERT_NO_THROW(DB::copyAzureBlobStorageFile( + container_client, + container_client, + /* src_container_for_logging */ "container", + /* src_blob */ "blob", + /* src_size */ blob_size, + /* dest_container_for_logging */ "container", + /* dest_blob */ "copy", + settings, + DB::ReadSettings{}, + /* object_to_attributes */ std::nullopt)); + + ASSERT_EQ(transport->copies_started, 1u); + ASSERT_GE(transport->properties_polled, 1u); + ASSERT_EQ(transport->unexpected_requests, 0u); +} + +#endif From 327a6a8a00c15f1e45337483a97244b29206fe57 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 10:19:43 +0000 Subject: [PATCH 030/296] Backport #118712 to 26.8: Make 01076_parallel_alter_replicated_zookeeper actually assign mutations --- ...rallel_alter_replicated_zookeeper.reference | 1 + ...1076_parallel_alter_replicated_zookeeper.sh | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference index ff9c6824f00ad..c7200be9f3178 100644 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference @@ -15,3 +15,4 @@ Finishing alters 0 1 0 +1 diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index d995dbf731e7f..9e2b81423db37 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -48,14 +48,13 @@ done INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_mutate_mt_1") -# Run mutation on random replica +# Run mutation on replica 1, which detach_attach_thread never takes down function correct_alter_thread() { local TIMELIMIT=$((SECONDS+TIMEOUT)) while [ $SECONDS -lt "$TIMELIMIT" ] do - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_$REPLICA UPDATE value1 = value1 + 1 WHERE 1"; + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_1 UPDATE value1 = value1 + 1 WHERE 1"; sleep 1 done } @@ -79,7 +78,9 @@ function detach_attach_thread() local TIMELIMIT=$((SECONDS+TIMEOUT)) while [ $SECONDS -lt "$TIMELIMIT" ] do - REPLICA=$(($RANDOM % 5 + 1)) + # Replicas 2.. only: correct_alter_thread needs replica 1 attached, or an ALTER that draws a + # DETACHed replica fails with UNKNOWN_TABLE and the run assigns no mutation to check. + REPLICA=$(($RANDOM % (REPLICAS - 1) + 2)) $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_mutate_mt_$REPLICA" sleep 0.$RANDOM sleep 0.$RANDOM @@ -142,10 +143,15 @@ for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_mutate_mt_$i" $CLICKHOUSE_CLIENT --query "CHECK TABLE concurrent_mutate_mt_$i" &> /dev/null # if we will remove something the output of select will be wrong $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_mutate_mt_$i" - $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE table='concurrent_mutate_mt_$i' and is_done=0" # all mutations have to be done - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table='concurrent_mutate_mt_$i' and is_done=0" # for verbose output + $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_mt_$i' and is_done=0" # all mutations have to be done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_mt_$i' and is_done=0" # for verbose output done +# The stress phase must have assigned at least one mutation: every check above is also satisfied by a +# run that assigned none, because the concurrent inserts alone raise the sum and a run without +# mutations has none unfinished. The SYSTEM SYNC REPLICA calls above make the entries visible here. +$CLICKHOUSE_CLIENT --query "SELECT count() > 0 FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' AND table LIKE 'concurrent_mutate_mt_%'" + for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_mutate_mt_$i" done From 03b17f7ab7e1cf8d6ceaa0004c8a672910e5a2b9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 13:05:11 +0000 Subject: [PATCH 031/296] Backport #116998 to 26.8: Bound PostgreSQL protocol messages by their declared size --- src/Core/PostgreSQLProtocol.h | 152 +++++++++++------ src/Core/tests/gtest_postgresql_protocol.cpp | 158 +++++++++++++++--- ...postgresql_protocol_message_size.reference | 6 + .../05054_postgresql_protocol_message_size.sh | 158 ++++++++++++++++++ 4 files changed, 404 insertions(+), 70 deletions(-) create mode 100644 tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference create mode 100755 tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 706465d5e20e5..0adea1b15bb52 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -173,6 +173,25 @@ class ColumnTypeSpec ColumnTypeSpec convertDataTypeToPostgresColumnTypeSpec(const DataTypePtr & data_type); +/// Reads exactly `size` bytes into `s`. The size is declared by the client and the payload may +/// never arrive, so the string grows as the bytes are received instead of being resized to the +/// declared size up front: otherwise a tiny packet declaring a huge field makes the server +/// allocate that much and then wait for data that never comes. +inline void readStringOfDeclaredSize(String & s, size_t size, ReadBuffer & in) +{ + s.clear(); + while (s.size() < size) + { + if (in.eof()) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Message from client declares a field of {} bytes, but contains only {}", size, s.size()); + + const size_t bytes_to_copy = std::min(size - s.size(), in.available()); + s.append(in.position(), bytes_to_copy); + in.position() += bytes_to_copy; + } +} + class MessageTransport { private: @@ -286,6 +305,40 @@ class FrontMessage : public IMessage * (if type is provided for the message by the protocol). */ virtual void deserialize(ReadBuffer & in) = 0; + +protected: + template + static void deserializePayload(ReadBuffer & in, std::string_view message_name, F && deserialize_payload) + { + Int32 size = 0; + readBinaryBigEndian(size, in); + if (size < 4) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong message length {} in {}, it must be at least 4", size, message_name); + + const size_t payload_size = static_cast(size - 4); + LimitReadBuffer payload_in(in, {.read_no_less = payload_size, .read_no_more = payload_size}); + try + { + deserialize_payload(payload_in); + } + catch (...) + { + /// Keep the stream aligned before the handler starts discarding messages through `Sync`. + /// When the frame itself could not be read (the client closed the connection before sending + /// the declared bytes), the buffer is canceled and there is nothing left to align. + if (!payload_in.isCanceled()) + payload_in.ignore(payload_size - payload_in.count()); + throw; + } + + const size_t unread_payload_bytes = payload_size - payload_in.count(); + payload_in.ignore(unread_payload_bytes); + if (unread_payload_bytes != 0) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong message length {} in {}, it has {} unexpected trailing payload bytes", + size, message_name, unread_payload_bytes); + } }; class BackendMessage : public IMessage, public ISerializable @@ -419,7 +472,7 @@ class Terminate : FrontMessage public: void deserialize(ReadBuffer & in) override { - in.ignore(4); + deserializePayload(in, "Terminate message", [](ReadBuffer &) {}); } MessageType getMessageType() const override @@ -534,22 +587,18 @@ class SASLInitialResponse : public Messaging::FrontMessage void deserialize(ReadBuffer & in) override { - UInt8 message_type = 0; - readBinaryBigEndian(message_type, in); - Int32 size = 0; - readBinaryBigEndian(size, in); - readNullTerminated(auth_method, in); - Int32 size_sasl_mechanism = 0; - readBinaryBigEndian(size_sasl_mechanism, in); - /// -1 is the protocol sentinel for "no initial response"; any other negative value is malformed. - if (size_sasl_mechanism < -1) - throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, - "Wrong SASL mechanism length {} in SASLInitialResponse, it must not be less than -1", size_sasl_mechanism); - if (size_sasl_mechanism > 0) + deserializePayload(in, "SASLInitialResponse message", [this](ReadBuffer & payload_in) { - sasl_mechanism.resize(size_sasl_mechanism); - in.readStrict(sasl_mechanism.data(), size_sasl_mechanism); - } + readNullTerminated(auth_method, payload_in); + Int32 size_sasl_mechanism = 0; + readBinaryBigEndian(size_sasl_mechanism, payload_in); + /// -1 is the protocol sentinel for "no initial response"; any other negative value is malformed. + if (size_sasl_mechanism < -1) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong SASL mechanism length {} in SASLInitialResponse, it must not be less than -1", size_sasl_mechanism); + if (size_sasl_mechanism > 0) + readStringOfDeclaredSize(sasl_mechanism, size_sasl_mechanism, payload_in); + }); } MessageType getMessageType() const override @@ -594,15 +643,10 @@ class SASLResponse : public Messaging::FrontMessage void deserialize(ReadBuffer & in) override { - UInt8 message_type = 0; - readBinaryBigEndian(message_type, in); - Int32 size = 0; - readBinaryBigEndian(size, in); - if (size < 4) - throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, - "Wrong message length {} in SASLResponse, it must be at least 4", size); - sasl_mechanism.resize(size - 4); - in.readStrict(sasl_mechanism.data(), size - 4); + deserializePayload(in, "SASLResponse message", [this](ReadBuffer & payload_in) + { + readStringUntilEOF(sasl_mechanism, payload_in); + }); } MessageType getMessageType() const override @@ -640,9 +684,10 @@ class PasswordMessage : FrontMessage void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); - readNullTerminated(password, in); + deserializePayload(in, "PasswordMessage", [this](ReadBuffer & payload_in) + { + readNullTerminated(password, payload_in); + }); } MessageType getMessageType() const override @@ -720,9 +765,10 @@ class Query : FrontMessage void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); - readNullTerminated(query, in); + deserializePayload(in, "Query message", [this](ReadBuffer & payload_in) + { + readNullTerminated(query, payload_in); + }); } MessageType getMessageType() const override @@ -836,8 +882,8 @@ class BindQuery : FrontMessage parameters.emplace_back(std::nullopt); continue; } - String current_param(sz_param, 0); - in.readStrict(current_param.data(), sz_param); + String current_param; + readStringOfDeclaredSize(current_param, sz_param, in); parameters.push_back(std::move(current_param)); } @@ -1000,8 +1046,11 @@ class SyncQuery : FrontMessage public: void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); + Int32 size = 0; + readBinaryBigEndian(size, in); + if (size != 4) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong message length {} in Sync message, it must be 4", size); } MessageType getMessageType() const override @@ -1149,9 +1198,10 @@ class CopyDataQuery : FrontMessage void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); - readNullTerminated(query, in); + deserializePayload(in, "CopyData message", [this](ReadBuffer & payload_in) + { + readNullTerminated(query, payload_in); + }); } MessageType getMessageType() const override @@ -1219,18 +1269,10 @@ class CopyInData : FrontMessage void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); - if (sz < static_cast(sizeof(Int32))) - throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, - "Wrong message length {} in CopyData, it must be at least 4", sz); - query.reserve(sz - sizeof(Int32)); - for (size_t i = 0; i < sz - sizeof(Int32); ++i) + deserializePayload(in, "CopyData message", [this](ReadBuffer & payload_in) { - char byte = 0; - readBinary(byte, in); - query.push_back(byte); - } + readStringUntilEOF(query, payload_in); + }); } MessageType getMessageType() const override @@ -1557,6 +1599,16 @@ class CleartextPasswordAuth : public AuthenticationMethod class ScrambleSHA256Auth : public AuthenticationMethod { + /// Both SASL messages of the SCRAM exchange are sent with the `PasswordMessage` type byte. + static void expectPasswordMessage(Messaging::MessageTransport & mt) + { + Messaging::FrontMessageType type = mt.receiveMessageType(); + if (type != Messaging::FrontMessageType::PASSWORD_MESSAGE) + throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, + "Client sent wrong message or closed the connection. Message byte was {}.", + static_cast(type)); + } + static size_t findPatternPosition(const String & key, const String & pattern) { size_t pos = key.size(); @@ -1668,6 +1720,7 @@ class ScrambleSHA256Auth : public AuthenticationMethod String auth_message; mt.send(Messaging::AuthenticationSASL(), true); + expectPasswordMessage(mt); auto rsp = mt.receive(); auto server_nonce = generateNonce(); @@ -1690,6 +1743,7 @@ class ScrambleSHA256Auth : public AuthenticationMethod auto sasl_continue_message = fmt::format("r={},s={},i={}", nonce, salt, num_iterations); mt.send(Messaging::AuthenticationSASLContinue(sasl_continue_message), true); auth_message += "," + sasl_continue_message; + expectPasswordMessage(mt); auto rsp_continue = mt.receive(); auto proof = parseProof(rsp_continue->sasl_mechanism); auto proof_position = findProofPosition(rsp_continue->sasl_mechanism); diff --git a/src/Core/tests/gtest_postgresql_protocol.cpp b/src/Core/tests/gtest_postgresql_protocol.cpp index 0e1e955968350..719a4fe8567fd 100644 --- a/src/Core/tests/gtest_postgresql_protocol.cpp +++ b/src/Core/tests/gtest_postgresql_protocol.cpp @@ -28,11 +28,6 @@ namespace PreparedStatements = DB::PostgreSQLProtocol::PostgresPreparedStatement namespace { -void putUInt8(std::string & s, UInt8 v) -{ - s.push_back(static_cast(v)); -} - void putInt16(std::string & s, Int16 v) { s.push_back(static_cast((v >> 8) & 0xFF)); @@ -45,6 +40,14 @@ void putInt32(std::string & s, Int32 v) s.push_back(static_cast((v >> (8 * i)) & 0xFF)); } +std::string framePayload(std::string payload) +{ + std::string bytes; + putInt32(bytes, static_cast(4 + payload.size())); + bytes += payload; + return bytes; +} + /// Run `body` over the bytes and report whether it threw UNKNOWN_PACKET_FROM_CLIENT. template bool throwsUnknownPacket(const std::string & bytes, F && body) @@ -62,6 +65,31 @@ bool throwsUnknownPacket(const std::string & bytes, F && body) } } +template +void expectTrailingPayloadIsRejectedAndAligned(std::string payload) +{ + /// These bytes look like a `Sync` frame, but they are trailing bytes in the current message. + payload.append("S\0\0\0\4", 5); + std::string bytes = framePayload(std::move(payload)); + bytes.push_back('X'); + + ReadBufferFromMemory in(bytes.data(), bytes.size()); + TMessage msg; + try + { + msg.deserialize(in); + FAIL() << "Expected UNKNOWN_PACKET_FROM_CLIENT"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); + } + + char marker = 0; + in.readStrict(marker); + EXPECT_EQ(marker, 'X'); +} + } TEST(PostgreSQLProtocol, DropMessageRejectsLengthBelowFour) @@ -91,10 +119,10 @@ TEST(PostgreSQLProtocol, DropMessageRejectsLengthBelowFour) TEST(PostgreSQLProtocol, SASLResponseRejectsLengthBelowFour) { + /// The message type byte is read by `receiveMessageType`, so the frame starts at the length. for (Int32 size = 0; size < 4; ++size) { std::string bytes; - putUInt8(bytes, 'p'); putInt32(bytes, size); EXPECT_TRUE(throwsUnknownPacket(bytes, [](ReadBuffer & in) { @@ -104,27 +132,38 @@ TEST(PostgreSQLProtocol, SASLResponseRejectsLengthBelowFour) } /// size == 4 means an empty SASL payload. - std::string bytes; - putUInt8(bytes, 'p'); - putInt32(bytes, 4); - ReadBufferFromMemory in(bytes.data(), bytes.size()); - Messaging::SASLResponse msg; - EXPECT_NO_THROW(msg.deserialize(in)); - EXPECT_TRUE(msg.sasl_mechanism.empty()); + { + std::string bytes = framePayload(""); + ReadBufferFromMemory in(bytes.data(), bytes.size()); + Messaging::SASLResponse msg; + EXPECT_NO_THROW(msg.deserialize(in)); + EXPECT_TRUE(msg.sasl_mechanism.empty()); + } + + /// The payload is the rest of the frame, with no terminator, and nothing beyond it. + { + std::string bytes = framePayload("c=biws,r=nonce,p=proof"); + bytes.push_back('X'); + ReadBufferFromMemory in(bytes.data(), bytes.size()); + Messaging::SASLResponse msg; + EXPECT_NO_THROW(msg.deserialize(in)); + EXPECT_EQ(msg.sasl_mechanism, "c=biws,r=nonce,p=proof"); + + char marker = 0; + in.readStrict(marker); + EXPECT_EQ(marker, 'X'); + } } TEST(PostgreSQLProtocol, SASLInitialResponseHandlesMechanismLength) { auto build = [](Int32 size_sasl_mechanism, const std::string & data) { - std::string bytes; - putUInt8(bytes, 'p'); - putInt32(bytes, 0); /// the outer size field is not used for bounds here - bytes += "SCRAM-SHA-256"; - bytes.push_back('\0'); - putInt32(bytes, size_sasl_mechanism); - bytes += data; - return bytes; + std::string payload = "SCRAM-SHA-256"; + payload.push_back('\0'); + putInt32(payload, size_sasl_mechanism); + payload += data; + return framePayload(std::move(payload)); }; /// Below -1 is malformed. @@ -151,6 +190,14 @@ TEST(PostgreSQLProtocol, SASLInitialResponseHandlesMechanismLength) EXPECT_NO_THROW(msg.deserialize(in)); EXPECT_EQ(msg.sasl_mechanism, "abc"); } + + /// A mechanism declaring more bytes than the frame carries is rejected on the frame boundary, + /// instead of the server allocating the declared size and waiting for bytes that never come. + EXPECT_TRUE(throwsUnknownPacket(build(1000000, "abc"), [](ReadBuffer & in) + { + Messaging::SASLInitialResponse msg; + msg.deserialize(in); + })); } TEST(PostgreSQLProtocol, BindHandlesParameterLength) @@ -195,6 +242,14 @@ TEST(PostgreSQLProtocol, BindHandlesParameterLength) ASSERT_EQ(msg.parameters.size(), 1u); EXPECT_EQ(msg.parameters[0], "hi"); } + + /// A parameter declaring more bytes than the frame carries is rejected on the frame boundary, + /// instead of the server allocating the declared size up front. + EXPECT_TRUE(throwsUnknownPacket(build(1000000, "hi"), [](ReadBuffer & in) + { + Messaging::BindQuery msg; + msg.deserialize(in); + })); } TEST(PostgreSQLProtocol, BindRejectsNegativeCounts) @@ -980,3 +1035,64 @@ TEST(PostgreSQLProtocol, CopyDataRejectsLengthBelowFour) EXPECT_NO_THROW(msg.deserialize(in)); EXPECT_EQ(msg.query, "ab"); } + +TEST(PostgreSQLProtocol, MessagesRejectFrameShorterThanDeclared) +{ + /// The declared length is a frame boundary in both directions: a frame that ends before it - + /// the client declared more bytes than it sent and then closed the write side - must be + /// rejected, not parsed from the bytes that did arrive. This holds whether the parser stops on + /// its own terminator (`Query`, `PasswordMessage`) or reads to the end of the frame + /// (`SASLResponse`, `CopyInData`). + auto truncated = [](const std::string & payload) + { + std::string bytes; + /// A thousand bytes of the declared frame are never sent. + putInt32(bytes, static_cast(payload.size() + sizeof(Int32) + 1000)); + bytes += payload; + return bytes; + }; + + auto expect_throws = [&](const std::string & payload, auto && parse) + { + std::string bytes = truncated(payload); + ReadBufferFromMemory in(bytes.data(), bytes.size()); + EXPECT_THROW(parse(in), Exception); + }; + + std::string query_payload = "SELECT 1"; + query_payload.push_back('\0'); + expect_throws(query_payload, [](ReadBuffer & in) + { + Messaging::Query msg; + msg.deserialize(in); + }); + + std::string password_payload = "x"; + password_payload.push_back('\0'); + expect_throws(password_payload, [](ReadBuffer & in) + { + Messaging::PasswordMessage msg; + msg.deserialize(in); + }); + + expect_throws("c=biws,r=nonce,p=proof", [](ReadBuffer & in) + { + Messaging::SASLResponse msg; + msg.deserialize(in); + }); + + expect_throws("ab", [](ReadBuffer & in) + { + Messaging::CopyInData msg; + msg.deserialize(in); + }); +} + +TEST(PostgreSQLProtocol, PasswordMessageRejectsTrailingPayload) +{ + /// A `PasswordMessage` whose password ends before the declared boundary must not let the + /// remainder be taken for the next message. + std::string password_payload = "hunter2"; + password_payload.push_back('\0'); + expectTrailingPayloadIsRejectedAndAligned(std::move(password_payload)); +} diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference new file mode 100644 index 0000000000000..ec35dc29b3b47 --- /dev/null +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference @@ -0,0 +1,6 @@ +SASL mechanism longer than the message: error response +password without a terminator: error response +well-formed exchange: query result received +query with a smuggled tail: error response +query shorter than declared: error response +oversized sync: error response diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh new file mode 100755 index 0000000000000..e5fd34c2db3ff --- /dev/null +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the PostgreSQL compatibility port is not enabled in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Authentication messages are parsed before the client is authenticated, so a field of a message +# must not be allocated at the size the client declares for it, and parsing a message must not +# continue past the size the client declared for the message. + +USER_SCRAM="user_scram_${CLICKHOUSE_DATABASE}" +USER_PLAIN="user_plain_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT --query "CREATE USER ${USER_SCRAM} IDENTIFIED WITH scram_sha256_password BY 'x'" +$CLICKHOUSE_CLIENT --query "CREATE USER ${USER_PLAIN} IDENTIFIED WITH plaintext_password BY 'x'" + +CLICKHOUSE_PORT_POSTGRESQL="$CLICKHOUSE_PORT_POSTGRESQL" USER_SCRAM="$USER_SCRAM" USER_PLAIN="$USER_PLAIN" python3 - <<'PYTHON' +import os +import socket +import struct + +port = int(os.environ["CLICKHOUSE_PORT_POSTGRESQL"]) +user_scram = os.environ["USER_SCRAM"] +user_plain = os.environ["USER_PLAIN"] + + +def connect(): + sock = socket.create_connection(("127.0.0.1", port), timeout=30) + sock.settimeout(30) + return sock + + +def startup(sock, user): + payload = ("user\x00" + user + "\x00\x00").encode() + sock.sendall(struct.pack(">ii", 8 + len(payload), 196608) + payload) + + +def outcome(sock): + """Drain the reply and describe it: a rejected message gets an error response, an accepted one + that is never completed leaves the server waiting for the payload.""" + data = b"" + while True: + try: + chunk = sock.recv(4096) + except socket.timeout: + return "TIMEOUT" + except ConnectionResetError: + break + if not chunk: + break + data += chunk + if data[0:1] == b"E": + return "error response" + if not data: + return "NO REPLY" + return "UNEXPECTED REPLY " + repr(data[:64]) + + +def first_reply(sock): + """Describe the first reply message, for a case where the server rejects the message but keeps + the connection open, so waiting for it to close would only hit the timeout.""" + try: + data = sock.recv(4096) + except (socket.timeout, ConnectionResetError): + return "TIMEOUT" + if data[0:1] == b"E": + return "error response" + if not data: + return "NO REPLY" + return "UNEXPECTED REPLY " + repr(data[:64]) + + +# A SASL initial response that declares a two-gigabyte mechanism in a thirty-byte message. +sock = connect() +startup(sock, user_scram) +sock.recv(4096) +body = b"SCRAM-SHA-256\x00" + struct.pack(">i", 0x7FFFFFFF) +sock.sendall(b"p" + struct.pack(">i", 4 + len(body)) + body) +print("SASL mechanism longer than the message:", outcome(sock)) +sock.close() + +# A password message that declares a small size and then streams a password without a terminator. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 10)) +try: + sock.sendall(b"x" * 200000) +except (BrokenPipeError, ConnectionResetError, socket.timeout): + pass +print("password without a terminator:", outcome(sock)) +sock.close() + +# A well-formed exchange still works: authenticate and run a query. +def read_until_ready(sock): + data = b"" + while not data.endswith(b"Z\x00\x00\x00\x05I"): + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + + +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +sock.sendall(b"Q" + struct.pack(">i", 4 + 9) + b"SELECT 1\x00") +data = read_until_ready(sock) +print("well-formed exchange:", "query result received" if b"C\x00\x00\x00\rSELECT 1\x00" in data else "UNEXPECTED REPLY " + repr(data[:96])) +sock.close() + +# A `Query` that terminates its string early and smuggles a whole second `Query` into the tail of the +# same declared frame. The declared length is a frame boundary, so the tail must be rejected instead +# of being read as the next message. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"SELECT 1\x00" + b"SELECT 2\x00" +sock.sendall(b"Q" + struct.pack(">i", 4 + len(body)) + body) +print("query with a smuggled tail:", first_reply(sock)) +sock.close() + +# A `Query` that declares more bytes than the client sends and then closes the write side of the +# connection. The declared length is a frame boundary in both directions, so the message must be +# rejected instead of being executed with the part of the payload that did arrive. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"SELECT 1\x00" +sock.sendall(b"Q" + struct.pack(">i", 4 + len(body) + 1000) + body) +sock.shutdown(socket.SHUT_WR) +print("query shorter than declared:", outcome(sock)) +sock.close() + +# An oversized `Sync`, whose parser reads nothing at all: its payload must not survive the message +# boundary and be reinterpreted as the next message. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"Q" + struct.pack(">i", 4 + 9) + b"SELECT 2\x00" +sock.sendall(b"S" + struct.pack(">i", 4 + len(body)) + body) +print("oversized sync:", outcome(sock)) +sock.close() +PYTHON + +$CLICKHOUSE_CLIENT --query "DROP USER ${USER_SCRAM}" +$CLICKHOUSE_CLIENT --query "DROP USER ${USER_PLAIN}" From 348d7f59770aa5034fc286b7d2f205d1206128b4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 15:04:59 +0000 Subject: [PATCH 032/296] Update autogenerated version to 26.8.8.8 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 202244fd2c8db..09f28a267b846 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54520) +SET(VERSION_REVISION 54521) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 8) -SET(VERSION_GITHASH f29d27def7ee0adb10e7e2fff080ba204054dd4b) -SET(VERSION_DESCRIBE v26.8.8.1-lts) -SET(VERSION_STRING 26.8.8.1) +SET(VERSION_PATCH 9) +SET(VERSION_GITHASH 0de36d42e8bbc78fdaf2bbf90a2853b5e9301bdc) +SET(VERSION_DESCRIBE v26.8.9.1-lts) +SET(VERSION_STRING 26.8.9.1) # end of autochange From f34055efe8b4ded9bc1b77af3557cfc4a017be08 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 17:23:37 +0000 Subject: [PATCH 033/296] Backport #116723 to 26.8: Parallelize Iceberg data manifest decoding --- programs/local/LocalServer.cpp | 8 + programs/server/Server.cpp | 13 + src/Common/CurrentMetrics.cpp | 3 + src/Common/setThreadName.h | 1 + src/Core/ServerSettings.cpp | 17 + src/Core/Settings.cpp | 15 +- src/Core/SettingsChangesHistory.cpp | 2 + src/IO/SharedThreadPools.h | 3 +- src/Interpreters/PreparedSets.cpp | 5 + src/Interpreters/PreparedSets.h | 4 + .../DataLakes/Iceberg/IcebergIterator.cpp | 376 +++++++++--------- .../DataLakes/Iceberg/IcebergIterator.h | 87 ++-- .../Iceberg/ManifestFileIterator.cpp | 15 +- .../DataLakes/Iceberg/ManifestFileIterator.h | 10 +- .../__init__.py | 0 .../configs/decode_pool.xml | 3 + .../test_iceberg_manifest_decode_pool/test.py | 98 +++++ .../test_data_manifest_decode_concurrency.py | 349 ++++++++++++++++ tests/performance/iceberg_data_manifests.xml | 220 ++++++++++ 19 files changed, 996 insertions(+), 233 deletions(-) create mode 100644 tests/integration/test_iceberg_manifest_decode_pool/__init__.py create mode 100644 tests/integration/test_iceberg_manifest_decode_pool/configs/decode_pool.xml create mode 100644 tests/integration/test_iceberg_manifest_decode_pool/test.py create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_data_manifest_decode_concurrency.py create mode 100644 tests/performance/iceberg_data_manifests.xml diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index ca93459ba7c50..99ee12067d987 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -212,6 +212,9 @@ namespace ServerSetting extern const ServerSettingsUInt64 max_format_parsing_thread_pool_size; extern const ServerSettingsUInt64 max_format_parsing_thread_pool_free_size; extern const ServerSettingsUInt64 format_parsing_thread_pool_queue_size; + extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_size; + extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_free_size; + extern const ServerSettingsUInt64 iceberg_manifest_decode_thread_pool_queue_size; extern const ServerSettingsUInt64 page_cache_history_window_ms; extern const ServerSettingsString page_cache_policy; extern const ServerSettingsDouble page_cache_size_ratio; @@ -462,6 +465,11 @@ void LocalServer::initialize(Poco::Util::Application & self) server_settings[ServerSetting::max_format_parsing_thread_pool_size], server_settings[ServerSetting::max_format_parsing_thread_pool_free_size], server_settings[ServerSetting::format_parsing_thread_pool_queue_size]); + + getIcebergManifestDecodeThreadPool().initialize( + server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_size], + server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], + server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8ef0e09deca40..ad78a0d204a21 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -425,6 +425,9 @@ namespace ServerSetting extern const ServerSettingsUInt64 max_format_parsing_thread_pool_size; extern const ServerSettingsUInt64 max_format_parsing_thread_pool_free_size; extern const ServerSettingsUInt64 format_parsing_thread_pool_queue_size; + extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_size; + extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_free_size; + extern const ServerSettingsUInt64 iceberg_manifest_decode_thread_pool_queue_size; extern const ServerSettingsUInt64 page_cache_history_window_ms; extern const ServerSettingsString page_cache_policy; extern const ServerSettingsDouble page_cache_size_ratio; @@ -1989,6 +1992,11 @@ try server_settings[ServerSetting::max_format_parsing_thread_pool_free_size], server_settings[ServerSetting::format_parsing_thread_pool_queue_size]); + getIcebergManifestDecodeThreadPool().initialize( + server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_size], + server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], + server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + std::string path_str = getCanonicalPath(String(server_settings[ServerSetting::path]), original_working_directory); fs::path path = path_str; @@ -3037,6 +3045,11 @@ try new_server_settings[ServerSetting::max_format_parsing_thread_pool_free_size], new_server_settings[ServerSetting::format_parsing_thread_pool_queue_size]); + getIcebergManifestDecodeThreadPool().reloadConfiguration( + new_server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_size], + new_server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], + new_server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + global_context->setMergeWorkload(new_server_settings[ServerSetting::merge_workload]); global_context->setMutationWorkload(new_server_settings[ServerSetting::mutation_workload]); global_context->setThrowOnUnknownWorkload(new_server_settings[ServerSetting::throw_on_unknown_workload]); diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 10e6f69d5ce5e..15e08b2903a9f 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -287,6 +287,9 @@ M(FormatParsingThreads, "Number of threads in the thread pool used for parsing input.") \ M(FormatParsingThreadsActive, "Number of threads in the thread pool used for parsing input running a task.") \ M(FormatParsingThreadsScheduled, "Number of queued or active jobs in the thread pool used for parsing input.") \ + M(IcebergManifestDecodeThreads, "Number of threads in the thread pool used for decoding Iceberg data manifest files.") \ + M(IcebergManifestDecodeThreadsActive, "Number of threads in the thread pool used for decoding Iceberg data manifest files running a task.") \ + M(IcebergManifestDecodeThreadsScheduled, "Number of queued or active jobs in the thread pool used for decoding Iceberg data manifest files.") \ M(OutdatedPartsLoadingThreads, "Number of threads in the threadpool for loading Outdated data parts.") \ M(OutdatedPartsLoadingThreadsActive, "Number of active threads in the threadpool for loading Outdated data parts.") \ M(OutdatedPartsLoadingThreadsScheduled, "Number of queued or active jobs in the threadpool for loading Outdated data parts.") \ diff --git a/src/Common/setThreadName.h b/src/Common/setThreadName.h index 9993ba64400b0..3c062991b6ddf 100644 --- a/src/Common/setThreadName.h +++ b/src/Common/setThreadName.h @@ -72,6 +72,7 @@ namespace DB M(HASHED_DICT_LOAD, "HashedDictLoad") \ M(HTTP_HANDLER, "HTTPHandler") \ M(HTTP_SERVER_CONN, "HTTPSrvConn") \ + M(ICEBERG_DELETE_DECODE, "IcebergDelDec") \ M(ICEBERG_ITERATOR, "IcebergIter") \ M(ICEBERG_SCHEDULE_POOL, "IcebergSchPool") \ M(INTERSERVER_HANDLER, "IntersrvHandler") \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index ac804220f1422..ac0aa4a27ee57 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -182,6 +182,17 @@ A value of `0` means unlimited. )", 0) \ DECLARE(UInt64, max_format_parsing_thread_pool_size, 100, R"( Maximum total number of threads to use for parsing input. +)", 0) \ + DECLARE(UInt64, max_iceberg_manifest_decode_thread_pool_size, 100, R"( +Maximum total number of threads to use for decoding Iceberg data manifest files. + +The pool is separate from the IO pool on purpose: a decode task can block until the query consumes the entries it has produced, while the delete manifest decode waits for its tasks on the IO pool before any entry is consumed - sharing one pool could deadlock. +)", 0) \ + DECLARE(UInt64, max_iceberg_manifest_decode_thread_pool_free_size, 0, R"( +Maximum number of idle standby threads to keep in the thread pool for decoding Iceberg data manifest files. +)", 0) \ + DECLARE(UInt64, iceberg_manifest_decode_thread_pool_queue_size, 10000, R"( +The maximum number of jobs that can be scheduled on the thread pool for decoding Iceberg data manifest files. )", 0) \ DECLARE(UInt64, max_format_parsing_thread_pool_free_size, 0, R"( Maximum number of idle standby threads to keep in the thread pool for parsing input. @@ -3645,6 +3656,12 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {getFormatParsingThreadPool().isInitialized() ? std::to_string(getFormatParsingThreadPool().get().getMaxFreeThreads()) : "0", ChangeableWithoutRestart::Yes}}, {"format_parsing_thread_pool_queue_size", {getFormatParsingThreadPool().isInitialized() ? std::to_string(getFormatParsingThreadPool().get().getQueueSize()) : "0", ChangeableWithoutRestart::Yes}}, + {"max_iceberg_manifest_decode_thread_pool_size", + {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getMaxThreads()) : "0", ChangeableWithoutRestart::Yes}}, + {"max_iceberg_manifest_decode_thread_pool_free_size", + {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getMaxFreeThreads()) : "0", ChangeableWithoutRestart::Yes}}, + {"iceberg_manifest_decode_thread_pool_queue_size", + {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getQueueSize()) : "0", ChangeableWithoutRestart::Yes}}, {"abort_on_logical_error", {std::to_string(DB::abort_on_logical_error), ChangeableWithoutRestart::Yes}}, diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 75a3e21b1aee6..87b6c01965f67 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6012,14 +6012,21 @@ Possible values: DECLARE(UInt64, iceberg_metadata_staleness_ms, 0, R"( If non-zero, skip fetching iceberg metadata from remote catalog if there is a cached metadata snapshot, more recent than the given staleness window. Zero means to always fetch the latest metadata version from the remote catalog. Setting this a non-zero trades staleness to a lower latency of read operations. )", 0) \ - DECLARE(NonZeroUInt64, iceberg_delete_manifest_decode_concurrency, 4, R"( -Maximum number of Iceberg delete manifest files decoded concurrently during query execution before any data file is read. + DECLARE_WITH_ALIAS(NonZeroUInt64, iceberg_manifest_decode_concurrency, 4, R"( +Maximum number of Iceberg manifest files decoded concurrently while reading a table. -All delete manifests must be decoded before any data file is read, so this work sits on the critical path before the first row is returned. Decoding several at a time overlaps both the object storage round-trips and the per-row pruning work. +Delete manifests are all decoded before any data file is read; data manifests are decoded while the list of data files for the query is produced, and new ones are decoded only as the query consumes already decoded entries. Decoding several manifests at a time overlaps the object storage round-trips and the per-entry pruning work. -Higher values raise peak memory during query initialization when the Iceberg metadata files cache is disabled or full, since each in-flight manifest then holds its own decoded contents. +Higher values raise peak memory when the Iceberg metadata files cache is disabled or full, since each in-flight manifest then holds its own decoded contents. Must be greater than zero; `1` decodes the manifests one at a time. +)", 0, iceberg_delete_manifest_decode_concurrency) \ + DECLARE(NonZeroUInt64, iceberg_file_entries_queue_size, 100, R"( +Capacity of the queue between the Iceberg data manifest decode tasks and the query, in data file entries. + +The decode tasks pause once the queue is full and the query is not consuming, so this also bounds the read-ahead. + +Must be greater than zero. )", 0) \ DECLARE(Bool, use_parquet_metadata_cache, true, R"( If turned on, parquet format may utilize the parquet metadata cache. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 910d9b7efe4e5..1bf5d5f1e21b5 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -139,6 +139,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"output_format_arrow_use_native_writer", true, true, "Obsolete setting, the native ClickHouse writer is now always used for the `Arrow` and `ArrowStream` formats (the Apache Arrow library-based writer has been removed)."}, {"distributed_cache_min_inflight_bytes_to_discard_connection_on_seek", 0, 4 * 1024 * 1024, "New setting to drop and reopen a distributed cache connection on a seek when too many in-flight bytes would otherwise be discarded. Defaults to 4 MiB; 0 restores the previous behavior (always reuse the connection via the read range id)."}, {"iceberg_delete_manifest_decode_concurrency", 2, 4, "New setting bounding how many Iceberg delete manifest files are decoded concurrently before the first row is read. Before 26.8 one manifest was decoded at a time with the next one's fetch already in flight, so `2` is the closest equivalent of the previous behavior, which put the sum of their object storage round-trips on the critical path before the first row."}, + {"iceberg_file_entries_queue_size", 100, 100, "New setting for the previously hardcoded capacity of the queue between the Iceberg data manifest decode tasks and the query."}, + {"iceberg_manifest_decode_concurrency", 2, 4, "New setting bounding how many Iceberg manifest files are decoded concurrently, for delete and data manifests alike. It replaces `iceberg_delete_manifest_decode_concurrency` (kept as an alias). `2` approximates the data path before this change, which decoded one manifest at a time with the next one's fetch already in flight."}, {"run_query_in_background", false, false, "New setting to run a query in the background, detached from the connection that submitted it, discarding the result."}, {"enable_cascades_optimizer", false, false, "New experimental setting."}, {"merge_tree_min_bytes_per_read_stream", 0, (64 * 1024), "New setting to cap the number of streams for ordinary local unordered `MergeTree` narrow-column scans using a sqrt cost model, reducing per-stream overhead on high-core-count machines. previous_value=0 (disabled) so `compatibility` with versions before 26.8 restores the pre-existing stream count."}, diff --git a/src/IO/SharedThreadPools.h b/src/IO/SharedThreadPools.h index 7bb85c730c1d9..07e76d8e12f70 100644 --- a/src/IO/SharedThreadPools.h +++ b/src/IO/SharedThreadPools.h @@ -70,7 +70,8 @@ class StaticThreadPool M(DatabaseCatalogDropTables, "DropTablesThreadPool", DatabaseCatalog) \ M(MergeTreePrefixesDeserialization, "MergeTreePrefixesDeserializationThreadPool", MergeTreeSubcolumnsReader) \ M(DropDistributedCache, "DropDistributedCacheThreadPool", DropDistributedCache) \ - M(FormatParsing, "FormatParsingThreadPool", FormatParsing) + M(FormatParsing, "FormatParsingThreadPool", FormatParsing) \ + M(IcebergManifestDecode, "IcebergManifestDecodeThreadPool", IcebergManifestDecode) #define DECLARE_STATIC_THREAD_POOL_GETTER(SUFFIX, NAME, METRIC) StaticThreadPool & get##SUFFIX##ThreadPool(); APPLY_FOR_STATIC_THREAD_POOLS(DECLARE_STATIC_THREAD_POOL_GETTER) diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index 9316912f06180..466a92cd7cfdf 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -503,6 +503,11 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) if (!context->getSettingsRef()[Setting::use_index_for_in_with_subqueries]) return nullptr; + /// Concurrent index analyses may share this set through cloned filter DAGs, and the build mutates + /// `set_and_key->set` and `source`. A mutex and not `callOnce` because this build may stop without + /// creating the set (e.g. a subquery timeout with `overflow_mode = 'break'`) and then be retried. + std::lock_guard lock(inplace_build_mutex); + if (auto set = get()) { if (set->hasExplicitSetElements()) diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index f0771403586d1..7ed557a491d7c 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -231,6 +232,9 @@ class FutureSetFromSubquery final : public FutureSet /// The set can never be built once that happened, so `build` rethrows this instead of returning a null /// plan, which its callers would silently take for "nothing left to build". std::exception_ptr in_place_build_failure; + + /// Serializes `buildOrderedSetInplace`; see the rationale at its lock site. + std::mutex inplace_build_mutex; }; using FutureSetFromSubqueryPtr = std::shared_ptr; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index ada57d40ae42e..099f445fa0fab 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -53,6 +54,7 @@ #include #include #include +#include #include #include @@ -79,7 +81,8 @@ extern const int LOGICAL_ERROR; namespace Setting { extern const SettingsBool use_iceberg_partition_pruning; -extern const SettingsNonZeroUInt64 iceberg_delete_manifest_decode_concurrency; +extern const SettingsNonZeroUInt64 iceberg_file_entries_queue_size; +extern const SettingsNonZeroUInt64 iceberg_manifest_decode_concurrency; }; @@ -88,8 +91,8 @@ using namespace Iceberg; namespace { -/// All entries of one delete manifest file, produced by a single decode task. -using DeleteManifestBatch = std::vector; +/// All entries of one manifest file, produced by a single decode task. +using ManifestEntryBatch = std::vector; /// decode path agrees on whether pruning applies. std::shared_ptr makeManifestFilterDag(const ActionsDAG * filter_dag, const ContextPtr & context) @@ -178,114 +181,151 @@ std::span defineDeletesSpan( } -std::optional SingleThreadIcebergKeysIterator::next() +namespace Iceberg { - if (!data_snapshot) + +DataFileEntriesStream::DataFileEntriesStream( + size_t queue_size_, + size_t decode_concurrency_, + IcebergDataSnapshotPtr data_snapshot_, + std::function prepare_, + CreateManifestIterator create_manifest_iterator_) + : chunk_size(queue_size_) + , decode_concurrency(decode_concurrency_) + , data_snapshot(std::move(data_snapshot_)) + , prepare(std::move(prepare_)) + , create_manifest_iterator(std::move(create_manifest_iterator_)) + , queue(queue_size_) +{ + producer = std::make_unique( + [this, thread_group = CurrentThread::getGroup()]() + { + DB::ThreadGroupSwitcher switcher(thread_group, DB::ThreadName::ICEBERG_ITERATOR); + try + { + run(); + } + catch (...) + { + std::lock_guard lock(exception_mutex); + if (!exception) + { + exception = std::current_exception(); + } + } + stop(); + }); +} + +DataFileEntriesStream::~DataFileEntriesStream() +{ + stop(); + if (producer) { - return std::nullopt; + producer->join(); } +} - while (true) - { - /// Try to get the next entry from the current manifest file iterator. - if (current_manifest_file_iterator) - { - auto entry = current_manifest_file_iterator->next(); - if (entry) - return entry; +bool DataFileEntriesStream::pop(ProcessedManifestFileEntryPtr & entry) +{ + return queue.pop(entry); +} - /// next() returned nullptr, meaning the manifest file is fully exhausted. - current_manifest_file_iterator = nullptr; - } +void DataFileEntriesStream::clearAndFinish() +{ + stopped.store(true, std::memory_order_relaxed); + queue.clearAndFinish(); +} - /// Make sure a prefetch of the next matching manifest is in flight. - schedulePrefetchIfPossible(); - if (!prefetched_manifest.has_value()) - return std::nullopt; - - /// Take ownership of the in-flight prefetch: move it out of the member and reset the - /// optional (a moved-from optional is still engaged, so schedulePrefetchIfPossible would - /// otherwise see an occupied slot and never start the next fetch). - auto curr_prefetched = std::move(prefetched_manifest); - prefetched_manifest.reset(); - /// While the future lives only in this local, the destructor's wait no longer covers it, - /// so the task capturing `this` could outlive the iterator if anything below throws - /// (for example scheduling the next prefetch while the pool is shutting down). - SCOPE_EXIT({ - if (curr_prefetched.has_value() && curr_prefetched->future.valid()) - curr_prefetched->future.wait(); - }); +void DataFileEntriesStream::stop() +{ + stopped.store(true, std::memory_order_relaxed); + queue.finish(); +} - /// Start scheduling the next prefetch before we block and parse. - schedulePrefetchIfPossible(); - - auto manifest_file_cacheable_part = curr_prefetched->future.get(); - const auto & manifest_list_entry = data_snapshot->manifest_list_entries[curr_prefetched->manifest_list_index]; - - current_manifest_file_iterator = Iceberg::ManifestFileIterator::create( - manifest_file_cacheable_part.deserializer, - manifest_list_entry.manifest_file_path, - persistent_components.path_resolver, - *persistent_components.schema_processor, - manifest_list_entry.added_sequence_number, - manifest_list_entry.added_snapshot_id, - local_context, - filter_dag, - table_snapshot->schema_id); - } +std::exception_ptr DataFileEntriesStream::getException() const +{ + std::lock_guard lock(exception_mutex); + return exception; } -void SingleThreadIcebergKeysIterator::schedulePrefetchIfPossible() +void DataFileEntriesStream::run() { - if (!data_snapshot || prefetched_manifest.has_value()) + if (!data_snapshot) return; - while (manifest_file_index < data_snapshot->manifest_list_entries.size()) + if (prepare) + prepare(); + + auto stream_runner = threadPoolCallbackRunnerUnsafe(getIcebergManifestDecodeThreadPool().get(), DB::ThreadName::ICEBERG_ITERATOR); + + std::deque> in_flight; + SCOPE_EXIT({ + for (auto & manifest : in_flight) + { + if (manifest->future.valid()) + manifest->future.wait(); + } + }); + + const auto & manifest_list_entries = data_snapshot->manifest_list_entries; + size_t next_index = 0; + while (!stopped.load(std::memory_order_relaxed)) { - const size_t index = manifest_file_index++; - const auto & manifest_list_entry = data_snapshot->manifest_list_entries[index]; - if (manifest_list_entry.content_type != manifest_file_content_type) - continue; - - auto fetch = [this, - path = manifest_list_entry.manifest_file_path, - bytes = manifest_list_entry.manifest_file_byte_size]() + while (in_flight.size() < decode_concurrency && next_index < manifest_list_entries.size()) { - return Iceberg::getManifestFile(object_storage, persistent_components, local_context, log, path, bytes); - }; - prefetched_manifest = PrefetchedManifest{index, prefetch_runner(std::move(fetch), Priority{})}; - return; + const size_t index = next_index++; + if (manifest_list_entries[index].content_type != ManifestFileContentType::DATA) + continue; + auto manifest = std::make_unique(manifest_list_entries[index]); + auto * scheduled = manifest.get(); + manifest->future = stream_runner([this, scheduled] { decodeChunk(*scheduled); }, Priority{}); + in_flight.push_back(std::move(manifest)); + } + + if (in_flight.empty()) + return; + + auto & manifest = *in_flight.front(); + manifest.future.get(); + + for (auto & entry : manifest.chunk) + { + if (!queue.push(std::move(entry))) + return; + } + manifest.chunk.clear(); + + if (manifest.exhausted) + in_flight.pop_front(); + else + manifest.future = stream_runner([this, scheduled = &manifest] { decodeChunk(*scheduled); }, Priority{}); } } -SingleThreadIcebergKeysIterator::~SingleThreadIcebergKeysIterator() +void DataFileEntriesStream::decodeChunk(InFlightManifest & manifest) { - /// The scheduled task captures `this`, so it must not outlive the iterator. - if (prefetched_manifest.has_value() && prefetched_manifest->future.valid()) - prefetched_manifest->future.wait(); + if (stopped.load(std::memory_order_relaxed)) + { + manifest.exhausted = true; + return; + } + + if (!manifest.iterator) + manifest.iterator = create_manifest_iterator(manifest.key, &stopped); + + while (manifest.chunk.size() < chunk_size) + { + auto entry = manifest.iterator->next(); + if (!entry) + { + manifest.exhausted = true; + return; + } + manifest.chunk.push_back(std::move(entry)); + } } -SingleThreadIcebergKeysIterator::SingleThreadIcebergKeysIterator( - ObjectStoragePtr object_storage_, - ContextPtr local_context_, - Iceberg::ManifestFileContentType manifest_file_content_type_, - const ActionsDAG * filter_dag_, - Iceberg::TableStateSnapshotPtr table_snapshot_, - Iceberg::IcebergDataSnapshotPtr data_snapshot_, - PersistentTableComponents persistent_components_) - : object_storage(object_storage_) - , filter_dag(makeManifestFilterDag(filter_dag_, local_context_)) - , local_context(local_context_) - , table_snapshot(table_snapshot_) - , data_snapshot(data_snapshot_) - , persistent_components(persistent_components_) - , log(getLogger("IcebergIterator")) - , manifest_file_content_type(manifest_file_content_type_) - , prefetch_runner(threadPoolCallbackRunnerUnsafe( - getIOThreadPool().get(), DB::ThreadName::ICEBERG_ITERATOR)) -{ - /// Warm the first manifest fetch. - schedulePrefetchIfPossible(); } IcebergIterator::IcebergIterator( @@ -302,64 +342,29 @@ IcebergIterator::IcebergIterator( , table_state_snapshot(table_snapshot_) , data_snapshot(data_snapshot_) , persistent_components(persistent_components_) - , deletes_filter_dag(makeManifestFilterDag(filter_dag_, local_context_)) - , data_files_iterator( - object_storage, - local_context_, - Iceberg::ManifestFileContentType::DATA, - filter_dag_, - table_snapshot_, - data_snapshot_, - persistent_components_) - , blocking_queue(100) + , manifest_filter_dag(makeManifestFilterDag(filter_dag_, local_context_)) , callback(std::move(callback_)) { - /// Decoding any manifest reads settings from the context, so a missing one is fatal either way. - if (!local_context) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Context is required to construct IcebergIterator"); + chassert(local_context); - producer_task = std::make_unique( - [this, thread_group = CurrentThread::getGroup()]() + data_files_stream = std::make_unique( + local_context->getSettingsRef()[Setting::iceberg_file_entries_queue_size], + local_context->getSettingsRef()[Setting::iceberg_manifest_decode_concurrency], + data_snapshot, + [this] { - DB::ThreadGroupSwitcher switcher(thread_group, DB::ThreadName::ICEBERG_ITERATOR); - while (!blocking_queue.isFinished()) - { - std::optional entry; - try - { - entry = data_files_iterator.next(); - } - catch (...) - { - std::lock_guard lock(exception_mutex); - if (!exception) - { - exception = std::current_exception(); - } - blocking_queue.finish(); - break; - } - if (!entry.has_value()) - break; - while (!blocking_queue.push(std::move(entry.value()))) - { - if (blocking_queue.isFinished()) - { - break; - } - } - } - blocking_queue.finish(); - }); + if (manifest_filter_dag) + VirtualColumnUtils::buildOrderedSetsForDAG(*manifest_filter_dag, local_context); + }, + [this](const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) + { return createManifestIterator(manifest_list_entry, stop_flag); }); } -// ensureDeletesReady is called lazily so that delete and data manifest downloads overlap. void IcebergIterator::ensureDeletesReady() { std::lock_guard lock(deletes_mutex); if (!deletes_ready) { - /// Deferred part of the iterator initialization, charged to the same event as the constructor. ProfileEventTimeIncrement watch(ProfileEvents::IcebergIteratorInitializationMicroseconds); try { @@ -368,8 +373,7 @@ void IcebergIterator::ensureDeletesReady() catch (...) { deletes_exception = std::current_exception(); - /// Every `next` rethrows this exception, so the buffered entries are dead and the producer must stop. - blocking_queue.clearAndFinish(); + data_files_stream->clearAndFinish(); } deletes_ready = true; } @@ -378,6 +382,43 @@ void IcebergIterator::ensureDeletesReady() std::rethrow_exception(deletes_exception); } +Iceberg::ManifestIteratorPtr IcebergIterator::createManifestIterator(const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) const +{ + auto manifest_file_cacheable_part = Iceberg::getManifestFile( + object_storage, + persistent_components, + local_context, + logger, + manifest_list_entry.manifest_file_path, + manifest_list_entry.manifest_file_byte_size); + + return Iceberg::ManifestFileIterator::create( + manifest_file_cacheable_part.deserializer, + manifest_list_entry.manifest_file_path, + persistent_components.path_resolver, + *persistent_components.schema_processor, + manifest_list_entry.added_sequence_number, + manifest_list_entry.added_snapshot_id, + local_context, + manifest_filter_dag, + table_state_snapshot->schema_id, + stop_flag); +} + +std::vector IcebergIterator::decodeManifest(const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) const +{ + if (stop_flag && stop_flag->load(std::memory_order_relaxed)) + return {}; + + auto manifest_file_iterator = createManifestIterator(manifest_list_entry, stop_flag); + + ManifestEntryBatch batch; + while (auto entry = manifest_file_iterator->next()) + batch.push_back(entry); + /// Iterator and deserializer die here, before the batch is handed over. + return batch; +} + void IcebergIterator::decodeDeleteManifests() { std::vector delete_manifests; @@ -391,12 +432,12 @@ void IcebergIterator::decodeDeleteManifests() } /// Cap concurrency: each in-flight manifest holds its decoded contents. - const size_t max_in_flight = local_context->getSettingsRef()[Setting::iceberg_delete_manifest_decode_concurrency]; + const size_t max_in_flight = local_context->getSettingsRef()[Setting::iceberg_manifest_decode_concurrency]; auto decode_runner - = threadPoolCallbackRunnerUnsafe(getIOThreadPool().get(), DB::ThreadName::ICEBERG_ITERATOR); + = threadPoolCallbackRunnerUnsafe(getIOThreadPool().get(), DB::ThreadName::ICEBERG_DELETE_DECODE); - std::deque> in_flight; + std::deque> in_flight; /// The tasks capture `this`, so none of them may still be running when this function is left. SCOPE_EXIT({ for (auto & future : in_flight) @@ -412,32 +453,7 @@ void IcebergIterator::decodeDeleteManifests() while (in_flight.size() < max_in_flight && next_to_decode < delete_manifests.size()) { auto decode = [this, manifest_list_entry = delete_manifests[next_to_decode++]]() - { - auto manifest_file_cacheable_part = Iceberg::getManifestFile( - object_storage, - persistent_components, - local_context, - logger, - manifest_list_entry.manifest_file_path, - manifest_list_entry.manifest_file_byte_size); - - auto manifest_file_iterator = Iceberg::ManifestFileIterator::create( - manifest_file_cacheable_part.deserializer, - manifest_list_entry.manifest_file_path, - persistent_components.path_resolver, - *persistent_components.schema_processor, - manifest_list_entry.added_sequence_number, - manifest_list_entry.added_snapshot_id, - local_context, - deletes_filter_dag, - table_state_snapshot->schema_id); - - DeleteManifestBatch batch; - while (auto entry = manifest_file_iterator->next()) - batch.push_back(entry); - /// Iterator and deserializer die here, before the batch is handed over. - return batch; - }; + { return decodeManifest(manifest_list_entry, /* stop_flag */ nullptr); }; in_flight.push_back(decode_runner(std::move(decode), Priority{})); } @@ -467,7 +483,7 @@ ObjectInfoPtr IcebergIterator::next(size_t) ProfileEventTimeIncrement watch(ProfileEvents::IcebergMetadataReadWaitTimeMicroseconds); ensureDeletesReady(); Iceberg::ProcessedManifestFileEntryPtr manifest_file_entry; - if (blocking_queue.pop(manifest_file_entry)) + if (data_files_stream->pop(manifest_file_entry)) { IcebergDataObjectInfoPtr object_info = std::make_shared( @@ -546,14 +562,11 @@ ObjectInfoPtr IcebergIterator::next(size_t) return object_info; } + if (auto exception = data_files_stream->getException()) { - std::lock_guard lock(exception_mutex); - if (exception) - { - auto exception_message = getExceptionMessage(exception, true, true); - auto exception_code = getExceptionErrorCode(exception); - throw DB::Exception(exception_code, "Iceberg iterator is failed with exception: {}", exception_message); - } + auto exception_message = getExceptionMessage(exception, true, true); + auto exception_code = getExceptionErrorCode(exception); + throw DB::Exception(exception_code, "Iceberg iterator is failed with exception: {}", exception_message); } return nullptr; @@ -564,14 +577,7 @@ size_t IcebergIterator::estimatedKeysCount() return std::numeric_limits::max(); } -IcebergIterator::~IcebergIterator() -{ - blocking_queue.finish(); - if (producer_task) - { - producer_task->join(); - } -} +IcebergIterator::~IcebergIterator() = default; } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h index bb86db7cfe771..21020350e4196 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h @@ -17,10 +17,11 @@ #include #include -#include +#include +#include #include -#include +#include #include #include @@ -35,45 +36,54 @@ namespace DB namespace Iceberg { -class SingleThreadIcebergKeysIterator +class DataFileEntriesStream { public: - SingleThreadIcebergKeysIterator( - ObjectStoragePtr object_storage_, - ContextPtr local_context_, - Iceberg::ManifestFileContentType manifest_file_content_type_, - const ActionsDAG * filter_dag_, - TableStateSnapshotPtr table_snapshot_, + using CreateManifestIterator = std::function *)>; + + DataFileEntriesStream( + size_t queue_size_, + size_t decode_concurrency_, IcebergDataSnapshotPtr data_snapshot_, - PersistentTableComponents persistent_components); + std::function prepare_, + CreateManifestIterator create_manifest_iterator_); - ~SingleThreadIcebergKeysIterator(); + ~DataFileEntriesStream(); - std::optional next(); + bool pop(ProcessedManifestFileEntryPtr & entry); + void clearAndFinish(); + std::exception_ptr getException() const; private: - void schedulePrefetchIfPossible(); - - ObjectStoragePtr object_storage; - std::shared_ptr filter_dag; - ContextPtr local_context; - Iceberg::TableStateSnapshotPtr table_snapshot; - Iceberg::IcebergDataSnapshotPtr data_snapshot; - PersistentTableComponents persistent_components; - LoggerPtr log; - - size_t manifest_file_index = 0; - Iceberg::ManifestIteratorPtr current_manifest_file_iterator; - - const Iceberg::ManifestFileContentType manifest_file_content_type; - - struct PrefetchedManifest + struct InFlightManifest { - size_t manifest_list_index; - std::future future; + explicit InFlightManifest(ManifestFileCacheKey key_) + : key(std::move(key_)) + { + } + + ManifestFileCacheKey key; + ManifestIteratorPtr iterator; + std::vector chunk; + bool exhausted = false; + std::future future; }; - std::optional prefetched_manifest; - ThreadPoolCallbackRunnerUnsafe prefetch_runner; + + void run(); + void decodeChunk(InFlightManifest & manifest); + void stop(); + + const size_t chunk_size; + const size_t decode_concurrency; + const IcebergDataSnapshotPtr data_snapshot; + + const std::function prepare; + const CreateManifestIterator create_manifest_iterator; + ConcurrentBoundedQueue queue; + std::atomic stopped{false}; + mutable std::mutex exception_mutex; + std::exception_ptr exception TSA_GUARDED_BY(exception_mutex); + std::unique_ptr producer; }; } @@ -98,6 +108,8 @@ class IcebergIterator : public IObjectIterator private: void ensureDeletesReady(); void decodeDeleteManifests(); + Iceberg::ManifestIteratorPtr createManifestIterator(const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) const; + std::vector decodeManifest(const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) const; LoggerPtr logger; ObjectStoragePtr object_storage; @@ -105,10 +117,8 @@ class IcebergIterator : public IObjectIterator const Iceberg::TableStateSnapshotPtr table_state_snapshot; Iceberg::IcebergDataSnapshotPtr data_snapshot; Iceberg::PersistentTableComponents persistent_components; - std::shared_ptr deletes_filter_dag; - Iceberg::SingleThreadIcebergKeysIterator data_files_iterator; - ConcurrentBoundedQueue blocking_queue; - std::unique_ptr producer_task; + /// Shared read-only by the concurrent data- and delete-manifest decode tasks. + std::shared_ptr manifest_filter_dag; IDataLakeMetadata::FileProgressCallback callback; /// Filled once under `deletes_mutex` and never mutated afterwards, so `next` may read them /// unguarded once it has gone through `ensureDeletesReady`. @@ -117,8 +127,9 @@ class IcebergIterator : public IObjectIterator std::mutex deletes_mutex; bool deletes_ready TSA_GUARDED_BY(deletes_mutex) = false; std::exception_ptr deletes_exception TSA_GUARDED_BY(deletes_mutex); - std::exception_ptr exception; - std::mutex exception_mutex; + /// Declared last: its tasks call back into `createManifestIterator`, so it must be destroyed + /// (producer joined, tasks drained) before any other member. + std::unique_ptr data_files_stream; }; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index 8ea4c06e835e0..9c0d2db914de9 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -267,7 +267,8 @@ std::shared_ptr ManifestFileIterator::create( Int64 inherited_snapshot_id_, DB::ContextPtr context_, std::shared_ptr filter_dag_, - Int32 table_snapshot_schema_id_) + Int32 table_snapshot_schema_id_, + const std::atomic * stop_flag_) { insertRowToLogTable( context_, @@ -370,7 +371,8 @@ std::shared_ptr ManifestFileIterator::create( std::move(partition_key_description), total_rows, std::move(filter_dag_), - table_snapshot_schema_id_)); + table_snapshot_schema_id_, + stop_flag_)); } ManifestFileIterator::ManifestFileIterator( @@ -387,7 +389,8 @@ ManifestFileIterator::ManifestFileIterator( std::optional partition_key_description_, size_t total_rows_, std::shared_ptr filter_dag_, - Int32 table_snapshot_schema_id_) + Int32 table_snapshot_schema_id_, + const std::atomic * stop_flag_) : manifest_file_deserializer(std::move(manifest_file_deserializer_)) , path_to_manifest_file(path_to_manifest_file_) , format_version(format_version_) @@ -400,6 +403,7 @@ ManifestFileIterator::ManifestFileIterator( , partition_key_description(std::move(partition_key_description_)) , table_snapshot_schema_id(table_snapshot_schema_id_) , total_rows(total_rows_) + , stop_flag(stop_flag_) , data_files_without_deleted(std::make_shared>()) , position_deletes_files_without_deleted(std::make_shared>()) , equality_deletes_files_without_deleted(std::make_shared>()) @@ -649,6 +653,11 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::next() fully_initialized.store(true); return nullptr; } + /// The data manifest decode tasks pass the stream's stopped flag here, so a cancelled + /// query stops decoding mid-manifest. Checked between rows rather than by the caller, + /// because a long stretch of pruned rows yields nothing the caller could check on. + if (stop_flag && stop_flag->load(std::memory_order_relaxed)) + return nullptr; auto entry = processRow(row_index); if (entry) return entry; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h index 71e5ac320e833..40bb9b4c45c19 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h @@ -11,6 +11,8 @@ #include #include +#include +#include #include #include @@ -92,7 +94,8 @@ class ManifestFileIterator : public boost::noncopyable Int64 inherited_snapshot_id, DB::ContextPtr context, std::shared_ptr filter_dag_, - Int32 table_snapshot_schema_id_); + Int32 table_snapshot_schema_id_, + const std::atomic * stop_flag = nullptr); ManifestFileEntriesHandle getFilesWithoutDeletedHandle() const; @@ -131,7 +134,8 @@ class ManifestFileIterator : public boost::noncopyable std::optional partition_key_description, size_t total_rows, std::shared_ptr filter_dag, - Int32 table_snapshot_schema_id); + Int32 table_snapshot_schema_id, + const std::atomic * stop_flag); ProcessedManifestFileEntryPtr processRow(size_t row_index); @@ -151,6 +155,8 @@ class ManifestFileIterator : public boost::noncopyable /// Iteration state const size_t total_rows; + /// When set and observed true, `next` gives up between rows and returns nullptr as on EOF. + const std::atomic * const stop_flag; std::atomic current_row_index{0}; std::atomic fully_initialized{false}; std::atomic active_fetchers{0}; diff --git a/tests/integration/test_iceberg_manifest_decode_pool/__init__.py b/tests/integration/test_iceberg_manifest_decode_pool/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_iceberg_manifest_decode_pool/configs/decode_pool.xml b/tests/integration/test_iceberg_manifest_decode_pool/configs/decode_pool.xml new file mode 100644 index 0000000000000..e0ad5b6cbbf8d --- /dev/null +++ b/tests/integration/test_iceberg_manifest_decode_pool/configs/decode_pool.xml @@ -0,0 +1,3 @@ + + 1 + diff --git a/tests/integration/test_iceberg_manifest_decode_pool/test.py b/tests/integration/test_iceberg_manifest_decode_pool/test.py new file mode 100644 index 0000000000000..3492ffa904c46 --- /dev/null +++ b/tests/integration/test_iceberg_manifest_decode_pool/test.py @@ -0,0 +1,98 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", main_configs=["configs/decode_pool.xml"]) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_nested_iceberg_read_with_single_pool_thread(started_cluster): + """A pruning subquery reading another Iceberg table must not deadlock the decode + pool: the sets are built on the producer thread before any pool task runs, so the + single pool thread is never held by a task that waits for a nested read.""" + node.query( + """ + CREATE TABLE ice_main (id Int64, part Int64) + ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/ice_main/')) + """ + ) + node.query( + """ + CREATE TABLE ice_parts (part Int64) + ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/ice_parts/')) + """ + ) + # One commit per insert, so both tables have several manifests to decode. + for part in range(8): + node.query( + f"INSERT INTO ice_main SELECT number + {part} * 10, {part} FROM numbers(10) " + "SETTINGS allow_insert_into_iceberg = 1" + ) + for part in [1, 3, 6]: + node.query( + f"INSERT INTO ice_parts SELECT {part} " + "SETTINGS allow_insert_into_iceberg = 1" + ) + + result = node.query( + "SELECT count() FROM ice_main WHERE part IN (SELECT part FROM ice_parts)", + settings={ + "iceberg_manifest_decode_concurrency": 4, + "use_iceberg_metadata_files_cache": 0, + }, + timeout=120, + ) + assert int(result.strip()) == 30 + + node.query("DROP TABLE ice_main") + node.query("DROP TABLE ice_parts") + + +def test_join_of_two_iceberg_tables_with_single_pool_thread(started_cluster): + node.query( + """ + CREATE TABLE ice_left (id Int64) + ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/ice_left/')) + """ + ) + node.query( + """ + CREATE TABLE ice_right (id Int64) + ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/ice_right/')) + """ + ) + # One commit per insert, so each side has several data files to hand over. + for i in range(8): + node.query( + f"INSERT INTO ice_left SELECT number + {i} * 10 FROM numbers(10) " + "SETTINGS allow_insert_into_iceberg = 1" + ) + node.query( + f"INSERT INTO ice_right SELECT number + {i} * 10 FROM numbers(10) " + "SETTINGS allow_insert_into_iceberg = 1" + ) + + result = node.query( + "SELECT count() FROM ice_left AS l INNER JOIN ice_right AS r ON l.id = r.id", + settings={ + "iceberg_manifest_decode_concurrency": 4, + # The smallest queue: the second data file of a side already has to wait for the + # query to consume the first one, which is what used to park a pool thread. + "iceberg_file_entries_queue_size": 1, + "use_iceberg_metadata_files_cache": 0, + }, + timeout=120, + ) + assert int(result.strip()) == 80 + + node.query("DROP TABLE ice_left") + node.query("DROP TABLE ice_right") diff --git a/tests/integration/test_storage_iceberg_with_spark/test_data_manifest_decode_concurrency.py b/tests/integration/test_storage_iceberg_with_spark/test_data_manifest_decode_concurrency.py new file mode 100644 index 0000000000000..e027ec5423822 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_data_manifest_decode_concurrency.py @@ -0,0 +1,349 @@ +from helpers.iceberg_utils import ( + create_iceberg_table, + default_upload_directory, + get_uuid_str, +) + +# One data manifest per `INSERT`, see `write_table`. +DATA_MANIFEST_COUNT = 8 +ROWS_PER_INSERT = 4 +ROW_COUNT = DATA_MANIFEST_COUNT * ROWS_PER_INSERT + +# Must match the delay the `iceberg_slow_manifest_read` failpoint injects. +SLEEP_PER_MANIFEST_SECONDS = 0.40 + +STORAGE_TYPE = "local" + + +def get_array(query_result: str): + return sorted([int(x) for x in query_result.strip().split("\n")]) + + +def elapsed(node, query, **kwargs): + query_id = get_uuid_str() + node.query(query, query_id=query_id, **kwargs) + node.query("SYSTEM FLUSH LOGS query_log") + duration_result = node.query( + f"""SELECT query_duration_ms / 1000.0 as duration FROM system.query_log + WHERE type = 'QueryFinish' AND query_id = '{query_id}' LIMIT 1""" + ) + return float(duration_result.strip()) + + +def write_table( + started_cluster, table_name: str, merge_on_read: bool = False, format_version: int = 2 +): + spark = started_cluster.spark_session + + extra_properties = "" + if merge_on_read: + extra_properties = """, + 'write.update.mode' = 'merge-on-read', + 'write.delete.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read'""" + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint, part int, data string) USING iceberg + PARTITIONED BY (part) + TBLPROPERTIES ( + 'format-version' = '{format_version}', + 'commit.manifest-merge.enabled' = 'false'{extra_properties} + ) + """ + ) + # One commit per partition so that every `INSERT` produces its own data manifest. + for part in range(DATA_MANIFEST_COUNT): + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, {part}, char(id % 26 + ascii('a')) + FROM range({part * ROWS_PER_INSERT}, {(part + 1) * ROWS_PER_INSERT}) + """ + ) + + default_upload_directory( + started_cluster, + STORAGE_TYPE, + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + +def check_data_manifests(instance, table_expression: str) -> None: + settings = {"iceberg_metadata_log_level": "manifest_list_entry"} + + query_id = "check_data_manifests_" + get_uuid_str() + instance.query( + f"SELECT id FROM {table_expression} FORMAT Null", query_id=query_id, settings=settings + ) + instance.query("SYSTEM FLUSH LOGS iceberg_metadata_log") + # Data manifests have content = 0. (Delete manifests have content = 1). + count = instance.query( + f""" + SELECT uniqExact(JSONExtractString(content, 'manifest_path')) + FROM system.iceberg_metadata_log + WHERE query_id = '{query_id}' + AND content_type = 'ManifestListEntry' + AND JSONExtractInt(content, 'content') = 0 + """ + ) + count = int(count.strip()) + assert count == DATA_MANIFEST_COUNT + + +def test_data_manifest_decode_concurrency(started_cluster_iceberg_with_spark): + """The result must not depend on `iceberg_manifest_decode_concurrency`.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + TABLE_NAME = "test_data_manifest_decode_concurrency_" + get_uuid_str() + + write_table(started_cluster_iceberg_with_spark, TABLE_NAME) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + check_data_manifests(instance, TABLE_NAME) + + expected = list(range(ROW_COUNT)) + for concurrency in [1, 2, 4, 16]: + assert ( + get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency + }, + ) + ) + == expected + ), f"wrong result with iceberg_manifest_decode_concurrency={concurrency}" + + # A filter on the partition column makes every concurrent decode task evaluate the same + # shared filter DAG (with its lazily materialized IN set) while pruning manifest entries. + filtered_parts = [1, 3, 6] + expected_filtered = sorted( + row_id + for row_id in range(ROW_COUNT) + if row_id // ROWS_PER_INSERT in filtered_parts + ) + for concurrency in [1, 2, 4, 16]: + result = get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} WHERE part IN (1, 3, 6) ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency + }, + ) + ) + assert ( + result == expected_filtered + ), f"wrong filtered result with iceberg_manifest_decode_concurrency={concurrency}" + + instance.query(f"DROP TABLE {TABLE_NAME}") + + +def test_data_and_delete_manifest_decode_concurrency( + started_cluster_iceberg_with_spark, +): + """Data- and delete-manifest decode run concurrently and share one filter DAG; + the result must not depend on the concurrency setting.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + TABLE_NAME = "test_data_and_delete_manifest_decode_concurrency_" + get_uuid_str() + + write_table(started_cluster_iceberg_with_spark, TABLE_NAME, merge_on_read=True) + spark = started_cluster_iceberg_with_spark.spark_session + # One delete manifest per `DELETE`; ids 4, 13 and 25 fall inside the filtered + # partitions below, id 0 outside them. + deleted_ids = [0, 4, 13, 25] + for row_id in deleted_ids: + spark.sql(f"DELETE FROM {TABLE_NAME} WHERE id = {row_id}") + default_upload_directory( + started_cluster_iceberg_with_spark, + STORAGE_TYPE, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + + filtered_parts = [1, 3, 6] + expected = sorted( + row_id + for row_id in range(ROW_COUNT) + if row_id // ROWS_PER_INSERT in filtered_parts and row_id not in deleted_ids + ) + for concurrency in [1, 4, 16]: + result = get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} WHERE part IN (1, 3, 6) ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency, + }, + ) + ) + assert result == expected, ( + f"wrong result with iceberg_manifest_decode_concurrency={concurrency}" + ) + + instance.query(f"DROP TABLE {TABLE_NAME}") + + +def test_data_manifest_decode_large_manifest(started_cluster_iceberg_with_spark): + """A single manifest holding three times more entries than the producer queue's + capacity (100), so pushes block in the middle of the manifest at every concurrency; + the result must not depend on the concurrency.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_data_manifest_decode_large_manifest_" + get_uuid_str() + + entry_count = 300 + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id bigint) USING iceberg + PARTITIONED BY (id) + TBLPROPERTIES ('commit.manifest-merge.enabled' = 'false') + """ + ) + # One commit writes one data file per partition value, producing a single + # manifest with `entry_count` entries. + spark.sql(f"INSERT INTO {TABLE_NAME} SELECT id FROM range({entry_count})") + default_upload_directory( + started_cluster_iceberg_with_spark, + STORAGE_TYPE, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + + expected = list(range(entry_count)) + for concurrency in [1, 4]: + result = get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency, + "use_iceberg_metadata_files_cache": 0, + }, + ) + ) + assert ( + result == expected + ), f"wrong result with iceberg_manifest_decode_concurrency={concurrency}" + + # A filter pruning every entry keeps the row loop inside `ManifestFileIterator::next` + # busy for the whole manifest without yielding. + for concurrency in [1, 16]: + count = instance.query( + f"SELECT count() FROM {TABLE_NAME} WHERE id < 0", + settings={ + "iceberg_manifest_decode_concurrency": concurrency, + "use_iceberg_metadata_files_cache": 0, + }, + ) + assert ( + int(count.strip()) == 0 + ), f"fully pruned read returned rows with iceberg_manifest_decode_concurrency={concurrency}" + + instance.query(f"DROP TABLE {TABLE_NAME}") + + +def test_data_manifest_decode_concurrency_subquery_filter( + started_cluster_iceberg_with_spark, +): + """`part IN (SELECT ...)` is backed by `FutureSetFromSubquery`, which the concurrent + decode tasks build lazily through the shared filter DAG; the result must not depend + on the concurrency. (Literal tuple `IN` goes through `FutureSetFromTuple` instead + and is covered above.)""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + TABLE_NAME = "test_data_manifest_decode_concurrency_subquery_" + get_uuid_str() + + write_table(started_cluster_iceberg_with_spark, TABLE_NAME) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + + # The subquery yields parts {1, 3, 5}. + filtered_parts = [1, 3, 5] + expected = sorted( + row_id + for row_id in range(ROW_COUNT) + if row_id // ROWS_PER_INSERT in filtered_parts + ) + for concurrency in [1, 4, 16]: + for _ in range(2): + result = get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} " + "WHERE part IN (SELECT toInt32(number * 2 + 1) FROM numbers(3)) " + "ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency, + "use_iceberg_metadata_files_cache": 0, + }, + ) + ) + assert result == expected, ( + f"wrong subquery-filtered result with " + f"iceberg_manifest_decode_concurrency={concurrency}" + ) + + instance.query(f"DROP TABLE {TABLE_NAME}") + + +def test_data_manifest_decode_concurrency_bounds_reads( + started_cluster_iceberg_with_spark, +): + """`iceberg_manifest_decode_concurrency = 1` reads the manifests one at a + time, so with the failpoint delaying every manifest read the query cannot run + faster than one delay per manifest; a higher value overlaps the reads.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + + # Skip timing asserts on slow builds. + slow_build = ( + instance.is_built_with_sanitizer() or instance.is_built_with_llvm_coverage() + ) + + TABLE_NAME = "test_data_manifest_decode_concurrency_bounds_" + get_uuid_str() + + write_table(started_cluster_iceberg_with_spark, TABLE_NAME) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + + instance.query("SYSTEM ENABLE FAILPOINT iceberg_slow_manifest_read") + try: + serial_duration = elapsed( + instance, + f"SELECT * FROM {TABLE_NAME} FORMAT Null", + settings={ + "iceberg_manifest_decode_concurrency": 1, + "use_iceberg_metadata_files_cache": 0, + }, + ) + parallel_duration = elapsed( + instance, + f"SELECT * FROM {TABLE_NAME} FORMAT Null", + settings={ + "iceberg_manifest_decode_concurrency": 16, + "use_iceberg_metadata_files_cache": 0, + }, + ) + finally: + instance.query("SYSTEM DISABLE FAILPOINT iceberg_slow_manifest_read") + instance.query(f"DROP TABLE {TABLE_NAME}") + + if slow_build: + return + + serial_floor = 0.9 * DATA_MANIFEST_COUNT * SLEEP_PER_MANIFEST_SECONDS + assert serial_duration >= serial_floor, ( + f"the serial read of {DATA_MANIFEST_COUNT} manifests took {serial_duration:.3f}s, " + f"below the {serial_floor:.3f}s floor of one failpoint delay per manifest, so " + f"iceberg_manifest_decode_concurrency = 1 did not decode them one at a time" + ) + assert parallel_duration < 0.6 * serial_duration, ( + f"the read with iceberg_manifest_decode_concurrency = 16 took " + f"{parallel_duration:.3f}s against {serial_duration:.3f}s serially, so the " + f"manifest reads were likely not overlapped" + ) diff --git a/tests/performance/iceberg_data_manifests.xml b/tests/performance/iceberg_data_manifests.xml new file mode 100644 index 0000000000000..2cc4e4c081d0f --- /dev/null +++ b/tests/performance/iceberg_data_manifests.xml @@ -0,0 +1,220 @@ + + + + + n + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 + 41 + 42 + 43 + 44 + 45 + 46 + 47 + 48 + 49 + 50 + 51 + 52 + 53 + 54 + 55 + 56 + 57 + 58 + 59 + 60 + 61 + 62 + 63 + 64 + 65 + 66 + 67 + 68 + 69 + 70 + 71 + 72 + 73 + 74 + 75 + 76 + 77 + 78 + 79 + 80 + 81 + 82 + 83 + 84 + 85 + 86 + 87 + 88 + 89 + 90 + 91 + 92 + 93 + 94 + 95 + 96 + 97 + 98 + 99 + 100 + 101 + 102 + 103 + 104 + 105 + 106 + 107 + 108 + 109 + 110 + 111 + 112 + 113 + 114 + 115 + 116 + 117 + 118 + 119 + 120 + 121 + 122 + 123 + 124 + 125 + 126 + 127 + 128 + 129 + 130 + 131 + 132 + 133 + 134 + 135 + 136 + 137 + 138 + 139 + 140 + 141 + 142 + 143 + 144 + 145 + 146 + 147 + 148 + 149 + 150 + 151 + 152 + 153 + 154 + 155 + 156 + 157 + 158 + 159 + 160 + 161 + 162 + 163 + 164 + 165 + 166 + 167 + 168 + 169 + 170 + 171 + 172 + 173 + 174 + 175 + 176 + 177 + 178 + 179 + 180 + 181 + 182 + 183 + 184 + 185 + 186 + 187 + 188 + 189 + 190 + 191 + 192 + 193 + 194 + 195 + 196 + 197 + 198 + 199 + + + + + DROP TABLE IF EXISTS perf_iceberg_data_manifests + CREATE TABLE perf_iceberg_data_manifests (id Int64, data String) ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/perf_iceberg_data_manifests/')) + INSERT INTO perf_iceberg_data_manifests SELECT number + {n} * 500, toString(number) FROM numbers(500) SETTINGS allow_insert_into_iceberg = 1 + + SELECT max(id) FROM perf_iceberg_data_manifests SETTINGS use_iceberg_metadata_files_cache = 0 + + DROP TABLE IF EXISTS perf_iceberg_data_manifests + From 7147d96ab7fabde1cf367e038d0b5df8ac849c50 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 18:04:27 +0000 Subject: [PATCH 034/296] Backport #121031 to 26.8: Shrink 05182_limit_shrinks_block_size_with_array_join's fixture to 20000 rows --- .../05182_limit_shrinks_block_size_with_array_join.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql index a4d19a379fddf..1fe7327f0c6f3 100644 --- a/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql +++ b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql @@ -1,7 +1,8 @@ -- LIMIT still shrinks the block size with arrayJoin, only the source-side limit stays off (#82279) DROP TABLE IF EXISTS t_aj_limit; CREATE TABLE t_aj_limit (k UInt64, a Array(UInt64)) ENGINE = MergeTree ORDER BY k SETTINGS index_granularity = 8; -INSERT INTO t_aj_limit SELECT number, [number] FROM numbers(200000); +-- the 5000 bound below needs an un-shrunk read above it, and marks = rows / index_granularity +INSERT INTO t_aj_limit SELECT number, [number] FROM numbers(20000); SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 FORMAT Null SETTINGS max_threads = 8, log_comment = '05182_limit'; From b829bcda457b46fc0a254386fba2a0011ce4da90 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 18:23:45 +0000 Subject: [PATCH 035/296] Backport #120950 to 26.8: Fix parsing of a column named `from` inside parentheses --- src/Parsers/ExpressionElementParsers.cpp | 14 +++++- src/Parsers/ExpressionListParsers.cpp | 38 ++++++++++++++++ src/Parsers/ExpressionListParsers.h | 10 +++++ ...from_in_parenthesized_expression.reference | 31 +++++++++++++ ...named_from_in_parenthesized_expression.sql | 44 +++++++++++++++++++ 5 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.reference create mode 100644 tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.sql diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index f9ab6405fb005..18850116a6d1e 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -220,11 +220,23 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (pos->type != TokenType::OpeningRoundBracket) return false; + const auto opening_bracket_pos = pos; ++pos; /// Lookahead for inner subquery const bool possible_inner_subquery = pos->type == TokenType::OpeningRoundBracket; + /// A subquery in the FROM-first form, where the SELECT clause is omitted (`(FROM t)` means + /// `(SELECT * FROM t)`), starts with a word that is also a valid unquoted column name: `from`. + /// Unlike a leading `SELECT`, `EXPLAIN` or `VALUES`, that word is therefore not evidence that the + /// parentheses hold a subquery at all. Where the contents read as an expression over a column + /// named `from`, that older reading wins and these are not a subquery. + const bool starts_with_from_clause + = pos->type == TokenType::BareWord && equalsCaseInsensitive(std::string_view(pos->begin, pos->size()), "from"); + + if (starts_with_from_clause && parenthesesHoldExpressionOverColumnNamedFrom(opening_bracket_pos)) + return false; + ASTPtr result_node = nullptr; if (ASTPtr select_node; select.parse(pos, select_node, expected)) @@ -314,7 +326,7 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } /// Inner subquery should be handled separately - starts_with_valid_select_or_explain = !possible_inner_subquery && result_node != nullptr; + starts_with_valid_select_or_explain = !possible_inner_subquery && !starts_with_from_clause && result_node != nullptr; if (pos->type != TokenType::ClosingRoundBracket) return false; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 7fa127a1873aa..10695c824f5fc 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -3506,6 +3506,44 @@ static bool isArrayQuantifierPredicate(std::string_view function_name) return predicates.contains(function_name); } +/// See the declaration for the ambiguity this resolves. The word is read as a column only when an +/// operator follows it, because otherwise it is the clause keyword in front of a table expression and +/// the subquery reading is the only one: `(FROM t)`, `(FROM numbers(10) |> LIMIT 1)`, +/// `(FROM (SELECT 1))`. Even then the reading is taken only if the parentheses really do hold an +/// expression, which keeps a relation named after an operator readable as one: `1 IN (FROM in)`. +bool parenthesesHoldExpressionOverColumnNamedFrom(IParser::Pos pos) +{ + if (pos->type != TokenType::OpeningRoundBracket) + return false; + ++pos; + + /// A clause keyword is always a BareWord token, so a quoted `` `from` `` never starts a FROM clause. + auto contents_pos = pos; + if (pos->type != TokenType::BareWord || !equalsCaseInsensitive(std::string_view(pos->begin, pos->size()), "from")) + return false; + ++pos; + + Expected expected; + bool operator_follows = false; + for (const auto & [lexeme, unused_operator] : ParserExpressionImpl::operators_table) + { + auto operator_pos = pos; + if (parseOperator(operator_pos, lexeme, expected)) + { + operator_follows = true; + break; + } + } + + if (!operator_follows) + return false; + + /// The contents can also be a tuple or carry an alias, so parse them the way RoundBracketsLayer does. + ASTPtr contents; + ParserExpressionList contents_parser(/*allow_alias_without_as_keyword*/ false); + return contents_parser.parse(contents_pos, contents, expected) && contents_pos->type == TokenType::ClosingRoundBracket; +} + Action ParserExpressionImpl::tryParseOperand(Layers & layers, IParser::Pos & pos, Expected & expected) { ASTPtr tmp; diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 069700dd4d4a8..8cc2465ea6b83 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -334,6 +334,16 @@ class ParserTTLExpressionList : public IParserBase bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +/// Whether the parentheses that `pos` points at hold an expression over a column named `from`, rather +/// than a subquery. Does not move `pos`. +/// +/// `from` is a keyword that ClickHouse has always accepted as an unquoted identifier, and a subquery +/// may omit its SELECT clause and start with the FROM clause instead (`(FROM t)` means +/// `(SELECT * FROM t)`), so both readings parse for `WHERE (from IN ('a'))`: the expression +/// `from IN ('a')`, and the subquery `SELECT * FROM IN('a')` over a table function named `IN`. The +/// column reading is older than the FROM-first form, so it wins the ambiguity. +bool parenthesesHoldExpressionOverColumnNamedFrom(IParser::Pos pos); + } #pragma clang diagnostic pop diff --git a/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.reference b/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.reference new file mode 100644 index 0000000000000..4cd327b6fa464 --- /dev/null +++ b/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.reference @@ -0,0 +1,31 @@ +1 +1 +1 +2 +1 +1 +2 +1 +2 +1 +1 a +2 b +1 +0 +1 +1 +1 +1 +3 +2 +1 +1 +1 +(1,2) +1 +SELECT count()\nFROM t\nWHERE (`from` IN (1)) +0 +0 +1 +3 +1 diff --git a/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.sql b/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.sql new file mode 100644 index 0000000000000..6f9a4860ceded --- /dev/null +++ b/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.sql @@ -0,0 +1,44 @@ +-- A column named `from` is an unquoted keyword identifier, which is allowed. Inside parentheses it is +-- ambiguous with a subquery in the FROM-first form (`(FROM t)` means `(SELECT * FROM t)`); the column +-- reading wins. + +DROP TABLE IF EXISTS t_column_named_from; +CREATE TABLE t_column_named_from (`from` Nullable(String), c1 UInt8) ENGINE = MergeTree ORDER BY c1; +INSERT INTO t_column_named_from VALUES ('a', 1), ('b', 2), (NULL, 3); + +SELECT count() FROM t_column_named_from WHERE (from IN ('a')); +SELECT count() FROM t_column_named_from WHERE (from NOT IN ('a')); +SELECT count() FROM t_column_named_from WHERE (from IS NULL); +SELECT count() FROM t_column_named_from WHERE (from IS NOT NULL); +SELECT count() FROM t_column_named_from WHERE (from LIKE 'a%'); +SELECT count() FROM t_column_named_from WHERE (from = 'a'); +SELECT count() FROM t_column_named_from WHERE (from BETWEEN 'a' AND 'b'); +SELECT count() FROM t_column_named_from WHERE ((from IS NOT NULL) AND (from != 'b')); +SELECT count() FROM t_column_named_from WHERE (from IS NULL) OR (from = 'a'); +SELECT count() FROM t_column_named_from WHERE ((((from IN ('a'))))); +SELECT c1, from FROM t_column_named_from WHERE (from IS NOT NULL) ORDER BY c1; + +DROP TABLE t_column_named_from; + +-- The same ambiguity without a table. +SELECT (from IN (1)) FROM (SELECT 1 AS `from`); +SELECT (from IS NULL) FROM (SELECT 1 AS `from`); +SELECT (from IS NOT NULL) FROM (SELECT 1 AS `from`); +SELECT (from BETWEEN 0 AND 2) FROM (SELECT 1 AS `from`); +SELECT (from AND 1) FROM (SELECT 1 AS `from`); +SELECT (from OR 0) FROM (SELECT 1 AS `from`); +SELECT (from IS NULL ? 2 : 3) FROM (SELECT 1 AS `from`); +SELECT (from + 1) FROM (SELECT 1 AS `from`); +SELECT (from::String) FROM (SELECT 1 AS `from`); +SELECT (from.1) FROM (SELECT (1, 2) AS `from`); +SELECT (from[1]) FROM (SELECT [1, 2] AS `from`); +SELECT (from IN (1), from + 1) FROM (SELECT 1 AS `from`); +SELECT (from IN (1) AS in_a) FROM (SELECT 1 AS `from`); +SELECT formatQuery('SELECT count() FROM t WHERE (from IN (1))'); + +-- A subquery in the FROM-first form keeps its reading in the same positions. +SELECT 1 IN (FROM system.one); +SELECT * FROM (FROM system.one); +SELECT 1 IN (FROM (SELECT 1)); +SELECT (FROM numbers(3) |> SELECT count()); +SELECT (FROM system.one |> SELECT 1); From 026164a29951f26fb3e709fef3a3baba71f30a74 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 18:34:26 +0000 Subject: [PATCH 036/296] Backport #120992 to 26.8: Measure read_rows on a single reader in 05182_limit_shrinks_block_size_with_array_join --- .../05182_limit_shrinks_block_size_with_array_join.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql index a4d19a379fddf..b215a049df0e0 100644 --- a/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql +++ b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql @@ -3,7 +3,8 @@ DROP TABLE IF EXISTS t_aj_limit; CREATE TABLE t_aj_limit (k UInt64, a Array(UInt64)) ENGINE = MergeTree ORDER BY k SETTINGS index_granularity = 8; INSERT INTO t_aj_limit SELECT number, [number] FROM numbers(200000); -SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 FORMAT Null SETTINGS max_threads = 8, log_comment = '05182_limit'; +-- the initiator's read_rows sums every replica's read, so the bound below needs a single reader +SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 FORMAT Null SETTINGS max_threads = 8, enable_parallel_replicas = 0, log_comment = '05182_limit'; SYSTEM FLUSH LOGS query_log; SELECT argMax(read_rows, event_time_microseconds) < 5000 FROM system.query_log @@ -13,7 +14,9 @@ WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comm SELECT DISTINCT bs FROM (SELECT arrayJoin(a), blockSize() AS bs FROM t_aj_limit LIMIT 3 SETTINGS max_threads = 1, enable_parallel_replicas = 0); -- the prefetched pool sizes its reads by marks, not by the block size, so a small LIMIT keeps it off -SELECT countIf(explain LIKE '%PrefetchedReadPool%') FROM (EXPLAIN PIPELINE SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 SETTINGS allow_prefetched_read_pool_for_local_filesystem = 1, local_filesystem_read_method = 'pread_threadpool', max_threads = 8, merge_tree_min_rows_for_concurrent_read = 1, merge_tree_min_bytes_for_concurrent_read = 1); +-- it is only a candidate for a single-node read that is multi-stream or all-remote, and the local +-- read settings gate an all-local read while the remote ones gate an all-remote read +SELECT countIf(explain LIKE '%PrefetchedReadPool%') FROM (EXPLAIN PIPELINE SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 SETTINGS allow_prefetched_read_pool_for_local_filesystem = 1, local_filesystem_read_method = 'pread_threadpool', allow_prefetched_read_pool_for_remote_filesystem = 1, remote_filesystem_read_method = 'threadpool', max_threads = 8, max_threads_min_free_memory_per_thread = 0, merge_tree_min_rows_for_concurrent_read = 1, merge_tree_min_bytes_for_concurrent_read = 1, enable_parallel_replicas = 0); DROP TABLE t_aj_limit; From 36c31d505aa50c42fab02fa9323e85a06e19d551 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 19:53:54 +0000 Subject: [PATCH 037/296] Backport #120765 to 26.8: Revert "Merge pull request #114313" --- .../Optimizations/filterPushDown.cpp | 40 ++------------- .../01655_plan_optimizations.reference | 2 +- .../0_stateless/01655_plan_optimizations.sh | 2 +- ..._any_inner_join_filter_push_down.reference | 30 ------------ .../04812_any_inner_join_filter_push_down.sql | 49 ------------------- 5 files changed, 6 insertions(+), 117 deletions(-) delete mode 100644 tests/queries/0_stateless/04812_any_inner_join_filter_push_down.reference delete mode 100644 tests/queries/0_stateless/04812_any_inner_join_filter_push_down.sql diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 67653da47fae3..79c3786a7b0e1 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -454,11 +454,6 @@ static void projectDagInputs(ActionsDAG & actions_dag) } } -static bool isAnyInnerJoin(JoinKind kind, JoinStrictness strictness) -{ - return kind == JoinKind::Inner && (strictness == JoinStrictness::Any || strictness == JoinStrictness::RightAny); -} - std::optional tryToExtractPartialPredicate( const ActionsDAG & original_dag, const std::string & filter_name, @@ -620,23 +615,6 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: else if (logical_join && logical_join->getJoinOperator().kind == JoinKind::Right) left_stream_filter_push_down_input_columns_available = false; - /** `ANY INNER` join emits at most one row per key, deduplicating both sides. - * Both sides are blocked: filtering the right stream can change which match is taken for the left row. - * If the left side has multiple rows with the same value, only one survives - * and pushing the filter down to the left may affect which one survives. - * Also the optimizer is allowed to swap the sides of the join after that pass, - * thus a filter on the left may end up on the right. - * Predicates over the equi-join keys are still pushed to both sides through the equivalent-columns path below. - */ - const bool is_any_inner_join = (table_join_ptr && isAnyInnerJoin(table_join_ptr->kind(), table_join_ptr->strictness())) - || (logical_join && isAnyInnerJoin(logical_join->getJoinOperator().kind, logical_join->getJoinOperator().strictness)); - - if (is_any_inner_join) - { - right_stream_filter_push_down_input_columns_available = false; - left_stream_filter_push_down_input_columns_available = false; - } - /** We disable push down to right table in cases: * 1. Right side is already filled. Example: JOIN with Dictionary. * 2. ASOF Right join is not supported. @@ -809,14 +787,9 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: for (const auto & [name, _] : equivalent_left_stream_column_to_right_stream_column) equivalent_columns_to_push_down.push_back(name); } - else if (is_any_inner_join - || (logical_join && logical_join->getJoinOperator().kind == JoinKind::Right && logical_join->getJoinOperator().strictness == JoinStrictness::Semi)) + else if (logical_join && logical_join->getJoinOperator().kind == JoinKind::Right && logical_join->getJoinOperator().strictness == JoinStrictness::Semi) { - /// The `typeChangingSides` check is not needed in case of physical join, because - /// 1. `INNER` joins never widen the columns' types. - /// 2. On this path an `INNER` join cannot have been produced from an `OUTER` one by - /// `tryConvertOuterJoinToInnerJoin` and thus it's not carrying the preserved nullability. - if (!logical_join || !logical_join->typeChangingSides().contains(JoinTableSide::Left)) + if (!logical_join->typeChangingSides().contains(JoinTableSide::Left)) { /// In this case we can also push down to left side of JOIN using equivalent sets. for (const auto & [name, _] : equivalent_left_stream_column_to_right_stream_column) @@ -829,14 +802,9 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: for (const auto & [name, _] : equivalent_right_stream_column_to_left_stream_column) equivalent_columns_to_push_down.push_back(name); } - else if (is_any_inner_join - || (logical_join && logical_join->getJoinOperator().kind == JoinKind::Left && logical_join->getJoinOperator().strictness == JoinStrictness::Semi)) + else if (logical_join && logical_join->getJoinOperator().kind == JoinKind::Left && logical_join->getJoinOperator().strictness == JoinStrictness::Semi) { - /// The `typeChangingSides` check is not needed in case of physical join, because - /// 1. `INNER` joins never widen the columns' types. - /// 2. On this path an `INNER` join cannot have been produced from an `OUTER` one by - /// `tryConvertOuterJoinToInnerJoin` and thus it's not carrying the preserved nullability. - if (!logical_join || !logical_join->typeChangingSides().contains(JoinTableSide::Right)) + if (!logical_join->typeChangingSides().contains(JoinTableSide::Right)) { /// In this case we can also push down to right side of JOIN using equivalent sets. for (const auto & [name, _] : equivalent_right_stream_column_to_left_stream_column) diff --git a/tests/queries/0_stateless/01655_plan_optimizations.reference b/tests/queries/0_stateless/01655_plan_optimizations.reference index 9543d5aebc82b..fe7cc6d1eb856 100644 --- a/tests/queries/0_stateless/01655_plan_optimizations.reference +++ b/tests/queries/0_stateless/01655_plan_optimizations.reference @@ -130,7 +130,7 @@ Filter column: notEquals(__table1.number, 1_UInt8) > (analyzer) one condition of filter is pushed down before INNER JOIN Join Filter column: and(notEquals(__table1.number, 1_UInt8), notEquals(__table1.number, 2_UInt8)) -Filter column: and(notEquals(__table2.b, 1_UInt8), notEquals(__table2.b, 2_UInt8)) +Filter column: and(notEquals(__table2.b, 2_UInt8), notEquals(__table2.b, 1_UInt8)) 3 3 > filter is pushed down before UNION Union diff --git a/tests/queries/0_stateless/01655_plan_optimizations.sh b/tests/queries/0_stateless/01655_plan_optimizations.sh index 0890be12d6e24..96760de57b9f7 100755 --- a/tests/queries/0_stateless/01655_plan_optimizations.sh +++ b/tests/queries/0_stateless/01655_plan_optimizations.sh @@ -182,7 +182,7 @@ $CLICKHOUSE_CLIENT --enable_analyzer=1 -q " select number as a, r.b from numbers(4) as l any inner join ( select number + 2 as b from numbers(3) ) as r on a = r.b where a != 1 and b != 2 settings enable_optimize_predicate_expression = 0, query_plan_join_swap_table = 0, enable_join_runtime_filters = 0" | - grep -o " Join\|Filter column: and(notEquals(__table1.number, 1_UInt8), notEquals(__table1.number, 2_UInt8))\|Filter column: and(notEquals(__table2.b, 1_UInt8), notEquals(__table2.b, 2_UInt8))" + grep -o " Join\|Filter column: and(notEquals(__table1.number, 1_UInt8), notEquals(__table1.number, 2_UInt8))\|Filter column: and(notEquals(__table2.b, 2_UInt8), notEquals(__table2.b, 1_UInt8))" $CLICKHOUSE_CLIENT -q " select number as a, r.b from numbers(4) as l any inner join ( select number + 2 as b from numbers(3) diff --git a/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.reference b/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.reference deleted file mode 100644 index f6e2b4ce1d462..0000000000000 --- a/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.reference +++ /dev/null @@ -1,30 +0,0 @@ --- non-key filter must not be pushed to the right side - Filter column: equals(__table2.v, 1_UInt8) (removed) - Type: INNER - Strictness: ANY - ReadType: Default - ReadType: Default --- non-key filter must not be pushed to the left side - Filter column: equals(__table1.a, 1_UInt8) (removed) - Type: INNER - Strictness: ANY - ReadType: Default - ReadType: Default --- non-key filter may change the output set if pushed right -1 --- non-key filter may change the output set if pushed left -1 --- equi-key filter on the right column is still pushed to both sides - Type: INNER - Strictness: ANY - ReadType: Default - Prewhere filter column: equals(__table1.k, 1_UInt8) (removed) - ReadType: Default - Prewhere filter column: equals(__table2.k, 1_UInt8) (removed) --- equi-key filter on the left column is still pushed to both sides - Type: INNER - Strictness: ANY - ReadType: Default - Prewhere filter column: equals(__table1.k, 1_UInt8) (removed) - ReadType: Default - Prewhere filter column: equals(__table2.k, 1_UInt8) (removed) diff --git a/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.sql b/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.sql deleted file mode 100644 index 7f3c4abd1debe..0000000000000 --- a/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.sql +++ /dev/null @@ -1,49 +0,0 @@ -SET explain_query_plan_default = 'legacy'; -SET enable_analyzer = 1; -SET enable_parallel_replicas = 0; -SET query_plan_join_swap_table = 0; -SET enable_join_runtime_filters = 0; -SET query_plan_optimize_join_order_randomize = 0; -- the test asserts on the join plan -SET join_algorithm = 'hash'; -SET max_bytes_before_external_join = 0, max_bytes_ratio_before_external_join = 0; -- Disable automatic spilling for this test -SET optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1; - -CREATE TABLE t1 (k UInt64, a UInt64) ENGINE = MergeTree ORDER BY k; -CREATE TABLE t2 (k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k; -INSERT INTO t1 VALUES (1, 100), (2, 200), (1, 200); -INSERT INTO t2 VALUES (1, 10), (1, 20), (2, 30); - -SELECT '-- non-key filter must not be pushed to the right side'; -SELECT explain FROM ( - EXPLAIN actions = 1 - SELECT * FROM t1 ANY INNER JOIN t2 ON t1.k = t2.k WHERE t2.v = 1 -) WHERE explain ilike '%Filter column%' OR explain ilike '%Strictness%' OR explain ilike '%Type:%'; - -SELECT '-- non-key filter must not be pushed to the left side'; -SELECT explain FROM ( - EXPLAIN actions = 1 - SELECT * FROM t1 ANY INNER JOIN t2 ON t1.k = t2.k WHERE t1.a = 1 -) WHERE explain ilike '%Filter column%' OR explain ilike '%Strictness%' OR explain ilike '%Type:%'; - -SELECT '-- non-key filter may change the output set if pushed right'; -SELECT - (SELECT count() FROM t1 ANY JOIN t2 USING (k) WHERE t2.v > 10) - = (SELECT sum(t2.v > 10) FROM t1 ANY JOIN t2 USING (k)); - -SELECT '-- non-key filter may change the output set if pushed left'; -SELECT - (SELECT count() FROM t1 ANY JOIN t2 USING (k) WHERE t1.a = 200) - = (SELECT sum(t1.a = 200) FROM t1 ANY JOIN t2 USING (k)); - -SELECT '-- equi-key filter on the right column is still pushed to both sides'; -SELECT explain FROM ( - EXPLAIN actions = 1 - SELECT * FROM t1 ANY INNER JOIN t2 ON t1.k = t2.k WHERE t2.k = 1 -) WHERE explain ilike '%Filter column%' OR explain ilike '%Strictness%' OR explain ilike '%Type:%'; - -SELECT '-- equi-key filter on the left column is still pushed to both sides'; -SELECT explain FROM ( - EXPLAIN actions = 1 - SELECT * FROM t1 ANY INNER JOIN t2 ON t1.k = t2.k WHERE t1.k = 1 -) WHERE explain ilike '%Filter column%' OR explain ilike '%Strictness%' OR explain ilike '%Type:%'; - From 30bd0922500f0ab8ec62010a9adf4338cb1d950b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 19 Sep 2026 23:56:01 +0000 Subject: [PATCH 038/296] Backport #114059 to 26.8: Fix quadratic gzip/zlib streaming decompression of single-block streams --- contrib/libdeflate | 2 +- src/IO/LibdeflateInflatingReadBuffer.cpp | 49 ++-- src/IO/LibdeflateInflatingReadBuffer.h | 5 +- src/IO/tests/gtest_libdeflate_inflate.cpp | 231 ++++++++++++++++++ ...6_http_gzip_single_deflate_block.reference | 3 + .../04836_http_gzip_single_deflate_block.sh | 82 +++++++ 6 files changed, 344 insertions(+), 28 deletions(-) create mode 100644 tests/queries/0_stateless/04836_http_gzip_single_deflate_block.reference create mode 100755 tests/queries/0_stateless/04836_http_gzip_single_deflate_block.sh diff --git a/contrib/libdeflate b/contrib/libdeflate index ec0718b8e06dc..43f4fc23b7abe 160000 --- a/contrib/libdeflate +++ b/contrib/libdeflate @@ -1 +1 @@ -Subproject commit ec0718b8e06dc172eb87ede6a493865ccf7610ec +Subproject commit 43f4fc23b7abeb9bd078d54cab63271b3980fce7 diff --git a/src/IO/LibdeflateInflatingReadBuffer.cpp b/src/IO/LibdeflateInflatingReadBuffer.cpp index b04e2a324bada..8f1c438e4824e 100644 --- a/src/IO/LibdeflateInflatingReadBuffer.cpp +++ b/src/IO/LibdeflateInflatingReadBuffer.cpp @@ -91,7 +91,8 @@ bool LibdeflateInflatingReadBuffer::fillInput() in->nextIfAtEnd(); /// Copy a bounded amount per call so in_buf stays small even when the nested buffer exposes a /// lot at once (e.g. a memory-mapped file): the rest stays in the nested buffer for next time. - /// libdeflate consumes whole DEFLATE blocks, so the leftover we must keep is at most one block. + /// libdeflate consumes input up to the last decoded symbol, so the leftover we must keep is a + /// few bytes inside a Huffman block, and at most one stored block or block header otherwise. const size_t avail = std::min(in->buffer().end() - in->position(), INPUT_CHUNK); if (avail == 0) { @@ -290,11 +291,11 @@ bool LibdeflateInflatingReadBuffer::decompressImpl() window_nbytes = new_win; /// Decompress into the output region after the window, accumulating across as many - /// libdeflate calls as needed. Output produced at a non-final block boundary - /// (LIBDEFLATE_STREAM_NEED_INPUT) is deliberately NOT exposed yet: the stream may end at - /// the very next block, and its trailer must be validated before the final bytes reach the - /// caller. We expose the accumulated output only when the buffer fills at a block boundary - /// (the stream is then provably incomplete, so more output follows) or once the final block + /// libdeflate calls as needed. Output produced when the input runs out + /// (LIBDEFLATE_STREAM_NEED_INPUT) is deliberately NOT exposed yet: the stream may end + /// very soon after, and its trailer must be validated before the final bytes reach the + /// caller. We expose the accumulated output only when the buffer fills mid-stream (the + /// suspended item did not fit, so more output provably follows) or once the final block /// is decoded and its trailer verified. This mirrors ZlibInflatingReadBuffer, which has /// validated the trailer by the time it hands back the final bytes, so a reader that /// consumes an exact byte count with readStrict (and never calls nextImpl again) cannot @@ -338,11 +339,12 @@ bool LibdeflateInflatingReadBuffer::decompressImpl() if (r == LIBDEFLATE_STREAM_NEED_INPUT) { - /// At a non-final block boundary with the input exhausted. With end_of_input=true - /// the decoder never returns NEED_INPUT, so this means more input must exist; if the - /// nested stream is already at EOF the data is truncated. Otherwise pull more input - /// (or mark EOF, so the retry passes end_of_input=true and the final block finishes) - /// and keep accumulating into the same buffer. + /// Input exhausted; everything up to the last decoded symbol (or block boundary) was + /// consumed, so the retry never re-decodes more than a constant amount. With + /// end_of_input=true the decoder never returns NEED_INPUT, so this means more input + /// must exist; if the nested stream is already at EOF the data is truncated. + /// Otherwise pull more input (or mark EOF, so the retry passes end_of_input=true and + /// the final block finishes) and keep accumulating into the same buffer. if (input_eof) throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Unexpected end of {} stream", gzip ? "gzip" : "zlib"); fillInput(); @@ -353,23 +355,20 @@ bool LibdeflateInflatingReadBuffer::decompressImpl() { if (produced == 0) { - /* A single DEFLATE block's uncompressed size exceeds the whole output buffer. - * libdeflate's streaming decoder only suspends at block boundaries, so the whole - * block must be buffered before any of its output is exposed: grow the buffer (the - * 32 KiB window at the front is preserved) and retry. We grow geometrically and - * don't impose an artificial ceiling; the buffer is allocated through ClickHouse's - * tracked allocator, so a crafted single-block decompression bomb runs into the - * query/server memory limit and throws MEMORY_LIMIT_EXCEEDED, exactly like any - * other oversized allocation. Every mainstream gzip/zlib/deflate encoder bounds its - * blocks to well under a megabyte of uncompressed data (zlib flushes roughly every - * lit_bufsize symbols, libdeflate's SOFT_MAX_BLOCK_LENGTH is 300000 bytes, etc.), - * so a real-world stream never reaches this grow path; only a hand-crafted one does. */ + /* The streaming decoder suspends at symbol boundaries, so no progress at all + * means a single indivisible item - one match (<= 258 bytes) or one stored + * block (<= 64 KiB) - exceeds the whole free output region. That can only + * happen when the caller passed a tiny buf_size; growth is bounded by those + * item sizes, not by anything an attacker controls. Grow geometrically (the + * 32 KiB window at the front is preserved) and retry; the buffer is allocated + * through ClickHouse's tracked allocator, so even that bounded growth stays + * subject to the query/server memory limits. */ memory.resize(std::max(memory.size() + out_capacity, memory.size() * 2)); continue; } - /// Output buffer full at a block boundary: the stream is provably incomplete (the - /// final block was not reached), so more output follows and it is safe to expose - /// what we have without a trailer check. The window carries to the next call. + /// Output buffer full mid-stream: the suspended item itself did not fit, so more + /// output provably follows and it is safe to expose what we have without a + /// trailer check. The window carries to the next call. produced_end = window_nbytes + produced; break; } diff --git a/src/IO/LibdeflateInflatingReadBuffer.h b/src/IO/LibdeflateInflatingReadBuffer.h index e29bcae08d372..eac747fc3be91 100644 --- a/src/IO/LibdeflateInflatingReadBuffer.h +++ b/src/IO/LibdeflateInflatingReadBuffer.h @@ -16,9 +16,10 @@ struct libdeflate_decompressor; namespace DB { -/// Streaming gzip/zlib decompressor built on libdeflate's block-boundary-suspendable decoder +/// Streaming gzip/zlib decompressor built on libdeflate's symbol-boundary-suspendable decoder /// (libdeflate_deflate_decompress_stream, a ClickHouse addition). It is faster than the zlib -/// streaming path while keeping memory bounded. +/// streaming path while keeping memory bounded and time linear regardless of the stream's +/// DEFLATE block structure (a single block may span the whole stream, as with zlib-ng level 1). /// /// We parse the gzip/zlib header and trailer ourselves and feed the raw DEFLATE body to libdeflate, /// carrying the last 32 KiB of output as the back-reference window. Concatenated members are handled diff --git a/src/IO/tests/gtest_libdeflate_inflate.cpp b/src/IO/tests/gtest_libdeflate_inflate.cpp index 912867a82e59b..4ec771d36d711 100644 --- a/src/IO/tests/gtest_libdeflate_inflate.cpp +++ b/src/IO/tests/gtest_libdeflate_inflate.cpp @@ -11,9 +11,15 @@ #include +#include + +#include +#include +#include #include #include #include +#include #include using namespace DB; @@ -97,6 +103,19 @@ TEST_P(LibdeflateInflateTest, RoundTripFromZlibNg) } } +/// zlib-ng's deflate_quick path (compression level 1) opens a single static-Huffman block and +/// never closes it until the end of the stream, so a multi-megabyte level-1 stream is the +/// real-encoder version of the single-block shape of issue #114045; keep it large and the input +/// chunks small so a decoder that re-decodes from the block start on every refill cannot hide. +TEST_P(LibdeflateInflateTest, RoundTripFromZlibNgLevelOneLarge) +{ + const CompressionMethod method = GetParam(); + const std::string data = makeData(8 << 20, 3); + const std::string compressed = zlibCompress(data, method, 1); + for (size_t chunk : {size_t(16 * 1024), size_t(256 * 1024)}) + EXPECT_EQ(decompressViaBuffer(compressed, method, chunk, 1 << 20), data) << "chunk=" << chunk; +} + /// Two concatenated members must decode to the concatenation of their contents. TEST_P(LibdeflateInflateTest, MultiMember) { @@ -178,4 +197,216 @@ TEST(LibdeflateInflateHeaderValidation, RejectsBadGzipHeaderCrc) EXPECT_ANY_THROW(decompressViaBuffer(bad, CompressionMethod::Gzip, 4, 4096)); } +/// Regression tests for issue #114045: a single DEFLATE block spanning the whole stream - the shape +/// zlib-ng's deflate_quick path (compression level 1, the default of the official .NET SDK) emits - +/// must decompress in linear time with bounded memory. No real encoder is available here, so the +/// stream is crafted directly: a static-Huffman DEFLATE writer producing one block of arbitrary size. + +namespace +{ + +/// Emits a raw DEFLATE stream as a single static-Huffman block (RFC 1951 section 3.2.6). +class SingleBlockDeflateWriter +{ +public: + void putLiteral(uint8_t b) + { + if (b < 144) + putCode(0x30 + b, 8); + else + putCode(0x190 + (b - 144), 9); + reference.push_back(static_cast(b)); + } + + void putMatch(unsigned len /* 3..258 */, unsigned dist /* 1..32768, <= bytes emitted */) + { + static constexpr struct { unsigned base; int extra; } lengths[29] = { + {3,0},{4,0},{5,0},{6,0},{7,0},{8,0},{9,0},{10,0}, + {11,1},{13,1},{15,1},{17,1},{19,2},{23,2},{27,2},{31,2}, + {35,3},{43,3},{51,3},{59,3},{67,4},{83,4},{99,4},{115,4}, + {131,5},{163,5},{195,5},{227,5},{258,0}}; + static constexpr struct { unsigned base; int extra; } distances[30] = { + {1,0},{2,0},{3,0},{4,0},{5,1},{7,1},{9,2},{13,2},{17,3},{25,3}, + {33,4},{49,4},{65,5},{97,5},{129,6},{193,6},{257,7},{385,7}, + {513,8},{769,8},{1025,9},{1537,9},{2049,10},{3073,10}, + {4097,11},{6145,11},{8193,12},{12289,12},{16385,13},{24577,13}}; + + int li = len == 258 ? 28 : 27; + while (lengths[li].base > len) + --li; + unsigned length_sym = 257 + li; + if (length_sym < 280) + putCode(length_sym - 256, 7); + else + putCode(0xC0 + (length_sym - 280), 8); + if (lengths[li].extra) + putBits(len - lengths[li].base, lengths[li].extra); + + int di = 29; + while (distances[di].base > dist) + --di; + putCode(di, 5); + if (distances[di].extra) + putBits(dist - distances[di].base, distances[di].extra); + + for (unsigned i = 0; i < len; ++i) + reference.push_back(reference[reference.size() - dist]); + } + + /// Returns the raw DEFLATE stream; the object must not be used afterwards. + std::string finish() + { + putCode(0, 7); /* end-of-block */ + if (nbits) + deflate.push_back(static_cast(bitbuf)); + return std::move(deflate); + } + + std::string reference; /* the uncompressed bytes described so far */ + + SingleBlockDeflateWriter() + { + putBits(1, 1); /* BFINAL */ + putBits(1, 2); /* BTYPE: static Huffman */ + } + +private: + /// Raw bits, LSB-first (DEFLATE bit order for headers and extra bits). + void putBits(uint32_t v, int n) + { + bitbuf |= static_cast(v) << nbits; + nbits += n; + while (nbits >= 8) + { + deflate.push_back(static_cast(bitbuf)); + bitbuf >>= 8; + nbits -= 8; + } + } + + /// A Huffman codeword is written starting from its most significant bit. + void putCode(uint32_t c, int n) + { + uint32_t reversed = 0; + for (int i = 0; i < n; ++i) + reversed |= ((c >> i) & 1) << (n - 1 - i); + putBits(reversed, n); + } + + std::string deflate; + uint64_t bitbuf = 0; + int nbits = 0; +}; + +/// One gzip/zlib member whose DEFLATE payload is a single static-Huffman block: 'literal_bytes' of +/// pseudo-random literals followed by matches (with distances reaching the full 32 KiB window) up to +/// 'total_bytes' of output. Returns {member, reference}. +std::pair craftSingleBlockMember(CompressionMethod method, size_t literal_bytes, size_t total_bytes) +{ + SingleBlockDeflateWriter writer; + std::mt19937 rng(42); /// NOLINT(bugprone-random-generator-seed,cert-msc32-c,cert-msc51-cpp) deterministic test data on purpose + for (size_t i = 0; i < literal_bytes; ++i) + writer.putLiteral(static_cast(rng())); + while (writer.reference.size() < total_bytes) + writer.putMatch( + 3 + rng() % 256, + static_cast(1 + rng() % std::min(writer.reference.size(), 32768))); + std::string reference = writer.reference; + const std::string deflate = writer.finish(); + + std::string member; + if (method == CompressionMethod::Gzip) + { + member = std::string("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff", 10); + member += deflate; + const uint32_t crc = libdeflate_crc32(0, reference.data(), reference.size()); + const auto size32 = static_cast(reference.size()); + for (int i = 0; i < 4; ++i) + member.push_back(static_cast(crc >> (8 * i))); + for (int i = 0; i < 4; ++i) + member.push_back(static_cast(size32 >> (8 * i))); + } + else + { + member = std::string("\x78\x9c", 2); + member += deflate; + const uint32_t adler = libdeflate_adler32(1, reference.data(), reference.size()); + for (int i = 3; i >= 0; --i) + member.push_back(static_cast(adler >> (8 * i))); + } + return {member, reference}; +} + +} + +/// The whole member is one DEFLATE block: decompression must be correct through the read buffer, +/// including matches that cross the suspension points where the nested input chunks end. +TEST_P(LibdeflateInflateTest, SingleDeflateBlockSpanningWholeMember) +{ + const CompressionMethod method = GetParam(); + const auto [member, reference] = craftSingleBlockMember(method, 4 << 20, 8 << 20); + for (size_t chunk : {size_t(16 * 1024), size_t(256 * 1024)}) + EXPECT_EQ(decompressViaBuffer(member, method, chunk, 1 << 20), reference) << "chunk=" << chunk; +} + +/// The linearity contract underlying #114045: when the input runs out inside a Huffman block, the +/// streaming decoder must have consumed everything up to the last decoded symbol, so that the caller +/// never re-feeds (and the decoder never re-decodes) more than a constant amount. Before the fix, a +/// suspension rolled back to the block start: every call consumed nothing and re-decoded the whole +/// fed prefix, making ingestion quadratic in the block's compressed size. +TEST(LibdeflateSingleBlock, StreamingConsumesInputWithinBlock) +{ + SingleBlockDeflateWriter writer; + std::mt19937 rng(7); /// NOLINT(bugprone-random-generator-seed,cert-msc32-c,cert-msc51-cpp) deterministic test data on purpose + for (size_t i = 0; i < (4 << 20); ++i) + writer.putLiteral(static_cast(rng())); + const std::string reference = writer.reference; + const std::string deflate = writer.finish(); + + libdeflate_decompressor * decompressor = libdeflate_alloc_decompressor(); + ASSERT_NE(decompressor, nullptr); + libdeflate_deflate_decompress_stream_reset(decompressor); + + constexpr size_t window_max = 32768; + constexpr size_t chunk = 64 * 1024; + std::vector out(window_max + (1 << 20)); + std::string result; + size_t window = 0; + size_t consumed = 0; + size_t fed = std::min(deflate.size(), chunk); + + while (true) + { + const bool end_of_input = fed == deflate.size(); + size_t in_used = 0; + size_t out_used = 0; + const libdeflate_result r = libdeflate_deflate_decompress_stream( + decompressor, end_of_input ? 1 : 0, + deflate.data() + consumed, fed - consumed, + out.data() + window, out.size() - window, window, &in_used, &out_used); + result.append(out.data() + window, out_used); + consumed += in_used; + + const size_t total = window + out_used; + const size_t new_window = std::min(total, window_max); + memmove(out.data(), out.data() + (total - new_window), new_window); + window = new_window; + + if (r == LIBDEFLATE_SUCCESS) + break; + if (r == LIBDEFLATE_STREAM_NEED_INPUT) + { + ASSERT_FALSE(end_of_input); + /// The heart of the regression test: all but at most a suspended symbol plus the + /// refill lookahead must have been consumed. + ASSERT_LE(fed - consumed, 64u); + fed = std::min(deflate.size(), fed + chunk); + continue; + } + ASSERT_EQ(r, LIBDEFLATE_STREAM_NEED_OUTPUT); + } + libdeflate_free_decompressor(decompressor); + EXPECT_EQ(result, reference); +} + #endif diff --git a/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.reference b/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.reference new file mode 100644 index 0000000000000..5213944ab0aae --- /dev/null +++ b/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.reference @@ -0,0 +1,3 @@ +90 36000000 10606447025795110145 +90 36000000 10606447025795110145 +single-block ingest within 10x of multi-block: 1 diff --git a/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.sh b/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.sh new file mode 100755 index 0000000000000..781746aec0af7 --- /dev/null +++ b/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/114045: ingesting a gzip +# HTTP body whose DEFLATE payload is a single block spanning the whole stream - the shape +# zlib-ng's deflate_quick path (compression level 1, the default of the official .NET SDK) +# emits. The streaming decompressor used to re-decode the block from its start on every socket +# refill, making the ingest quadratic in the compressed size; a correct decoder handles this +# shape in linear time. No available encoder produces the shape on demand, so the gzip file is +# crafted directly: with the static Huffman code, every literal byte below 144 is an 8-bit +# codeword, so the block body is a byte translation of the payload shifted by the 3 header bits. +# +# The assertion is on time, because the quadratic decoder still produces the right bytes, just +# far too slowly. An absolute budget is not robust: the linear ingest of a 36 MB body takes +# 0.2 s on a release build but well over 15 s under TSan when the flaky check runs 18 copies of +# the test at once. So the same payload is ingested twice - first as an ordinary multi-block +# gzip, which every decoder handles in linear time, then as the single block - and the test +# asserts that the single-block ingest is at most 10x slower than the multi-block one. Both +# sides scale together with machine speed and load, so the ratio does not depend on either: +# measured on a release build, the quadratic decoder needs about 22 s for the single block +# against about 0.3 s for the multi-block stream (over 70x), and the linear decoder about the +# same time for both. The quadratic growth was confirmed to hold over the whole range +# (10/20/40/60 MB took 1.8/6.9/27.3/61.1 s), so the pre-fix margin does not depend on the +# machine being as fast as the one measured. The payload is 90 lines of 400 KB rather than many +# short ones so that line parsing and the `MergeTree` write cost nothing next to the +# decompression under test (with 520000 short lines they dominated at 5.8 s). The multi-block +# baseline is ingested first so that any one-off warm-up cost lands on the baseline side. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +MULTI_BLOCK_FILE=${CLICKHOUSE_TMP}/04836_multi_block.gz +SINGLE_BLOCK_FILE=${CLICKHOUSE_TMP}/04836_single_block.gz + +python3 -c " +import hashlib, struct, sys, zlib +size, line = 36000000, 400000 +chunk = b''.join(hashlib.sha256(b'%d' % i).hexdigest().encode() for i in range(size // 64 + 1))[:size] +raw = b'\n'.join(chunk[i:i + line] for i in range(0, size, line)) + b'\n' + +# The baseline: an ordinary gzip stream of many DEFLATE blocks, as any regular encoder emits. +# Level 6 (not 1) so that the shape does not depend on whether Python links zlib or zlib-ng. +multi = zlib.compressobj(6, zlib.DEFLATED, 31) +open(sys.argv[1], 'wb').write(multi.compress(raw) + multi.flush()) + +# The shape under test: the whole payload as one static-Huffman DEFLATE block. +assert max(raw) < 144 +table = bytes(int(format(0x30 + b, '08b')[::-1], 2) for b in range(144)) + bytes(112) +body = raw.translate(table) +# 3 header bits (BFINAL=1, BTYPE=01 static), 8 bits per literal, 7 zero bits of end-of-block. +n = (int.from_bytes(body, 'little') << 3) | 0b011 +deflate = n.to_bytes(len(body) + 2, 'little') +blob = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff' + deflate + + struct.pack(' MULTI_BLOCK_NS * 10 )); then + echo "single-block ingest took ${SINGLE_BLOCK_NS} ns, multi-block ingest took ${MULTI_BLOCK_NS} ns" +fi +echo "single-block ingest within 10x of multi-block: $(( SINGLE_BLOCK_NS <= MULTI_BLOCK_NS * 10 ))" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE t_04836" +rm -f "${MULTI_BLOCK_FILE}" "${SINGLE_BLOCK_FILE}" From 8acc8b34dc63ab35ea318d365e0a7a4bf442428c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 20 Sep 2026 02:38:43 +0000 Subject: [PATCH 039/296] Backport #113385 to 26.8: Fix TTLDrop merge of a table with a text index or an inert index --- src/Storages/MergeTree/MergeTask.cpp | 81 +++-- .../04790_ttl_drop_merge_text_index.reference | 26 ++ .../04790_ttl_drop_merge_text_index.sh | 280 ++++++++++++++++++ 3 files changed, 361 insertions(+), 26 deletions(-) create mode 100644 tests/queries/0_stateless/04790_ttl_drop_merge_text_index.reference create mode 100755 tests/queries/0_stateless/04790_ttl_drop_merge_text_index.sh diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 74f4d50043e73..7138a2685bb35 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -758,7 +758,31 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const hasLightweightDelete(global_ctx->future_part) || global_ctx->merging_params.mode != MergeTreeData::MergingParams::Ordinary; - prepareProjectionsToMergeAndRebuild(); + /// For TTLDrop merges, all source parts are fully expired. + /// Skip creating the read pipeline to avoid opening source parts + /// and allocating read/prefetch buffers. + /// + /// We restrict this to tables that have only an unconditional rows TTL + /// (no column TTL, moves, recompression, GROUP BY, or WHERE-clause TTL). + /// When other TTL families are present, TTLTransform::finalize rebuilds + /// their maps from scratch, and replicating that logic here would be + /// fragile. hasOnlyRowsTTL already excludes WHERE-clause TTLs. + /// + /// A merge cancelled after selection has `need_remove_expired_values` cleared above and must + /// not drop rows, so it falls through to the normal pipeline, which builds no TTLTransform. + const bool can_short_circuit_ttl_drop = + global_ctx->future_part->merge_type == MergeType::TTLDrop + && global_ctx->metadata_snapshot->hasOnlyRowsTTL() + && ctx->need_remove_expired_values; + + /// The short-circuit below commits a 0-row part without ever running a pipeline, so nothing + /// would retire these projections. Decide before the bookkeeping rather than undoing it + /// after: `prepareProjectionsToMergeAndRebuild` increments `MergedProjections` and + /// `RebuiltProjections` and pushes the names into `MergeListElement::projections_pending`, + /// which only the rebuild and merge paths erase from. Clearing the worklists afterwards left + /// those names in `system.merges.projections_remaining` for the lifetime of the merge entry. + if (!can_short_circuit_ttl_drop) + prepareProjectionsToMergeAndRebuild(); const auto & merge_tree_settings = global_ctx->data_settings; @@ -1036,19 +1060,6 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const && !use_const_adaptive_granularity && global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical; - /// For TTLDrop merges, all source parts are fully expired. - /// Skip creating the read pipeline to avoid opening source parts - /// and allocating read/prefetch buffers. - /// - /// We restrict this to tables that have only an unconditional rows TTL - /// (no column TTL, moves, recompression, GROUP BY, or WHERE-clause TTL). - /// When other TTL families are present, TTLTransform::finalize rebuilds - /// their maps from scratch, and replicating that logic here would be - /// fragile. hasOnlyRowsTTL already excludes WHERE-clause TTLs. - const bool can_short_circuit_ttl_drop = - global_ctx->future_part->merge_type == MergeType::TTLDrop - && global_ctx->metadata_snapshot->hasOnlyRowsTTL(); - if (can_short_circuit_ttl_drop) { LOG_DEBUG(ctx->log, "TTLDrop merge: skipping data pipeline, " @@ -1063,10 +1074,11 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const global_ctx->new_data_part->ttl_infos = {}; global_ctx->new_data_part->ttl_infos.table_ttl = {0, 0, true}; - /// Clear projections — no rows means no projection data to merge or rebuild. - global_ctx->projections_to_rebuild.clear(); - global_ctx->projections_to_merge.clear(); - global_ctx->projections_to_merge_parts.clear(); + /// No rows means no projection data to merge or rebuild, and the bookkeeping that would + /// have announced some was skipped above, so there is nothing to undo here. + chassert(global_ctx->projections_to_rebuild.empty()); + chassert(global_ctx->projections_to_merge.empty()); + chassert(global_ctx->projections_to_merge_parts.empty()); /// Force Horizontal algorithm. This prevents the Vertical stage from trying /// to finalize an empty rows_sources file, and ensures finalizePart takes @@ -1090,15 +1102,27 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const global_ctx->skip_indexes_by_column.clear(); global_ctx->text_indexes_to_merge.clear(); - auto all_skip_indexes = global_ctx->metadata_snapshot->getSecondaryIndices(); - for (const auto & index : all_skip_indexes) + /// Repopulate only when the setting asks for it: the clear above this branch already + /// honoured `materialize_skip_indexes_on_merge = 0`, and putting the indexes back would + /// override the user's choice on a TTLDrop merge only. + if ((*merge_tree_settings)[MergeTreeSetting::materialize_skip_indexes_on_merge]) { - if (!exclude_index_names.contains(index.name)) + auto all_skip_indexes = global_ctx->metadata_snapshot->getSecondaryIndices(); + for (const auto & index : all_skip_indexes) { - if (index.type == "text") - global_ctx->text_indexes_to_merge.push_back(index); - else - global_ctx->merging_skip_indexes.push_back(index); + if (!exclude_index_names.contains(index.name)) + { + /// Inert indices (a removed index type kept only for attach compatibility) hold + /// no data and cannot be recomputed. Skip them so the merge does not wedge + /// trying to aggregate them. + if (MergeTreeIndexFactory::instance().get(global_ctx->metadata_snapshot, index, *global_ctx->data_settings)->isInert()) + continue; + + if (index.type == "text") + global_ctx->text_indexes_to_merge.push_back(index); + else + global_ctx->merging_skip_indexes.push_back(index); + } } } } @@ -2478,7 +2502,12 @@ bool MergeTask::MergeTextIndexStage::prepare() const auto index_ptr = MergeTreeIndexFactory::instance().get(global_ctx->metadata_snapshot, index, *global_ctx->data_settings); std::vector segments; - if (global_ctx->merge_may_reduce_rows) + if (global_ctx->ttl_drop_short_circuit) + { + /// No read pipeline ran, so no transform built segments. The resulting part has no + /// rows, which is exactly what a 0-row pipeline would have produced anyway. + } + else if (global_ctx->merge_may_reduce_rows) { /// Text index was built for the resulting part. segments = getTextIndexSegments(global_ctx->new_data_part->name, index.name, 0); diff --git a/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.reference b/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.reference new file mode 100644 index 0000000000000..d2e67b3bbdc33 --- /dev/null +++ b/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.reference @@ -0,0 +1,26 @@ +-- Case 1: text index +0 +idx_txt text 1 +1 +-- Case 2: materialize_skip_indexes_on_merge = 0 is respected +0 +idx_set set 0 +idx_txt text 0 +1 +-- Case 3: inert index +1 +0 +i0 hypothesis 0 +idx_txt text 1 +1 +-- Case 4: default TTL settings +0 +1 +1 +-- Case 5: projections are not announced as pending work +0 +1 +1 +0 +1 +1 diff --git a/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.sh b/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.sh new file mode 100755 index 0000000000000..f1c57f1260d91 --- /dev/null +++ b/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash +# Tags: long + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A TTLDrop merge takes the short-circuit that skips the read pipeline. Every index the +# resulting empty part must still carry has to be handled there, because the builders that +# normally produce them live inside the skipped pipeline. +# +# Only the background selector assigns MergeType::TTLDrop; OPTIMIZE TABLE FINAL assigns +# MergeType::Regular. Every case below therefore drives a background TTL merge and waits for +# it, bounded by wall-clock time rather than a fixed iteration count. + +function wait_for_ttl_drop() +{ + local table=$1 + local deadline=$((SECONDS + 90)) + while [ "$SECONDS" -lt "$deadline" ]; do + local rows + rows=$(${CLICKHOUSE_CLIENT} -q "SELECT count() FROM $table") + if [ "$rows" = "0" ]; then + return + fi + sleep 0.5 + done + echo "timed out waiting for the TTL drop merge on $table" +} + +echo "-- Case 1: text index" + +# Before the fix the merge threw LOGICAL_ERROR 'Text index transform for index ... not found' +# and retried forever, so the expired rows were never dropped. +${CLICKHOUSE_CLIENT} -q " + SET allow_experimental_full_text_index = 1; + + CREATE TABLE t_ttl_drop_text + ( + id UInt64, + value String, + event_time DateTime DEFAULT now() - INTERVAL 2 DAY, + INDEX idx_txt value TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 + ) + ENGINE = MergeTree() + ORDER BY id + TTL event_time + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 1, + -- keep the 0-row part so the index assertions below have something to read + remove_empty_parts = 0; + + SYSTEM STOP MERGES t_ttl_drop_text; + + INSERT INTO t_ttl_drop_text (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + INSERT INTO t_ttl_drop_text (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + + SYSTEM START MERGES t_ttl_drop_text; +" + +wait_for_ttl_drop "t_ttl_drop_text" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_ttl_drop_text;" + +# The empty part must still carry the text index files, byte-for-byte as the normal 0-row +# path writes them: a replica that fetches the part compares checksums, so an index dropped +# here would diverge. data_compressed_bytes counts the serialized index header, which is +# non-empty even with no tokens -- it reads 0 when the index files are missing entirely. +${CLICKHOUSE_CLIENT} -q " + SELECT name, type, data_compressed_bytes > 0 + FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = 't_ttl_drop_text'; +" + +${CLICKHOUSE_CLIENT} -q "CHECK TABLE t_ttl_drop_text SETTINGS check_query_single_value_result = 1;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_ttl_drop_text;" + +echo "-- Case 2: materialize_skip_indexes_on_merge = 0 is respected" + +# The 'forget about skip indexes' clear runs before the short-circuit, which used to put +# every index straight back -- so the setting was honoured for a normal merge and silently +# ignored for a TTLDrop merge. With a text index that also reintroduced case 1's failure. +${CLICKHOUSE_CLIENT} -q " + SET allow_experimental_full_text_index = 1; + + CREATE TABLE t_ttl_drop_no_materialize + ( + id UInt64, + value String, + event_time DateTime DEFAULT now() - INTERVAL 2 DAY, + INDEX idx_txt value TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1, + INDEX idx_set id TYPE set(100) GRANULARITY 1 + ) + ENGINE = MergeTree() + ORDER BY id + TTL event_time + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 1, + materialize_skip_indexes_on_merge = 0, + remove_empty_parts = 0; + + SYSTEM STOP MERGES t_ttl_drop_no_materialize; + + INSERT INTO t_ttl_drop_no_materialize (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + INSERT INTO t_ttl_drop_no_materialize (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + + SYSTEM START MERGES t_ttl_drop_no_materialize; +" + +wait_for_ttl_drop "t_ttl_drop_no_materialize" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_ttl_drop_no_materialize;" + +# The setting suppresses both index families, so neither writes any files. +${CLICKHOUSE_CLIENT} -q " + SELECT name, type, data_compressed_bytes > 0 + FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = 't_ttl_drop_no_materialize' + ORDER BY name; +" + +${CLICKHOUSE_CLIENT} -q "CHECK TABLE t_ttl_drop_no_materialize SETTINGS check_query_single_value_result = 1;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_ttl_drop_no_materialize;" + +echo "-- Case 3: inert index" + +# A legacy 'hypothesis' index is inert: it holds no data and cannot be recomputed, so +# createIndexAggregator rejects it with ILLEGAL_INDEX. The short-circuit used to omit the +# isInert filter the normal path applies, which wedged the merge and left the rows in place. +# The non-inert sibling pins that the filter is per index rather than per table: it must still +# materialize while the inert one is skipped. +# Full-definition ATTACH is the only way to get such a table, and it needs an explicit UUID. +uuid=$(${CLICKHOUSE_CLIENT} -q "SELECT generateUUIDv4()") +${CLICKHOUSE_CLIENT} -q " + SET send_logs_level = 'fatal'; + SET allow_experimental_full_text_index = 1; + + ATTACH TABLE t_ttl_drop_inert UUID '$uuid' + ( + id UInt64, + value String, + event_time DateTime DEFAULT now() - INTERVAL 2 DAY, + INDEX i0 90 % id TYPE hypothesis, + INDEX idx_txt value TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 + ) + ENGINE = MergeTree() + PRIMARY KEY tuple() + TTL event_time + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 1, + remove_empty_parts = 0; +" + +${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = 't_ttl_drop_inert' AND type = 'hypothesis'; +" + +${CLICKHOUSE_CLIENT} -q " + SYSTEM STOP MERGES t_ttl_drop_inert; + + INSERT INTO t_ttl_drop_inert (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + INSERT INTO t_ttl_drop_inert (id, value) SELECT number + 100, 'w' || toString(number) FROM numbers(100); + + SYSTEM START MERGES t_ttl_drop_inert; +" + +wait_for_ttl_drop "t_ttl_drop_inert" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_ttl_drop_inert;" + +# The inert index is skipped individually; its non-inert sibling still gets materialized. +${CLICKHOUSE_CLIENT} -q " + SELECT name, type, data_compressed_bytes > 0 + FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = 't_ttl_drop_inert' + ORDER BY name; +" + +${CLICKHOUSE_CLIENT} -q "CHECK TABLE t_ttl_drop_inert SETTINGS check_query_single_value_result = 1;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_ttl_drop_inert;" + +echo "-- Case 4: default TTL settings" + +# TTLDrop selection ignores merge_with_ttl_timeout: TTLPartDropMergeSelector passes a null +# merge_due_times, and selectPartsToMerge schedules no next time for MergeType::TTLDrop. So +# the short-circuit is reachable with every TTL setting left at its default, including +# ttl_only_drop_parts = 0, which the cases above do not cover. +# +# No OPTIMIZE here: with one level-0 part it would select a MergeType::Regular merge of its +# own and could empty the table through the normal pipeline before the background TTLDrop is +# picked up. Only the background selector produces the merge type this case is about. +${CLICKHOUSE_CLIENT} -q " + SET allow_experimental_full_text_index = 1; + + CREATE TABLE t_ttl_drop_default + ( + c0 UInt64, + c1 Date, + c2 String, + INDEX idx0 c2 TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 + ) + ENGINE = MergeTree() + ORDER BY c0 + TTL c1 + INTERVAL 1 SECOND DELETE; + + INSERT INTO t_ttl_drop_default SELECT number, toDate('2020-01-01'), 'w' || toString(number) FROM numbers(100); +" + +wait_for_ttl_drop "t_ttl_drop_default" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_ttl_drop_default;" + +# Emptying the table is not enough on its own: assert the merge that did it was a TTLDrop +# that succeeded and read no rows. error = 0 excludes the failed-and-retried merge the +# unfixed code produced, which MergePlainMergeTreeTask also logs under this merge_reason; +# read_rows = 0 is what distinguishes the skipped pipeline from one that ran. +${CLICKHOUSE_CLIENT} -q " + SYSTEM FLUSH LOGS part_log; + SELECT countIf(merge_reason = 'TTLDropMerge' AND error = 0 AND read_rows = 0) > 0 + FROM system.part_log + WHERE database = currentDatabase() AND table = 't_ttl_drop_default' AND event_type = 'MergeParts'; +" + +${CLICKHOUSE_CLIENT} -q "CHECK TABLE t_ttl_drop_default SETTINGS check_query_single_value_result = 1;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_ttl_drop_default;" + +echo "-- Case 5: projections are not announced as pending work" + +# prepareProjectionsToMergeAndRebuild both bumps the projection profile events and pushes the +# names into MergeListElement::projections_pending, which system.merges.projections_remaining +# reads and which only the rebuild and merge paths erase from. The short-circuit runs neither. +# +# projections_pending is fed from two lists, and deduplicate_merge_projection_mode decides which +# one a projection lands in, so both modes are covered: asserting RebuiltProjections alone passes +# on unfixed code under 'ignore', where the name arrives via projections_to_merge instead. +for mode in throw ignore +do + table="t_ttl_drop_projection_$mode" + + ${CLICKHOUSE_CLIENT} -q " + CREATE TABLE $table + ( + c0 UInt64, + c1 Date, + c2 String, + PROJECTION p0 (SELECT c2, count() GROUP BY c2) + ) + ENGINE = MergeTree() + ORDER BY c0 + TTL c1 + INTERVAL 1 SECOND DELETE + SETTINGS deduplicate_merge_projection_mode = '$mode'; + + INSERT INTO $table SELECT number, toDate('2020-01-01'), 'w' || toString(number) FROM numbers(100); + " + + wait_for_ttl_drop "$table" + + ${CLICKHOUSE_CLIENT} -q "SELECT count() FROM $table;" + + # read_rows = 0 pins the short-circuit; both counters at 0 are the assertion. Each mode gets + # its own table so this reads only its own merge. + ${CLICKHOUSE_CLIENT} -q " + SYSTEM FLUSH LOGS part_log; + SELECT countIf(merge_reason = 'TTLDropMerge' AND error = 0 AND read_rows = 0 + AND ProfileEvents['RebuiltProjections'] = 0 + AND ProfileEvents['MergedProjections'] = 0) > 0 + FROM system.part_log + WHERE database = currentDatabase() AND table = '$table' AND event_type = 'MergeParts'; + " + + ${CLICKHOUSE_CLIENT} -q "CHECK TABLE $table SETTINGS check_query_single_value_result = 1;" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE $table;" +done From da3e19a2c03fafa24aa39cfb5672fe7517756f58 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 20 Sep 2026 08:16:40 +0000 Subject: [PATCH 040/296] Backport #120948 to 26.8: Fix CAST of an Array(Map) to an array whose element type hides the Map --- src/Functions/FunctionsConversion.cpp | 47 ++++++++++++++----- ...229_cast_array_of_map_to_variant.reference | 7 +++ .../05229_cast_array_of_map_to_variant.sql | 21 +++++++++ 3 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 tests/queries/0_stateless/05229_cast_array_of_map_to_variant.reference create mode 100644 tests/queries/0_stateless/05229_cast_array_of_map_to_variant.sql diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index c71f20f6ab383..726781d7ea8c4 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -842,21 +842,42 @@ FunctionCast::WrapperType FunctionCast::createAggregateFunctionWrapper(const Dat namespace { -/// `Map(K, V)` is physically an `Array(Tuple(K, V))`, and `CAST` converts between the two shapes at -/// the top level. It is also how a `Map` constant is written down when the analyzer serializes a -/// query for a remote server: `ConstantNode::toASTImpl` renders the field as an array of tuples and -/// wraps it into `_CAST(..., 'Array(Map(K, V))')`. Counting a `Map` as the array it stands for puts -/// both shapes at the same depth, so the nesting check below does not reject a cast the element -/// wrappers can perform. A genuine depth mismatch, such as `Array(String)` to `Array(Array(String))`, -/// is still rejected. A `Tuple` adds no depth, hence a `Map` counts as one dimension regardless of -/// its value type. -size_t getNumberOfDimensionsWithMapAsArray(const IDataType & type) +/// The number of `Array` dimensions of a type as `CAST` sees it. It is a range rather than a single +/// number, because `Map(K, V)` is physically an `Array(Tuple(K, V))` and `CAST` converts between the +/// two spellings: a `Map` can be matched either by another `Map`, contributing no dimension of its +/// own, or by the `Array(Tuple(K, V))` it stands for, contributing one. A `Tuple` adds no dimension, +/// hence a `Map` is at most one dimension whatever its value type is. +/// +/// The array spelling is how a `Map` constant is written down when the analyzer serializes a query +/// for a remote server: `ConstantNode::toASTImpl` renders the field as an array of tuples and wraps +/// it into `_CAST(..., 'Array(Map(K, V))')`. +struct DimensionsRange +{ + size_t min; + size_t max; +}; + +DimensionsRange getNumberOfDimensionsForCast(const IDataType & type) { if (const auto * type_array = typeid_cast(&type)) - return 1 + getNumberOfDimensionsWithMapAsArray(*type_array->getNestedType()); + { + const DimensionsRange nested = getNumberOfDimensionsForCast(*type_array->getNestedType()); + return {nested.min + 1, nested.max + 1}; + } if (typeid_cast(&type)) - return 1; - return 0; + return {0, 1}; + return {0, 0}; +} + +/// The nesting depths have to be reconcilable under some spelling of the `Map`s the two types +/// contain. This rejects a genuine depth mismatch, such as `Array(String)` to `Array(Array(String))`, +/// before an element wrapper does something unexpected with it (`CAST` from a `String` parses it). +/// Anything else is left to the element wrappers, which name the types that cannot be converted. +bool canHaveSameNumberOfDimensions(const IDataType & from, const IDataType & to) +{ + const DimensionsRange from_dimensions = getNumberOfDimensionsForCast(from); + const DimensionsRange to_dimensions = getNumberOfDimensionsForCast(to); + return from_dimensions.min <= to_dimensions.max && to_dimensions.min <= from_dimensions.max; } } @@ -914,7 +935,7 @@ FunctionCast::WrapperType FunctionCast::createArrayWrapper(const DataTypePtr & f /// In query SELECT CAST([] AS Array(Array(String))) from type is Array(Nothing) bool from_empty_array = isNothing(from_nested_type); - if (getNumberOfDimensionsWithMapAsArray(*from_type) != getNumberOfDimensionsWithMapAsArray(to_type) && !from_empty_array) + if (!canHaveSameNumberOfDimensions(*from_type, to_type) && !from_empty_array) throw Exception(ErrorCodes::TYPE_MISMATCH, "CAST AS Array can only be performed between same-dimensional array types"); diff --git a/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.reference b/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.reference new file mode 100644 index 0000000000000..b4dbd49ecd01a --- /dev/null +++ b/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.reference @@ -0,0 +1,7 @@ +[{'a':'x'}] +[{'k':['v']}] +[{'a':'x'}] +[[{'a':'x'}]] +[{'a':'x'}] +['{"a":"x"}'] +['{\'a\':\'x\'}'] diff --git a/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.sql b/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.sql new file mode 100644 index 0000000000000..e472cff5b8a9e --- /dev/null +++ b/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.sql @@ -0,0 +1,21 @@ +-- A `Map(K, V)` is physically an `Array(Tuple(K, V))` and `CAST` converts between the two spellings, +-- so a `Map` in one of the types may or may not be spelled out as an array by the other one. +-- Counting a `Map` as one dimension unconditionally made the nesting check of `CAST AS Array` reject +-- an `Array(Map(...))` whose target element type keeps the `Map` behind a `Variant`, a `Dynamic`, +-- a `JSON` or a `String`. + +SELECT CAST([map('a', 'x')], 'Array(Variant(String, Map(String, String)))'); +SELECT CAST([map('k', ['v'])], 'Array(Variant(String, Map(String, Array(String))))'); +SELECT CAST(materialize([map('a', 'x')]), 'Array(Variant(String, Map(String, String)))'); +SELECT CAST([[map('a', 'x')]], 'Array(Array(Variant(String, Map(String, String))))'); +SELECT CAST([map('a', 'x')], 'Array(Dynamic)'); +SELECT CAST([map('a', 'x')], 'Array(JSON)'); +SELECT CAST([map('a', 'x')], 'Array(String)'); + +-- What cannot be converted is reported by the element wrapper, which names the offending types. +SELECT CAST([map('a', 'x')], 'Array(Variant(String, UInt8))'); -- { serverError CANNOT_CONVERT_TYPE } +SELECT CAST([map('a', 'x')], 'Array(Tuple(String, String))'); -- { serverError TYPE_MISMATCH } + +-- A genuine nesting mismatch is still rejected. +SELECT CAST(['v'], 'Array(Array(String))'); -- { serverError TYPE_MISMATCH } +SELECT CAST([['v']], 'Array(String)'); -- { serverError TYPE_MISMATCH } From 01b6baf4f412f86776aa1ba62f3d85f2c2b207c8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 20 Sep 2026 08:17:30 +0000 Subject: [PATCH 041/296] Backport #120698 to 26.8: Fix referring to a parameterized view by its own name in the analyzer --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 24 ++++++ src/Analyzer/TableFunctionNode.cpp | 9 ++- src/Analyzer/TableFunctionNode.h | 3 + src/Planner/PlannerJoinTree.cpp | 2 +- src/Planner/Utils.cpp | 7 +- ...4653_parameterized_view_own_name.reference | 37 +++++++++ .../04653_parameterized_view_own_name.sql | 78 +++++++++++++++++++ ...eterized_view_own_name_qualifier.reference | 33 ++++++++ ...7_parameterized_view_own_name_qualifier.sh | 60 ++++++++++++++ 9 files changed, 245 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/04653_parameterized_view_own_name.reference create mode 100644 tests/queries/0_stateless/04653_parameterized_view_own_name.sql create mode 100644 tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.reference create mode 100755 tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.sh diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 25e58030e9e3b..3c684a85673b4 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -905,6 +905,11 @@ void QueryAnalyzer::validateJoinTableExpressionWithoutAlias(const QueryTreeNodeP if ((query_node && !query_node->getCTEName().empty()) || (union_node && !union_node->getCTEName().empty())) return; + /// A parameterized view has a name to qualify its columns with, so it is exempt like the plain table below. + if (const auto * table_function_node = table_expression_node->as(); + table_function_node && table_function_node->isParameterizedView()) + return; + auto table_expression_node_type = table_expression_node->getNodeType(); if (table_expression_node_type == QueryTreeNodeType::TABLE_FUNCTION || @@ -1650,6 +1655,13 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & if (table_node->isMaterializedCTE()) additional_column_qualification_parts = {table_node->getMaterializedCTE()->cte_name}; } + else if (auto * table_function_node = table_expression_node->as(); + table_function_node && table_function_node->isParameterizedView()) + { + /// A parameterized view has a name of its own, qualify with it exactly like for a `TableNode`. + const auto & table_storage_id = table_function_node->getStorageID(); + additional_column_qualification_parts = {table_storage_id.getDatabaseName(), table_storage_id.getTableName()}; + } else if (auto * query_node = table_expression_node->as(); query_node && query_node->isCTE()) additional_column_qualification_parts = {query_node->getCTEName()}; else if (auto * union_node = table_expression_node->as(); union_node && union_node->isCTE()) @@ -1685,6 +1697,9 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & else forced_qualifier = table_node->getStorageID().getTableName(); } + else if (auto * table_function_node = table_expression_node->as(); + table_function_node && table_function_node->isParameterizedView()) + forced_qualifier = table_function_node->getStorageID().getTableName(); else if (auto * query_node = table_expression_node->as(); query_node && query_node->isCTE()) forced_qualifier = query_node->getCTEName(); else if (auto * union_node = table_expression_node->as(); union_node && union_node->isCTE()) @@ -4432,6 +4447,15 @@ void QueryAnalyzer::initializeTableExpressionData(const TableExpressionNodePtr & else if (table_function_node) { table_expression_data.table_expression_description = "table_function"; + + /// A parameterized view has a name of its own, expose it exactly like a `TableNode` does. + if (table_function_node->isParameterizedView()) + { + const auto & table_storage_id = table_function_node->getStorageID(); + table_expression_data.database_name = table_storage_id.database_name; + table_expression_data.table_name = table_storage_id.table_name; + table_expression_data.table_expression_name = table_storage_id.getFullNameNotQuoted(); + } } if (table_expression_node->hasAlias()) diff --git a/src/Analyzer/TableFunctionNode.cpp b/src/Analyzer/TableFunctionNode.cpp index a4aeb95a021d5..dcd62f670a175 100644 --- a/src/Analyzer/TableFunctionNode.cpp +++ b/src/Analyzer/TableFunctionNode.cpp @@ -71,6 +71,12 @@ const StorageSnapshotPtr & TableFunctionNode::getStorageSnapshot() const return storage_snapshot; } +bool TableFunctionNode::isParameterizedView() const +{ + const auto * storage_view = storage ? storage->as() : nullptr; + return storage_view && storage_view->isParameterizedView(); +} + void TableFunctionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "TABLE_FUNCTION id: " << format_state.getNodeId(this); @@ -167,8 +173,7 @@ ASTPtr TableFunctionNode::toASTImpl(const ConvertToASTOptions & options) const /// An unqualified parameterized-view name re-resolves against the receiving server's default /// database, so qualify it from `storage_id`. Only a 2-part result is resolvable as a /// parameterized view, so a dotted database name is left alone. - if (const auto * storage_view = storage ? storage->as() : nullptr; - storage_view && storage_view->isParameterizedView() && storage_id.hasDatabase() + if (isParameterizedView() && storage_id.hasDatabase() && Identifier{table_function_name}.getPartsSize() == 1) { const auto database_name = storage_id.getDatabaseName(); diff --git a/src/Analyzer/TableFunctionNode.h b/src/Analyzer/TableFunctionNode.h index 8548b5d0c26e6..d8f150e1e7271 100644 --- a/src/Analyzer/TableFunctionNode.h +++ b/src/Analyzer/TableFunctionNode.h @@ -101,6 +101,9 @@ class TableFunctionNode : public ITableExpressionNode return storage; } + /// True for a parameterized view call resolved into its `StorageView` (no real table function behind it) + bool isParameterizedView() const; + /// Resolve table function with table function, storage and context void resolve(TableFunctionPtr table_function_value, StoragePtr storage_value, ContextPtr context, VectorWithMemoryTracking unresolved_arguments_indexes_); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index e4a6018359a4d..67ca7fb5945a6 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -752,7 +752,7 @@ void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expr /// would let the query read the view without any `SELECT` grant. Enforce the same column-aware `SELECT` /// check the underlying view would receive as a `TableNode`. const auto & storage = table_function_node->getStorage(); - if (const auto * storage_view = storage ? storage->as() : nullptr; storage_view && storage_view->isParameterizedView()) + if (table_function_node->isParameterizedView()) { const auto & column_names_with_aliases = table_expression_data.getSelectedColumnsNames(); columns_names_allowed_to_select = checkAccessRights( diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 67311e1635a36..a54f859f69fb3 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -26,7 +26,6 @@ #include #include -#include #include #include @@ -639,12 +638,10 @@ static void checkAccessRightsForFilter(const QueryTreeNodePtr & filter_query_tre { /// A parameterized view is resolved as a `TableFunctionNode` wrapping a real `StorageView`, see /// `prepareBuildQueryPlanForTableExpression`. Regular table functions are checked in `ITableFunction::execute`. - const auto & table_function_storage = table_function_node->getStorage(); - const auto * storage_view = table_function_storage ? table_function_storage->as() : nullptr; - if (!storage_view || !storage_view->isParameterizedView()) + if (!table_function_node->isParameterizedView()) return; - storage = table_function_storage; + storage = table_function_node->getStorage(); storage_id = table_function_node->getStorageID(); storage_snapshot = table_function_node->getStorageSnapshot(); } diff --git a/tests/queries/0_stateless/04653_parameterized_view_own_name.reference b/tests/queries/0_stateless/04653_parameterized_view_own_name.reference new file mode 100644 index 0000000000000..00065a0b9fce2 --- /dev/null +++ b/tests/queries/0_stateless/04653_parameterized_view_own_name.reference @@ -0,0 +1,37 @@ +-- a parameterized view may be joined without an alias +1 +1 +1 +1 +-- its columns may be qualified with the view name +1 +1 +t1 +1 +1 +1 +-- matcher-expanded columns are qualified with the view name +tenant_id String +host_id UInt64 +pv.tenant_id String +pv.host_id UInt64 +tenant_id String +host_id UInt64 +pv.host_id UInt64 +-- control: a real table function contributes no qualifier +tenant_id String +host_id UInt64 +number UInt64 +-- the parameter type is irrelevant +1 +-- controls: an alias or no qualifier always worked +1 +1 +-- controls: a regular table function and an ordinary view must not gain a bindable name +tenant_id String +host_id UInt64 +tenant_id String +-- a view in another database binds by the view name too +1 +1 +-- control: qualifying with a database that does not hold the view does not bind diff --git a/tests/queries/0_stateless/04653_parameterized_view_own_name.sql b/tests/queries/0_stateless/04653_parameterized_view_own_name.sql new file mode 100644 index 0000000000000..99a2904a47be1 --- /dev/null +++ b/tests/queries/0_stateless/04653_parameterized_view_own_name.sql @@ -0,0 +1,78 @@ +-- Tags: need-query-parameters + +-- The fix is in the query-tree analyzer, so pin it: the `old analyzer` CI jobs link +-- `users.d/analyzer.xml` and the randomized `compatibility='<24.3'` setting also reverts +-- `allow_experimental_analyzer`, and on the legacy path a JOINed parameterized view is not +-- expanded at all (only the left table storage is), so these rows would assert a different +-- code path's behaviour. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS local_data; +DROP VIEW IF EXISTS pv; +DROP VIEW IF EXISTS pv2; + +CREATE TABLE local_data (tenant_id String, host_id UInt64) ENGINE = MergeTree ORDER BY tenant_id; +INSERT INTO local_data SELECT 't1', 1; + +CREATE VIEW pv AS SELECT tenant_id, host_id FROM local_data WHERE tenant_id IN ({tenants:Array(String)}); +CREATE VIEW pv2 AS SELECT tenant_id, host_id FROM local_data WHERE tenant_id = {tenant:String}; + +SELECT '-- a parameterized view may be joined without an alias'; +SELECT count() FROM local_data JOIN pv(tenants = ['t1']) USING (tenant_id); +SELECT count() FROM local_data AS t JOIN pv(tenants = ['t1']) ON t.tenant_id = pv.tenant_id; +SELECT count() FROM local_data, pv(tenants = ['t1']); +SELECT count() FROM pv(tenants = ['t1']) JOIN local_data AS t USING (tenant_id); + +SELECT '-- its columns may be qualified with the view name'; +SELECT pv.host_id FROM pv(tenants = ['t1']); +SELECT host_id FROM pv(tenants = ['t1']) WHERE pv.host_id = 1; +SELECT pv.tenant_id FROM pv(tenants = ['t1']) GROUP BY pv.tenant_id; +SELECT pv.host_id FROM pv(tenants = ['t1']) ORDER BY pv.host_id; +SELECT {CLICKHOUSE_DATABASE:Identifier}.pv.host_id FROM pv(tenants = ['t1']); +SELECT count() FROM local_data AS t JOIN pv(tenants = ['t1']) ON t.tenant_id = pv.tenant_id SETTINGS joined_subquery_requires_alias = 0; + +SELECT '-- matcher-expanded columns are qualified with the view name'; +DESCRIBE (SELECT * FROM local_data, pv(tenants = ['t1'])); +DESCRIBE (SELECT * FROM local_data JOIN pv(tenants = ['t1']) USING (tenant_id)); + +SELECT '-- control: a real table function contributes no qualifier'; +DESCRIBE (SELECT * FROM local_data AS t JOIN numbers(3) AS n ON 1 = 1); + +SELECT '-- the parameter type is irrelevant'; +SELECT pv2.host_id FROM pv2(tenant = 't1'); + +SELECT '-- controls: an alias or no qualifier always worked'; +SELECT p.host_id FROM pv(tenants = ['t1']) AS p; +SELECT host_id FROM pv(tenants = ['t1']); + +SELECT '-- controls: a regular table function and an ordinary view must not gain a bindable name'; +SELECT numbers.number FROM numbers(3); -- { serverError UNKNOWN_IDENTIFIER } +SELECT count() FROM local_data JOIN numbers(3) ON 1 = 1; -- { serverError ALIAS_REQUIRED } +SELECT view.dummy FROM view(SELECT 1 AS dummy); -- { serverError UNKNOWN_IDENTIFIER } +SELECT count() FROM local_data JOIN view(SELECT 1 AS dummy) ON 1 = 1; -- { serverError ALIAS_REQUIRED } +-- `tenant_id` collides with `local_data`'s, so the matcher must decide whether to qualify it; +-- an ordinary view contributes no qualification parts, so the second one stays bare. +DESCRIBE (SELECT * FROM local_data, view(SELECT 't1' AS tenant_id)) SETTINGS joined_subquery_requires_alias = 0; + +DROP VIEW pv2; +DROP VIEW pv; +DROP TABLE local_data; + +-- The view also binds by its own name when it lives outside the session's current database. +-- The call itself stays unqualified: a query parameter is not accepted in the database +-- position of a table function, and a literal database name would not be parallel-safe. +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +USE {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE TABLE local_data (tenant_id String, host_id UInt64) ENGINE = MergeTree ORDER BY tenant_id; +INSERT INTO local_data SELECT 't1', 1; +CREATE VIEW pv AS SELECT tenant_id, host_id FROM local_data WHERE tenant_id IN ({tenants:Array(String)}); + +SELECT '-- a view in another database binds by the view name too'; +SELECT pv.host_id FROM pv(tenants = ['t1']); +SELECT {CLICKHOUSE_DATABASE_1:Identifier}.pv.host_id FROM pv(tenants = ['t1']); + +SELECT '-- control: qualifying with a database that does not hold the view does not bind'; +SELECT {CLICKHOUSE_DATABASE:Identifier}.pv.host_id FROM pv(tenants = ['t1']); -- { serverError UNKNOWN_IDENTIFIER } + +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.reference b/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.reference new file mode 100644 index 0000000000000..359b94fc535ae --- /dev/null +++ b/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.reference @@ -0,0 +1,33 @@ +-- columns qualified by the view name (issue case 1) +x +x +x +-- qualifier inside IN (issue case 2) +dev +dev +-- unaliased JOIN operand (issue case 3) +4 4 4 +3 3 3 +2 2 2 +1 1 1 +0 0 0 +5 +5 +5 +-- qualified matcher and matcher-expanded column names +0 0 +c1 c2 k1 pv1.c1 +t1.c1 c2 k1 m1 t2.c1 c3 +t1.c1 t1.c2 pv3.k1 pv3.m1 t2.c1 t2.c3 +0 0 +1 +-- arguments still survive per call (issue 112148 must stay fixed) +1 +1 +-- controls: regular table functions gain nothing +UNKNOWN_IDENTIFIER +ALIAS_REQUIRED +UNKNOWN_IDENTIFIER +ALIAS_REQUIRED +c1 c2 c1 +t1.c1 t1.c2 t2.c1 t2.c3 c1 diff --git a/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.sh b/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.sh new file mode 100755 index 0000000000000..e1f90de28154b --- /dev/null +++ b/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# A parameterized view call is resolved as a `TableFunctionNode`, but unlike a real table function it has a +# name of its own: its columns bind by that name and it needs no alias in a JOIN. +# https://github.com/ClickHouse/ClickHouse/issues/119837 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The fix is in the analyzer and the test pins it: with enable_analyzer = 0 a dotted `db.view(...)` call fails with UNKNOWN_FUNCTION. +CLIENT="$CLICKHOUSE_CLIENT --enable_analyzer=1 --joined_subquery_requires_alias=1" +DB=$CLICKHOUSE_DATABASE + +$CLIENT --query " +CREATE TABLE t1 (c1 String, c2 Float64) ENGINE = ReplacingMergeTree ORDER BY c1; +INSERT INTO t1 SELECT toString(number), toFloat64(number) FROM numbers(5); +CREATE TABLE t2 (c1 String, c3 UInt8) ENGINE = MergeTree ORDER BY c1; +INSERT INTO t2 SELECT toString(number), number FROM numbers(3); +CREATE VIEW pv1 AS SELECT {p1:String} AS k1, 'x' AS c1; +CREATE VIEW pv2 AS SELECT arrayJoin({p1:Array(Int64)}) AS k1, 'dev' AS c1; +CREATE VIEW pv3 AS SELECT c1 AS k1, sum(c2) AS m1 FROM t1 WHERE length({p1:Array(String)}) = 0 OR c1 IN ({p1:Array(String)}) GROUP BY c1; +" + +echo '-- columns qualified by the view name (issue case 1)' +$CLIENT --query "SELECT pv1.c1 FROM \`$DB.pv1\`(p1 = 'a') LIMIT 1" +$CLIENT --query "SELECT pv1.c1 FROM pv1(p1 = 'a')" +$CLIENT --query "SELECT $DB.pv1.c1 FROM $DB.pv1(p1 = 'a')" + +echo '-- qualifier inside IN (issue case 2)' +$CLIENT --query "SELECT c1 FROM \`$DB.pv2\`(p1 = [157]) WHERE pv2.k1 IN (157) LIMIT 100" +$CLIENT --query "SELECT c1 FROM pv2(p1 = [157]) WHERE $DB.pv2.k1 IN (157)" + +echo '-- unaliased JOIN operand (issue case 3)' +$CLIENT --query "SELECT $DB.t1.c1, $DB.pv3.k1, $DB.pv3.m1 FROM $DB.t1 FINAL INNER JOIN \`$DB.pv3\`(p1 = []) ON $DB.t1.c1 = $DB.pv3.k1 ORDER BY m1 DESC LIMIT 200" +$CLIENT --query "SELECT count() FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1" +$CLIENT --query "SELECT count() FROM t1, pv3(p1 = ['1'])" +$CLIENT --query "SELECT count() FROM pv3(p1 = []) INNER JOIN t1 ON t1.c1 = pv3.k1" + +echo '-- qualified matcher and matcher-expanded column names' +$CLIENT --query "SELECT pv3.* FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1 ORDER BY k1 LIMIT 1" +# `c1` of the view clashes with `t1.c1`, so the view's copy must be qualified with the view name +$CLIENT --query "SELECT * FROM t1 INNER JOIN pv1(p1 = 'a') ON t1.c1 = pv1.c1 LIMIT 0 SETTINGS analyzer_compatibility_multiple_joins_qualify_column_names = 0 FORMAT TSVWithNames" +$CLIENT --query "SELECT * FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1 INNER JOIN t2 ON t1.c1 = t2.c1 LIMIT 0 SETTINGS analyzer_compatibility_multiple_joins_qualify_column_names = 0 FORMAT TSVWithNames" +$CLIENT --query "SELECT * FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1 INNER JOIN t2 ON t1.c1 = t2.c1 LIMIT 0 SETTINGS analyzer_compatibility_multiple_joins_qualify_column_names = 1 FORMAT TSVWithNames" +$CLIENT --query "SELECT $DB.pv3.* FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1 ORDER BY k1 LIMIT 1" +$CLIENT --query "SELECT count() FROM pv3(p1 = []) INNER JOIN pv3(p1 = ['1']) USING (k1)" + +echo '-- arguments still survive per call (issue 112148 must stay fixed)' +$CLIENT --query "SELECT k1 FROM pv3(p1 = ['1'])" +$CLIENT --query "SELECT count() FROM pv3(p1 = []) AS a INNER JOIN pv3(p1 = ['1']) AS b USING (k1)" + +echo '-- controls: regular table functions gain nothing' +$CLIENT --query "SELECT numbers.number FROM numbers(3)" 2>&1 | grep -o -m1 'UNKNOWN_IDENTIFIER' +$CLIENT --query "SELECT count() FROM t1 INNER JOIN numbers(3) ON 1 = 1" 2>&1 | grep -o -m1 'ALIAS_REQUIRED' +$CLIENT --query "SELECT view.dummy FROM view(SELECT 1 AS dummy)" 2>&1 | grep -o -m1 'UNKNOWN_IDENTIFIER' +$CLIENT --query "SELECT count() FROM t1 INNER JOIN view(SELECT 1 AS dummy) ON 1 = 1" 2>&1 | grep -o -m1 'ALIAS_REQUIRED' +# a clashing column of `view(...)` has no name to be qualified with, in both qualification modes +$CLIENT --query "SELECT * FROM t1, view(SELECT '0' AS c1) LIMIT 0 SETTINGS joined_subquery_requires_alias = 0, analyzer_compatibility_multiple_joins_qualify_column_names = 0 FORMAT TSVWithNames" +$CLIENT --query "SELECT * FROM t1 INNER JOIN t2 ON t1.c1 = t2.c1, view(SELECT '0' AS c1) LIMIT 0 SETTINGS joined_subquery_requires_alias = 0, analyzer_compatibility_multiple_joins_qualify_column_names = 1 FORMAT TSVWithNames" From 1d57bdcdf64ef0b4c91ddd0559cde1ec4a447d07 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 20 Sep 2026 08:18:56 +0000 Subject: [PATCH 042/296] Backport #117573 to 26.8: Fix wrong aggregates from limit pushdown when ORDER BY is a strict prefix of GROUP BY --- src/Processors/QueryPlan/AggregatingStep.cpp | 2 + src/Processors/QueryPlan/AggregatingStep.h | 9 +- .../optimizeLimitForAggregationInOrder.cpp | 7 +- .../AggregatingInOrderTransform.cpp | 39 +++- .../Transforms/AggregatingInOrderTransform.h | 12 + ...n_in_order_limit_order_by_prefix.reference | 53 +++++ ...egation_in_order_limit_order_by_prefix.sql | 208 ++++++++++++++++++ 7 files changed, 321 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.reference create mode 100644 tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.sql diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 11d17539b5205..272d0f4f0e352 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -690,6 +690,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B many_data, counter++, limit_hint, + limit_hint_prefix_columns, nullptr // `dataflow_cache_updater` will be passed to `MergingAggregatedBucketTransform` below ); }); @@ -741,6 +742,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B sort_description_for_merging, group_by_sort_description, max_block_size, aggregation_in_order_max_block_bytes, limit_hint, + limit_hint_prefix_columns, dataflow_cache_updater); }); diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 3140a1d309956..57ebee67f2e5b 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -94,7 +94,13 @@ class AggregatingStep : public ITransformingStep void applyTopKOptimization(Aggregator::Params::TopKParams top_k); bool memoryBoundMergingWillBeUsed() const; void skipMerging() { skip_merging = true; } - void setLimitHint(size_t limit) { limit_hint = limit; } + /// `prefix_columns` is the number of leading columns of the group-by sort description + /// the query is ordered by; the in-order streams may stop only at a boundary of them. + void setLimitHint(size_t limit, size_t prefix_columns) + { + limit_hint = limit; + limit_hint_prefix_columns = prefix_columns; + } size_t getLimitHint() const { return limit_hint; } const SortDescription & getGroupBySortDescription() const { return group_by_sort_description; } @@ -186,6 +192,7 @@ class AggregatingStep : public ITransformingStep bool enable_sharding_aggregator; size_t limit_hint = 0; + size_t limit_hint_prefix_columns = 0; Processors aggregating_in_order; Processors aggregating_sorted; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeLimitForAggregationInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeLimitForAggregationInOrder.cpp index ffec0be141a2c..9481e1cee3da2 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeLimitForAggregationInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeLimitForAggregationInOrder.cpp @@ -122,6 +122,11 @@ void optimizeLimitForAggregationInOrder(QueryPlan::Node & root) || aggregating_step->isGroupingSets()) break; + /// The sort description may be a strict prefix of the group-by sort description. + /// Each in-order stream then stops only at a boundary of that prefix (see + /// `AggregatingInOrderTransform`), so the groups it drops sort strictly after + /// at least `limit` complete groups and the final sort can never pick a group + /// with a partial aggregate value (issue #116849). const auto & sort_desc = sorting_step->getSortDescription(); const auto & agg_sort_desc = aggregating_step->getGroupBySortDescription(); if (sort_desc.empty() || !agg_sort_desc.hasPrefix(sort_desc)) @@ -130,7 +135,7 @@ void optimizeLimitForAggregationInOrder(QueryPlan::Node & root) /// Use the smallest limit if multiple LimitSteps point to the same AggregatingStep. size_t current_hint = aggregating_step->getLimitHint(); if (!current_hint || limit < current_hint) - aggregating_step->setLimitHint(limit); + aggregating_step->setLimitHint(limit, sort_desc.size()); break; } diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index b4d0d8a5fbb77..f4e048fae60e0 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -27,12 +27,14 @@ AggregatingInOrderTransform::AggregatingInOrderTransform( const SortDescription & group_by_description_, size_t max_block_size_, size_t max_block_bytes_, size_t limit_hint_, + size_t limit_prefix_columns_, RuntimeDataflowStatisticsCacheUpdaterPtr dataflow_cache_updater_) : AggregatingInOrderTransform(std::move(header), std::move(params_), sort_description_for_merging, group_by_description_, max_block_size_, max_block_bytes_, std::make_unique(1), 0, limit_hint_, + limit_prefix_columns_, std::move(dataflow_cache_updater_)) { } @@ -44,6 +46,7 @@ AggregatingInOrderTransform::AggregatingInOrderTransform( size_t max_block_size_, size_t max_block_bytes_, ManyAggregatedDataPtr many_data_, size_t current_variant, size_t limit_hint_, + size_t limit_prefix_columns_, RuntimeDataflowStatisticsCacheUpdaterPtr dataflow_cache_updater_) : IProcessor({std::move(header)}, {params_->getCustomHeader(false)}) , max_block_size(max_block_size_) @@ -72,6 +75,22 @@ AggregatingInOrderTransform::AggregatingInOrderTransform( /// group_by_description may contains duplicates, so we use keys_size from Aggregator::params key_columns_raw.resize(params->params.keys_size); } + + /// The `ORDER BY` may extend past the columns the input is sorted by; a boundary of the + /// sorted columns alone is then a boundary of the `ORDER BY` prefix as well. + limit_prefix_columns = std::min(limit_prefix_columns_, group_by_description.size()); +} + +bool AggregatingInOrderTransform::isLimitPrefixBoundary(const Columns & key_columns, size_t row) const +{ + for (size_t i = 0; i < limit_prefix_columns; ++i) + { + const auto & elem = group_by_description[i]; + size_t ind = elem.column_number; + if (res_key_columns[ind]->compareAt(cur_block_size - 1, row, *key_columns[ind], elem.base.nulls_direction) != 0) + return true; + } + return false; } AggregatingInOrderTransform::~AggregatingInOrderTransform() = default; @@ -214,9 +233,16 @@ void AggregatingInOrderTransform::consume(Chunk chunk) if (!group_by_key) params->aggregator.addSingleKeyToAggregateColumns(variants, res_aggregate_columns); + /// Enough groups have been emitted and the new key starts a new value of the + /// `ORDER BY` prefix, so the rest of the input cannot be needed (see `limit_prefix_columns`). + /// With `group_by_key`, `cur_block_size` counts runs of the sorted key columns rather than + /// groups; the groups accumulated for the current block are the entries of the hash table. + size_t cur_block_groups = group_by_key ? variants.size() : cur_block_size; + if (limit_hint && cur_block_groups + res_rows >= limit_hint && isLimitPrefixBoundary(key_columns, key_end)) + limit_reached = true; + /// If max_block_size or limit_hint is reached we have to stop consuming and generate the block. Save the extra rows into new chunk. - if (cur_block_size >= max_block_size || cur_block_bytes + current_memory_usage >= max_block_bytes - || (limit_hint && cur_block_size + res_rows >= limit_hint)) + if (cur_block_size >= max_block_size || cur_block_bytes + current_memory_usage >= max_block_bytes || limit_reached) { if (group_by_key) group_by_chunk @@ -225,9 +251,8 @@ void AggregatingInOrderTransform::consume(Chunk chunk) /// When limit is reached, don't save leftover rows — we're done. /// Set block_end_reached + need_generate to trigger generate(), - /// which will produce the output and then set is_consume_finished - /// via the limit check after res_rows is updated. - if (limit_hint && cur_block_size + res_rows >= limit_hint) + /// which will produce the output and then set is_consume_finished. + if (limit_reached) { block_end_reached = true; need_generate = true; @@ -416,8 +441,8 @@ void AggregatingInOrderTransform::generate() res_rows += to_push_chunk.getNumRows(); need_generate = false; - /// If we have emitted enough groups, stop consuming more input. - if (limit_hint && res_rows >= limit_hint) + /// If we have emitted enough groups up to a boundary of the `ORDER BY` prefix, stop consuming more input. + if (limit_reached) is_consume_finished = true; } diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.h b/src/Processors/Transforms/AggregatingInOrderTransform.h index 28b312c607d49..e308f30187a0b 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.h +++ b/src/Processors/Transforms/AggregatingInOrderTransform.h @@ -35,6 +35,7 @@ class AggregatingInOrderTransform final : public IProcessor ManyAggregatedDataPtr many_data, size_t current_variant, size_t limit_hint_, + size_t limit_prefix_columns_, RuntimeDataflowStatisticsCacheUpdaterPtr dataflow_cache_updater_); AggregatingInOrderTransform( @@ -45,6 +46,7 @@ class AggregatingInOrderTransform final : public IProcessor size_t max_block_size_, size_t max_block_bytes_, size_t limit_hint_, + size_t limit_prefix_columns_, RuntimeDataflowStatisticsCacheUpdaterPtr dataflow_cache_updater_); ~AggregatingInOrderTransform() override; @@ -61,6 +63,9 @@ class AggregatingInOrderTransform final : public IProcessor private: void generate(); void finalizeCurrentChunk(Chunk chunk, size_t key_end); + /// Whether the key at `row` of `key_columns` differs from the current key in the first + /// `limit_prefix_columns` columns, i.e. whether a boundary of the `ORDER BY` prefix is reached. + bool isLimitPrefixBoundary(const Columns & key_columns, size_t row) const; size_t max_block_size; size_t max_block_bytes; @@ -89,7 +94,14 @@ class AggregatingInOrderTransform final : public IProcessor UInt64 src_bytes = 0; UInt64 res_rows = 0; + /// The stream may stop early once it has emitted `limit_hint` groups, but only at a + /// boundary of the first `limit_prefix_columns` columns of `group_by_description`, which + /// are the columns the query is ordered by. Otherwise a group tied on those columns with + /// the emitted ones could be cut off here while another stream emits its partial state, + /// and the final sort could pick that partial group. size_t limit_hint = 0; + size_t limit_prefix_columns = 0; + bool limit_reached = false; bool need_generate = false; bool block_end_reached = false; diff --git a/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.reference b/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.reference new file mode 100644 index 0000000000000..cebd374be5e10 --- /dev/null +++ b/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.reference @@ -0,0 +1,53 @@ +1 +1 +1 +1 +1 +1 +1 1 10 +1 2 10 +1 3 10 +1 4 11 +1 5 11 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +PUSHDOWN_FIRES +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +PUSHDOWN_FIRES diff --git a/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.sql b/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.sql new file mode 100644 index 0000000000000..17ea64ae2024d --- /dev/null +++ b/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.sql @@ -0,0 +1,208 @@ +-- Regression test for issue #116849: with `optimize_aggregation_in_order_limit` +-- enabled, a query whose `ORDER BY` is a strict prefix of the `GROUP BY` key +-- returned incomplete aggregate values when groups tie on that prefix and a +-- group's rows span more than one part. Each in-order stream stopped as soon as +-- it had emitted `LIMIT` groups, at a boundary of the full group key; now it +-- stops only at a boundary of the `ORDER BY` prefix, so the groups it drops +-- always sort after at least `LIMIT` complete groups. + +DROP TABLE IF EXISTS t_agg_in_order_limit_prefix; + +CREATE TABLE t_agg_in_order_limit_prefix (a UInt32, b UInt32, x UInt32) +ENGINE = MergeTree ORDER BY (a, b); + +SYSTEM STOP MERGES t_agg_in_order_limit_prefix; + +-- Two parts; groups with b in 4..20 span both parts. +INSERT INTO t_agg_in_order_limit_prefix SELECT 1, number, 10 FROM numbers(1, 20); +INSERT INTO t_agg_in_order_limit_prefix SELECT 1, number, 1 FROM numbers(4, 17); + +-- Ground truth: sum(x) is 10 for groups with b in 1..3 and 11 for groups with b in 4..20. +-- All groups tie on `a`, so any three groups are a legal answer; assert that every +-- returned group carries its complete aggregate value. The check is done in the +-- projection on purpose: wrapping the query into a subquery changes the plan +-- (the aggregation is no longer executed in order), which hides the bug. The block +-- settings are pinned because tiny blocks also hide it, and `max_threads` because with a +-- single thread and `read_in_order_two_level_merge_threshold` at most the number of parts +-- the parts are merged into one stream before the aggregation, where the bug cannot occur. +SELECT sum(x) = if(b <= 3, 10, 11) +FROM t_agg_in_order_limit_prefix +GROUP BY a, b +ORDER BY a +LIMIT 3 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +-- Same with OFFSET. +SELECT sum(x) = if(b <= 3, 10, 11) +FROM t_agg_in_order_limit_prefix +GROUP BY a, b +ORDER BY a +LIMIT 3 OFFSET 2 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +-- The full-key `ORDER BY` still admits the push-down and stays correct. +SELECT a, b, sum(x) +FROM t_agg_in_order_limit_prefix +GROUP BY a, b +ORDER BY a, b +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1; + +DROP TABLE t_agg_in_order_limit_prefix; + +-- With enough distinct values of the prefix the push-down still stops the streams +-- early, and the groups it returns are complete even though every group spans two parts. + +DROP TABLE IF EXISTS t_agg_in_order_limit_prefix_reads; + +CREATE TABLE t_agg_in_order_limit_prefix_reads (a UInt32, b UInt32, x UInt32) +ENGINE = MergeTree ORDER BY (a, b) +SETTINGS index_granularity = 8; + +SYSTEM STOP MERGES t_agg_in_order_limit_prefix_reads; + +-- Two parts, 100 values of `a` with 10 values of `b` each; sum(x) is 3 for every group. +INSERT INTO t_agg_in_order_limit_prefix_reads SELECT intDiv(number, 10), number % 10, 1 FROM numbers(1000); +INSERT INTO t_agg_in_order_limit_prefix_reads SELECT intDiv(number, 10), number % 10, 2 FROM numbers(1000); + +-- Every returned group must be complete. +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_prefix_reads +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_prefix_reads +GROUP BY a, b +ORDER BY a +LIMIT 5 OFFSET 7 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +-- The push-down is observed through `read_rows`. The small-block settings expose the +-- effect on a 2000-row table; `enable_parallel_replicas = 0` is required because +-- `read_rows` is accounted per reading node, and `read_in_order_two_level_merge_threshold` +-- keeps the two parts in separate streams instead of merging them before the aggregation. +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_prefix_reads +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 1, max_block_size = 16, read_in_order_two_level_merge_threshold = 100, + merge_tree_min_rows_for_concurrent_read = 0, merge_tree_min_bytes_for_concurrent_read = 0, + merge_tree_min_rows_for_seek = 0, + enable_parallel_replicas = 0, + log_comment = '05055_prefix_pushdown_on'; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_prefix_reads +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 0, + max_threads = 1, max_block_size = 16, read_in_order_two_level_merge_threshold = 100, + merge_tree_min_rows_for_concurrent_read = 0, merge_tree_min_bytes_for_concurrent_read = 0, + merge_tree_min_rows_for_seek = 0, + enable_parallel_replicas = 0, + log_comment = '05055_prefix_pushdown_off'; + +SYSTEM FLUSH LOGS query_log; + +SELECT if(on_reads < off_reads, 'PUSHDOWN_FIRES', format('FAIL: on={} off={}', on_reads, off_reads)) +FROM +( + SELECT + anyIf(read_rows, log_comment = '05055_prefix_pushdown_on') AS on_reads, + anyIf(read_rows, log_comment = '05055_prefix_pushdown_off') AS off_reads + FROM system.query_log + WHERE current_database = currentDatabase() + AND log_comment IN ('05055_prefix_pushdown_on', '05055_prefix_pushdown_off') + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - 600 +); + +DROP TABLE t_agg_in_order_limit_prefix_reads; + +-- The table is sorted only by `a`, so the in-order aggregation of `GROUP BY a, b` keeps the +-- groups of one `a` run in a hash table (the `group_by_key` path of `AggregatingInOrderTransform`). +-- The stream must count the groups it accumulated, not the `a` runs: a single run already +-- holds 10 complete groups, so a `LIMIT 5` stream stops right after the first run. + +DROP TABLE IF EXISTS t_agg_in_order_limit_partial_key; + +CREATE TABLE t_agg_in_order_limit_partial_key (a UInt32, b UInt32, x UInt32) +ENGINE = MergeTree ORDER BY a +SETTINGS index_granularity = 8; + +SYSTEM STOP MERGES t_agg_in_order_limit_partial_key; + +-- Two parts, 100 values of `a` with 10 values of `b` each; sum(x) is 3 for every group. +INSERT INTO t_agg_in_order_limit_partial_key SELECT intDiv(number, 10), number % 10, 1 FROM numbers(1000); +INSERT INTO t_agg_in_order_limit_partial_key SELECT intDiv(number, 10), number % 10, 2 FROM numbers(1000); + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_partial_key +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_partial_key +GROUP BY a, b +ORDER BY a, b +LIMIT 5 OFFSET 7 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_partial_key +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 1, max_block_size = 16, read_in_order_two_level_merge_threshold = 100, + merge_tree_min_rows_for_concurrent_read = 0, merge_tree_min_bytes_for_concurrent_read = 0, + merge_tree_min_rows_for_seek = 0, + enable_parallel_replicas = 0, + log_comment = '05055_partial_key_pushdown_on'; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_partial_key +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 0, + max_threads = 1, max_block_size = 16, read_in_order_two_level_merge_threshold = 100, + merge_tree_min_rows_for_concurrent_read = 0, merge_tree_min_bytes_for_concurrent_read = 0, + merge_tree_min_rows_for_seek = 0, + enable_parallel_replicas = 0, + log_comment = '05055_partial_key_pushdown_off'; + +SYSTEM FLUSH LOGS query_log; + +-- Stopping after the first `a` run reads a small fraction of the table; counting `a` runs +-- instead of groups would need five runs, i.e. several times more rows. +SELECT if(on_reads * 20 <= off_reads, 'PUSHDOWN_FIRES', format('FAIL: on={} off={}', on_reads, off_reads)) +FROM +( + SELECT + anyIf(read_rows, log_comment = '05055_partial_key_pushdown_on') AS on_reads, + anyIf(read_rows, log_comment = '05055_partial_key_pushdown_off') AS off_reads + FROM system.query_log + WHERE current_database = currentDatabase() + AND log_comment IN ('05055_partial_key_pushdown_on', '05055_partial_key_pushdown_off') + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - 600 +); + +DROP TABLE t_agg_in_order_limit_partial_key; From a6d129accb34eeacc92ace25136adf6876e779e5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 20 Sep 2026 12:26:32 +0000 Subject: [PATCH 043/296] Backport #116724 to 26.8: Verify the server certificate hostname on outbound TLS connections --- base/poco/NetSSL_OpenSSL/src/SSLManager.cpp | 4 +- .../_server_settings_outside_source.mdx | 2 +- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 27 +++++++- src/Core/ServerSettings.cpp | 2 +- .../configs/ssl_config_ca_signed.xml | 15 ++++ .../test_accept_invalid_certificate/test.py | 69 +++++++++++++++++++ .../test_acme_tls/configs/config.xml | 3 + .../test_acme_tls/configs/config_multi.xml | 3 + .../configs/config.d/ssl_conf.xml | 3 + .../configs/generate_certs.sh | 14 ++++ .../configs/server.crt | 34 ++++----- .../configs/server.key | 52 +++++++------- .../configs/ssl_conf_verify.xml | 23 +++++++ .../test_keeper_secure_client/test.py | 16 +++++ .../test_ssl_cert_authentication/test.py | 2 + 15 files changed, 222 insertions(+), 47 deletions(-) create mode 100644 tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml create mode 100755 tests/integration/test_keeper_secure_client/configs/generate_certs.sh create mode 100644 tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml diff --git a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp index 94b66cdcf5f1e..80d85855fe3c9 100644 --- a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp @@ -340,7 +340,9 @@ void SSLManager::initDefaultContext(bool server) { _ptrDefaultClientContext->enableSessionCache(cacheSessions); } - bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, false); + /// Only an outbound connection has a requested host name to check: on an accepted socket this + /// would run against the client's address, which a client certificate does not name. + bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, !server); if (server) _ptrDefaultServerContext->enableExtendedCertificateVerification(extendedVerification); else diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 88f55cc2d0d94..03357b3470b04 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1168,7 +1168,7 @@ Keys for server/client settings: | `certificateFile` | Path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contains the certificate. | | | `cipherList` | Supported OpenSSL encryptions. | `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH` | | `disableProtocols` | Protocols that are not allowed to be used. | | -| `extendedVerification` | If enabled, verify that the certificate CN or SAN matches the peer hostname. | `false` | +| `extendedVerification` | If enabled, verify that the certificate CN or SAN matches the peer hostname. | `true` for `openSSL.client`, `false` for `openSSL.server` | | `fips` | Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. | `false` | | `invalidCertificateHandler` | Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` RejectCertificateHandler ` . | `RejectCertificateHandler` | | `loadDefaultCAFile` | Wether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `/etc/ssl/cert.pem` (resp. the directory `/etc/ssl/certs`) or in file (resp. directory) specified by the environment variable `SSL_CERT_FILE` (resp. `SSL_CERT_DIR`). | `true` | diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 90d6c1bed91a6..0e61bf81933b2 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -372,6 +372,25 @@ void triggerWatchCallback( } } +#if USE_SSL +/// The host part of ":", split lexically at the same character Poco::Net::SocketAddress +/// splits it at, so that a port spelled as a service name still resolves. The result is matched +/// against a certificate, so an IPv6 literal loses its brackets; a shape naming no host is empty. +std::string peerHostName(const std::string & host_and_port) +{ + if (host_and_port.starts_with('/')) + return {}; + + if (host_and_port.starts_with('[')) + { + size_t closing_bracket = host_and_port.find(']'); + return closing_bracket == std::string::npos ? std::string{} : host_and_port.substr(1, closing_bracket - 1); + } + + return host_and_port.substr(0, host_and_port.find(':')); +} +#endif + } template @@ -617,7 +636,13 @@ void ZooKeeper::connect( if (node.secure) { #if USE_SSL - socket = Poco::Net::SecureStreamSocket(); + auto secure_socket = Poco::Net::SecureStreamSocket(); + /// The certificate names the configured host while the socket connects to the + /// address it resolved to, so the name has to be carried explicitly. This is + /// also what puts the host into the SNI extension. + if (const auto peer_host_name = peerHostName(node.host); !peer_host_name.empty()) + secure_socket.setPeerHostName(peer_host_name); + socket = secure_socket; #else throw Poco::Exception( "Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support."); diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index ac0aa4a27ee57..0f94a06ba3bc3 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -1912,7 +1912,7 @@ Configured as `named_collections_storage.type` (`` (resp. the directory ``) or in file (resp. directory) specified by the environment variable `` (resp. ``).)", 0, "openSSL.client.loadDefaultCAFile") \ DECLARE(String, openssl_client_chipher_list, "ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH", R"(Supported OpenSSL encryptions.)", 0, "openSSL.client.cipherList") \ DECLARE(Bool, openssl_client_cache_sessions, false, R"(Enables or disables caching sessions. Must be used in combination with ``. Acceptable values: ``, ``.)", 0, "openSSL.client.cacheSessions") \ - DECLARE(Bool, openssl_client_extended_verification, false, R"(If enabled, verify that the certificate CN or SAN matches the peer hostname.)", 0, "openSSL.client.extendedVerification") \ + DECLARE(Bool, openssl_client_extended_verification, true, R"(If enabled, verify that the certificate CN or SAN matches the peer hostname.)", 0, "openSSL.client.extendedVerification") \ DECLARE(Bool, openssl_client_required_tls_v1, false, R"(Require a TLSv1 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1") \ DECLARE(Bool, openssl_client_required_tls_v1_1, false, R"(Require a TLSv1.1 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1_1") \ DECLARE(Bool, openssl_client_required_tls_v1_2, false, R"(Require a TLSv1.2 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1_2") \ diff --git a/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml b/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml new file mode 100644 index 0000000000000..b1dccbb15c66e --- /dev/null +++ b/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml @@ -0,0 +1,15 @@ + + + 9440 + + + + /etc/clickhouse-server/config.d/client-cert.pem + /etc/clickhouse-server/config.d/client-key.pem + /etc/clickhouse-server/config.d/ca-cert.pem + none + + + diff --git a/tests/integration/test_accept_invalid_certificate/test.py b/tests/integration/test_accept_invalid_certificate/test.py index afea8011c4b99..b66f311b92355 100644 --- a/tests/integration/test_accept_invalid_certificate/test.py +++ b/tests/integration/test_accept_invalid_certificate/test.py @@ -9,6 +9,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) MAX_RETRY = 5 +CA_CERT = f"{SCRIPT_DIR}/certs/ca-cert.pem" cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( @@ -35,6 +36,18 @@ ) +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs/ssl_config_ca_signed.xml", + "certs/client-key.pem", + "certs/client-cert.pem", + "certs/ca-cert.pem", + ], + with_zookeeper=False, +) + + @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: @@ -61,6 +74,36 @@ def started_cluster(): """ +# node2 presents a certificate this CA signs, so the chain is valid and only the name can fail. +config_ca_signed = """ + + + {caConfig} + false + + +""" + +config_ca_signed_sni_override = """ + client + + + {caConfig} + false + + +""" + +config_ca_signed_no_extended_verification = """ + + + {caConfig} + false + false + + +""" + def execute_query_native(node, query, config): fd, config_path = tempfile.mkstemp( @@ -132,3 +175,29 @@ def test_strict_connection_reject(): config_connection_accept.format(ip_address=f"{instance.ip_address}"), ) assert "certificate verify failed" in str(err.value) + + +def test_hostname_mismatch_rejected_by_default(): + with pytest.raises(Exception) as err: + execute_query_native(node2, "SELECT 1", config_ca_signed.format(caConfig=CA_CERT)) + assert "Unacceptable certificate" in str(err.value) + + +def test_hostname_match_accepted(): + assert ( + execute_query_native( + node2, "SELECT 1", config_ca_signed_sni_override.format(caConfig=CA_CERT) + ) + == "1\n" + ) + + +def test_extended_verification_disabled(): + assert ( + execute_query_native( + node2, + "SELECT 1", + config_ca_signed_no_extended_verification.format(caConfig=CA_CERT), + ) + == "1\n" + ) diff --git a/tests/integration/test_acme_tls/configs/config.xml b/tests/integration/test_acme_tls/configs/config.xml index 4594c72e885e3..0977ed12d7114 100644 --- a/tests/integration/test_acme_tls/configs/config.xml +++ b/tests/integration/test_acme_tls/configs/config.xml @@ -14,6 +14,9 @@ + + false AcceptCertificateHandler diff --git a/tests/integration/test_acme_tls/configs/config_multi.xml b/tests/integration/test_acme_tls/configs/config_multi.xml index 56c9d415886c4..9cda6eb0ba878 100644 --- a/tests/integration/test_acme_tls/configs/config_multi.xml +++ b/tests/integration/test_acme_tls/configs/config_multi.xml @@ -14,6 +14,9 @@ + + false AcceptCertificateHandler diff --git a/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml b/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml index c11c5736d96f8..e51749f354385 100644 --- a/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml +++ b/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml @@ -7,6 +7,9 @@ /etc/clickhouse-server/config.d/server.key + + false AcceptCertificateHandler diff --git a/tests/integration/test_keeper_secure_client/configs/generate_certs.sh b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh new file mode 100755 index 0000000000000..e398289b8cc9b --- /dev/null +++ b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Self-signed certificate that Keeper presents on its secure port. It is also its own trust +# anchor (CA:TRUE), so a client can point caConfig at it directly. +# +# The subjectAltName has to list every name a client may connect to: `node1` is the cluster +# member name used in use_secure_keeper.xml, and `localhost` is kept because the raft +# configuration and the existing arms use it. Once a dNSName SAN is present OpenSSL ignores +# the common name, so localhost must be listed explicitly rather than left to CN. +openssl req -newkey rsa:2048 -x509 -days 36500 -nodes -batch \ + -keyout server.key -out server.crt \ + -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,DNS:node1" \ + -addext "basicConstraints=critical,CA:TRUE" diff --git a/tests/integration/test_keeper_secure_client/configs/server.crt b/tests/integration/test_keeper_secure_client/configs/server.crt index 7ade2d9627338..d9ad8941d6158 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.crt +++ b/tests/integration/test_keeper_secure_client/configs/server.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow -FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI -4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T -4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU -7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj -sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg -pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC -kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j -4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr -85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C -L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD -+UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L -P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp -0g== +MIIDKDCCAhCgAwIBAgIUBTZCvmMS+9yaX8IRKGkGqbcMBOIwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI2MDgyNzExMzMzOFoYDzIxMjYw +ODAzMTEzMzM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCyOozYThV2lJBPteVCm7YMVJ/w4hralGidnbkpGZxQ +OnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6EBM0RYv7SQtcoUG2YErKHuASHSSc +DdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxhUFROGgguoIdVOKUtDwib/mq27048 ++uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUGle4rMlAR8oTDBmHlrowoEOjN/mwd +WnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUItvXreLuvdt1wQoCC1bH6h6K3ymsu8 +1xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCFXtunioRbAgMBAAGjcDBuMB0GA1Ud +DgQWBBQP0fRsGOqD4ZxQ1oTe7v9HIzc3RTAfBgNVHSMEGDAWgBQP0fRsGOqD4ZxQ +1oTe7v9HIzc3RTAbBgNVHREEFDASgglsb2NhbGhvc3SCBW5vZGUxMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGw73J/dlsYphCnXZzVfOWiwFtRX +ij9M3/GjtCfmfh2MriYcNlCGqw+7GvzUUJ44rKjRxWyF+FfVM7DC4O0a6oimvtwE +yhb9mFshGDUpMX6WtpjhFGDtNjbgXrz6FXkt8U5SQayVoVLQjIT6XZSmXEccMJpM +U3lGAqPaeMVYVZqQPcvdO+t+Jfrjjznpa66biD87BvJ4CCWNynWi/ftpK76bk1fj +SbL0RgG1SUBAJikJirL2OW+ADgCyPxP86IgTKdOV92e+00spcfvSlms7UzrChXA7 +Kv0Z/9WUgL3tL87kiP0eG3XaZp83w+WS0OOcaU8uQ0AhofzWXMnQT0Bgq/8= -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_secure_client/configs/server.key b/tests/integration/test_keeper_secure_client/configs/server.key index f0fb61ac443fa..390ae5e9995f6 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.key +++ b/tests/integration/test_keeper_secure_client/configs/server.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn -DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L -OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus -TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 -JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F -KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 -ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP -TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f -7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN -MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 -212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu -plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr -/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu -Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa -CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g -1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 -bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq -cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez -on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K -GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 -nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU -2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh -vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa -dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr -vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 -KCBtmIrQLqzMpnBpLNuSY+Q= +MIIEugIBADANBgkqhkiG9w0BAQEFAASCBKQwggSgAgEAAoIBAQCyOozYThV2lJBP +teVCm7YMVJ/w4hralGidnbkpGZxQOnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6 +EBM0RYv7SQtcoUG2YErKHuASHSScDdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxh +UFROGgguoIdVOKUtDwib/mq27048+uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUG +le4rMlAR8oTDBmHlrowoEOjN/mwdWnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUIt +vXreLuvdt1wQoCC1bH6h6K3ymsu81xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCF +XtunioRbAgMBAAECgf9xetYy44TG+9Dm7+oJfxtqvncX6N8wTU5PDR3YDXZ5nrz+ +q2ZcE2+A2RdE52nDoI2wT0u3Vw1k4K+VSKbxoVZ/qQKS+BnfBXCZQQ9qeSPWOPWY +70CQhZOwjfp2oY8v/3a6DoYp74zPRCutynfjQq/zAEMq/+Yrymaw7E0GLVijehE7 +eLhAFL/1IAjzvr5ikaN2MgSGHwcZ7XNpTN0Lvpz1PH2LkpDmh7PWUKcvu7nV3cQc +x8BkEFFaqvlXq0NfgPNCZ9qL9/Y+VrJjv0Bta7KCO/6nbmJubUGVZbzv0qwOJBVB +/3iki0BoEG2YJzJLuKnJJ0Ugf6usNai1NY9PKtkCgYEA3V+QJzcL0fWrvw6DBP0m +WWv1afZGr2ONjHmz65IvegFpt9qmm2CnRVzFPXet1VJkxK+MNFiPs+h29s1d2G7/ +VdwU5+b2KdQkgG9QKWsRv9bkNUqK48LWvLYkUCNhYW7oxCH4SxBNbSdVDUpXnZJ/ +u6T71YRgY9yYRI6HnHrJr40CgYEAzhtZ7phK4inHokcXWLJBhYy6HwiENzNUlEm9 +dLRTeExzrdb/zA8WCPc+Vb1dSoNa0l/5TFfyoRpZ2RsfX9iGZIR2GiQhC8ftK2XH +ZynQBN+rGNBoarWhPfpyMU8g5pHCIxChOJaMTA5sb52oHE5D03oQXa0jUQDS7ie8 +IIUh9YcCgYB2I8zLkM3nFAO/J6B2Zh80M7A6B2t7qFZfNIf3XQGnS9++dZraav95 +veOpRRhpMtqCOOlK+kRej94FIl5F5h6wFI63oAOZGRUN0dnm8LP9PFJ3ujtbB50Q +AuhoOCl9FPJ10WcvsBiR+x6hxB30Ar3WR1r6fPXv9Ubxt1raAJFn9QKBgCCNg1vC +4/jqnmRMFCznIqAkRPVH1BIf4lj1eJ3pKVHTyWWIkqg5LcLGwiqqmQR+0KZvkCF1 +tpWpYs1+kisCON/BaCHi2PFSZ2w2TBNIMFnEtfJaYFOSUmBGoSiWldy04tPE+aHF +bW1JzeinHvkxh1bsbY9OHEjb93S0A+ui+2UbAoGADWmYVq4ksRSrHMa/9gwqGC5k +bWx791nrTf2UKlpsBA2b7kQ76DLMzgPGzCAGo5condd65y/4oP/Mr2+mEZkDu+Dh +W8jCsFN9RW2XP4ekgJj41hKAyKedP7JkbBb7pznvtqq1oLndIKN00jjrlfIbsKD0 +Reoz9LQnDowqs/fGhh8= -----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml new file mode 100644 index 0000000000000..55a744c632f5a --- /dev/null +++ b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml @@ -0,0 +1,23 @@ + + + + /etc/clickhouse-server/config.d/server.crt + /etc/clickhouse-server/config.d/server.key + none + true + sslv2,sslv3 + + + + /etc/clickhouse-server/config.d/server.crt + false + relaxed + true + sslv2,sslv3 + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_keeper_secure_client/test.py b/tests/integration/test_keeper_secure_client/test.py index 1f00e68810287..fe8f65aa7bde8 100644 --- a/tests/integration/test_keeper_secure_client/test.py +++ b/tests/integration/test_keeper_secure_client/test.py @@ -24,6 +24,16 @@ "configs/server.key", ], ) +# Same secure Keeper, but this client actually verifies the certificate it is presented. +node3 = cluster.add_instance( + "node3", + main_configs=[ + "configs/use_secure_keeper.xml", + "configs/ssl_conf_verify.xml", + "configs/server.crt", + "configs/server.key", + ], +) @pytest.fixture(scope="module") @@ -40,3 +50,9 @@ def started_cluster(): def test_connection(started_cluster): # just nothrow node2.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") + + +def test_connection_verifying_certificate(started_cluster): + # The socket connects to the address node1 resolves to, so this only succeeds if + # the certificate is matched against `node1` rather than against that address. + node3.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 4edb7865241bb..977bc82ea2766 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -44,6 +44,7 @@ def started_cluster(): config = """ + {sslHost} strict @@ -62,6 +63,7 @@ def execute_query_native(node, query, user, cert_name, password=None): certificateFile=f"{SCRIPT_DIR}/certs/{cert_name}-cert.pem", privateKeyFile=f"{SCRIPT_DIR}/certs/{cert_name}-key.pem", caConfig=f"{SCRIPT_DIR}/certs/ca-cert.pem", + sslHost=SSL_HOST, ) file = open(config_path, "w") From 6943194a27d972c238ee5f1a9c3ad92eadc62def Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 20 Sep 2026 15:01:54 +0000 Subject: [PATCH 044/296] Update autogenerated version to 26.8.9.10 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 09f28a267b846..0f528e8c5392b 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54521) +SET(VERSION_REVISION 54522) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 9) -SET(VERSION_GITHASH 0de36d42e8bbc78fdaf2bbf90a2853b5e9301bdc) -SET(VERSION_DESCRIBE v26.8.9.1-lts) -SET(VERSION_STRING 26.8.9.1) +SET(VERSION_PATCH 10) +SET(VERSION_GITHASH 661c91e962f04611bad10ac83a0aca1a2aa0eb65) +SET(VERSION_DESCRIBE v26.8.10.1-lts) +SET(VERSION_STRING 26.8.10.1) # end of autochange From 6f491a75d99a2f1587cc94f9703234342cc74634 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 20 Sep 2026 21:04:20 +0000 Subject: [PATCH 045/296] Backport #120844 to 26.8: Use plain LRU for the query-local text index caches --- .../MergeTree/MergeTreeIndexConditionText.cpp | 20 +++-- ...local_postings_cache_no_rebuilds.reference | 3 + ...index_local_postings_cache_no_rebuilds.sql | 74 +++++++++++++++++++ 3 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.reference create mode 100644 tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.sql diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index 8f341ba083f53..548148d12bd7d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -152,31 +152,35 @@ MergeTreeIndexConditionText::MergeTreeIndexConditionText( return; } + /// Plain LRU: the caches live for one query and readers sweep the segments in task order. + /// Not using SLRU, because it would only pin stale entries. + static constexpr auto cache_policy = "LRU"; + /// Local caches: ~10% of the query memory budget, capped at 200 MiB. + static constexpr size_t local_cache_size_cap = 200ULL * 1024 * 1024; + const auto & settings = context_->getSettingsRef(); - static constexpr auto cache_policy = "SLRU"; - /// Local caches: ~10% of the query memory budget, capped at 100 MiB; max_memory_usage == 0 (unlimited) uses the cap, not a 0-size cache. - static constexpr size_t local_cache_size_cap = 100ULL * 1024 * 1024; const size_t query_memory_limit = settings[Setting::max_memory_usage]; - const size_t local_cache_max_size - = query_memory_limit == 0 ? local_cache_size_cap : std::min(query_memory_limit / 10, local_cache_size_cap); + const size_t local_cache_max_size = query_memory_limit == 0 + ? local_cache_size_cap + : std::min(query_memory_limit / 10, local_cache_size_cap); /// If usage of global text index caches is disabled, create local /// one to share them between threads that read the same data parts. if (settings[Setting::use_text_index_tokens_cache]) tokens_cache = context_->getTextIndexTokensCache(); else - tokens_cache = std::make_shared(cache_policy, local_cache_max_size, 0, 1.0); + tokens_cache = std::make_shared(cache_policy, local_cache_max_size, 0, /*size_ratio=*/ 0.0); use_global_header_cache = settings[Setting::use_text_index_header_cache]; if (use_global_header_cache) header_cache = context_->getTextIndexHeaderCache(); else - header_cache = std::make_shared(cache_policy, local_cache_max_size, 0, 1.0); + header_cache = std::make_shared(cache_policy, local_cache_max_size, 0, /*size_ratio=*/ 0.0); if (settings[Setting::use_text_index_postings_cache]) postings_cache = context_->getTextIndexPostingsCache(); else - postings_cache = std::make_shared(cache_policy, local_cache_max_size, 0, 1.0); + postings_cache = std::make_shared(cache_policy, local_cache_max_size, 0, /*size_ratio=*/ 0.0); rpn = std::move(RPNBuilder( predicate, diff --git a/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.reference b/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.reference new file mode 100644 index 0000000000000..363198023a737 --- /dev/null +++ b/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.reference @@ -0,0 +1,3 @@ +parts 1 +16384 +4096 1 diff --git a/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.sql b/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.sql new file mode 100644 index 0000000000000..bca6fc376403e --- /dev/null +++ b/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.sql @@ -0,0 +1,74 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: parallel replicas split the part between replicas, each with its own query-local +-- cache, so the segments at the replica boundaries are built more than once and the count below is not +-- deterministic. + +-- The postings cache that a query creates for itself when `use_text_index_postings_cache = 0` is sized as 10% +-- of `max_memory_usage`. The cache used to be SLRU with the protected queue as large as the whole cache: a +-- segment that was prepared twice was promoted to the protected queue and stayed there for good, and once the +-- promoted entries filled the cache every later insert was evicted right after it landed in the probationary +-- queue, so every later segment was read from disk again on its next preparation. With plain LRU a segment is +-- built exactly once even when the segments of the query do not fit into the cache. +-- +-- The query below prepares almost every segment twice regardless of how the read pool splits the part: the two +-- `hasAllTokens` are two independent search queries, the reader keeps a separate posting list cursor per token +-- per search query, and the 31 tokens the two share are walked twice over the same rows. `segments_reused` +-- asserts that the second walk really goes through the cache instead of the disk. + +SET enable_full_text_index = 1; +SET log_queries = 1; +-- A single part: the segment count below assumes that every token has exactly 16384 postings in one part. +SET max_insert_threads = 1; + +DROP TABLE IF EXISTS tab_local_postings_cache; + +-- Every row has all 32 tokens, so each token has 16384 / 128 = 128 segments of 128 rows. The 4096 segments +-- weigh about 4.8 MB in the cache, while `max_memory_usage = 30 MB` gives the query-local cache 3 MB: enough +-- for the segments of the granule being read, but not for all of them, so a policy that pins entries runs out +-- of space halfway through the part. +CREATE TABLE tab_local_postings_cache +( + k UInt64, + s String, + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha', posting_list_codec = 'bitpacking', posting_list_block_size = 128) +) +ENGINE = MergeTree ORDER BY k +SETTINGS index_granularity = 32, index_granularity_bytes = '10M', min_bytes_for_wide_part = 0; + +INSERT INTO tab_local_postings_cache +SELECT number, arrayStringConcat(arrayMap(i -> 'tok' || toString(i), range(32)), ' ') +FROM numbers(16384); + +SELECT 'parts', count() FROM system.parts WHERE database = currentDatabase() AND table = 'tab_local_postings_cache' AND active; + +SELECT count() FROM tab_local_postings_cache +WHERE hasAllTokens(s, ['tok0', 'tok1', 'tok2', 'tok3', 'tok4', 'tok5', 'tok6', 'tok7', 'tok8', 'tok9', 'tok10', 'tok11', 'tok12', 'tok13', 'tok14', 'tok15', + 'tok16', 'tok17', 'tok18', 'tok19', 'tok20', 'tok21', 'tok22', 'tok23', 'tok24', 'tok25', 'tok26', 'tok27', 'tok28', 'tok29', 'tok30', 'tok31']) + AND hasAllTokens(s, ['tok1', 'tok2', 'tok3', 'tok4', 'tok5', 'tok6', 'tok7', 'tok8', 'tok9', 'tok10', 'tok11', 'tok12', 'tok13', 'tok14', 'tok15', + 'tok16', 'tok17', 'tok18', 'tok19', 'tok20', 'tok21', 'tok22', 'tok23', 'tok24', 'tok25', 'tok26', 'tok27', 'tok28', 'tok29', 'tok30', 'tok31']) +SETTINGS + text_index_posting_list_apply_mode = 'lazy', + query_plan_direct_read_from_text_index = 1, + use_skip_indexes_on_data_read = 1, + query_plan_optimize_count_from_text_index = 0, + use_query_condition_cache = 0, + use_text_index_postings_cache = 0, + merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0, + max_threads = 1, + max_memory_usage = 30000000, + log_comment = '05229_local_postings_cache'; + +SYSTEM FLUSH LOGS query_log; + +-- Every segment was read and decoded exactly once (32 tokens * 128 segments), and the repeated preparations +-- of the 31 shared tokens were served from the query-local cache. +SELECT + ProfileEvents['TextIndexLazySegmentsBuilt'] AS segments_built, + ProfileEvents['TextIndexLazySegmentsPrepared'] > ProfileEvents['TextIndexLazySegmentsBuilt'] AS segments_reused +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 + AND current_database = currentDatabase() + AND type = 'QueryFinish' + AND log_comment = '05229_local_postings_cache'; + +DROP TABLE tab_local_postings_cache; From b4832d0ccf38caf0d50afdc87ee4416667e1ad3e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 07:44:04 +0000 Subject: [PATCH 046/296] Backport #115991 to 26.8: Validate the Keeper path when converting a table with ATTACH AS REPLICATED --- src/Databases/DatabaseOrdinary.cpp | 18 ++ src/Databases/DatabaseOrdinary.h | 4 + src/Interpreters/InterpreterCreateQuery.cpp | 5 + src/Storages/TableZnodeInfo.h | 4 +- .../configs/config.d/clusters_name_path.xml | 23 ++ .../test_unsafe_name.py | 222 ++++++++++++++++++ ..._as_replicated_keeper_path_macro.reference | 8 + ..._attach_as_replicated_keeper_path_macro.sh | 74 ++++++ 8 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml create mode 100644 tests/integration/test_modify_engine_on_restart/test_unsafe_name.py create mode 100644 tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference create mode 100755 tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 199f3eccaf55b..7bb6c4ac0100b 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -141,6 +142,22 @@ static void checkReplicaPathExists(ASTCreateQuery & create_query, ContextPtr loc ); } +void DatabaseOrdinary::checkReplicaPathIsSafe(const ASTCreateQuery & create_query, ContextPtr local_context) +{ + /// A conversion mints a path the table never had, so the substituted name is validated as strictly + /// as a CREATE validates it -- but one level below CREATE, because the requirement that a path start + /// with '/' applies to a genuinely new table, not to a template this server has long been expanding. + const auto & server_settings = local_context->getServerSettings(); + TableZnodeInfo::resolve( + server_settings[ServerSetting::default_replica_path], + server_settings[ServerSetting::default_replica_name], + StorageID(create_query.getDatabase(), create_query.getTable(), create_query.uuid), + create_query, + LoadingStrictnessLevel::SECONDARY_CREATE, + local_context, + /*validate_substitutions=*/true); +} + void DatabaseOrdinary::setMergeTreeEngine(ASTCreateQuery & create_query, ContextPtr local_context, bool replicated) { auto * storage = create_query.storage; @@ -232,6 +249,7 @@ void DatabaseOrdinary::convertMergeTreeToReplicatedIfNeeded(ASTPtr ast, const Qu LOG_INFO(log, "Found {} flag for table {}. Will try to change it's engine in metadata to replicated.", CONVERT_TO_REPLICATED_FLAG_NAME, backQuote(qualified_name.getFullName())); + checkReplicaPathIsSafe(create_query, getContext()); checkReplicaPathExists(create_query, getContext()); setMergeTreeEngine(create_query, getContext(), /*replicated*/ true); diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 72893830f53d0..73956b751a101 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -88,6 +88,10 @@ class DatabaseOrdinary : public DatabaseOnDisk static void setMergeTreeEngine(ASTCreateQuery & create_query, ContextPtr context, bool replicated); + /// Rejects a conversion to a replicated engine whose Keeper path would not be a safe one. + /// Contacts nothing and mutates nothing, so a caller can run it before its own side effects. + static void checkReplicaPathIsSafe(const ASTCreateQuery & create_query, ContextPtr context); + protected: /// Erase pending async load/startup task references for a table. Must hold `mutex`. /// Shared by detachTableUnlocked and the Atomic rename detach path (issue #91777). diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 73a9fd4bd0517..bac302032c8a6 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -3647,6 +3647,11 @@ void InterpreterCreateQuery::convertMergeTreeTableIfPossible(ASTCreateQuery & cr else if (!to_replicated) throw Exception(ErrorCodes::INCORRECT_QUERY, "Can not attach table as not replicated, table is already not replicated"); + /// Must precede every side effect below: neither the transaction metadata removal nor the + /// metadata rewrite can be rolled back. The other direction takes no Keeper path at all. + if (to_replicated) + DatabaseOrdinary::checkReplicaPathIsSafe(create, getContext()); + /// Ensure the old detached table instance is destroyed before we remove /// transaction metadata files. Otherwise the old table's parts still hold /// in-memory version metadata referencing those files, and the debug diff --git a/src/Storages/TableZnodeInfo.h b/src/Storages/TableZnodeInfo.h index b75cfbf65cae5..ff0c69ec681f5 100644 --- a/src/Storages/TableZnodeInfo.h +++ b/src/Storages/TableZnodeInfo.h @@ -58,7 +58,9 @@ struct TableZnodeInfo /// `validate_substitutions` rejects a {database}/{table} value that would not stay a single safe /// ZooKeeper path component. It must be requested only for a freshly supplied definition: enabling /// it while merely re-deriving the path of an existing table (a short ATTACH, a Replicated-database - /// recovery replay, a RESTORE) would break that table. + /// recovery replay, a RESTORE) would break that table. Converting a MergeTree table to a + /// replicated engine counts as fresh: the path is minted from the server's template at + /// conversion time rather than read back from the table's own metadata. static TableZnodeInfo resolve( const String & requested_path, const String & requested_replica_name, const StorageID & table_id, const ASTCreateQuery & query, LoadingStrictnessLevel mode, diff --git a/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml b/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml new file mode 100644 index 0000000000000..7f5c93e20a719 --- /dev/null +++ b/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml @@ -0,0 +1,23 @@ + + + + + true + + ch1 + 9000 + + + + + + + 01 + + + +/clickhouse/tables/{database}/{table} + + diff --git a/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py b/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py new file mode 100644 index 0000000000000..d6619fa56640e --- /dev/null +++ b/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py @@ -0,0 +1,222 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import get_table_path, set_convert_flags + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + main_configs=[ + "configs/config.d/clusters_name_path.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, + macros={"replica": "node1"}, + stay_alive=True, +) + +database_name = "modify_engine_unsafe_name" + +# A name-based `default_replica_path` splices the table's own name into the Keeper path, so a name +# carrying '/' resolves inside another table's subtree. `victim` is the co-tenant that gets damaged; +# the ghost's name is exactly the replica path underneath it. +VICTIM = "victim" +GHOST = "victim/replicas/ghost" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def q(query): + return ch1.query(database=database_name, sql=query) + + +def victim_total_replicas(): + return q( + f"SELECT total_replicas FROM system.replicas WHERE database = '{database_name}' AND table = '{VICTIM}'" + ).strip() + + +def active_part_paths(table): + """Absolute in-container paths of the table's active parts, each with a trailing '/'. + + `system.parts` rather than a directory listing: a MergeTree table's data directory always holds + a top-level `detached` directory too, and Outdated parts keep their own, so enumerating + directories counts more than the parts. + """ + return q( + "SELECT path FROM system.parts" + f" WHERE database = '{database_name}' AND table = '{table}' AND active" + ).split() + + +def count_txn_version_files(table): + path = get_table_path(ch1, table, database_name) + return int( + ch1.exec_in_container( + ["bash", "-c", f"find {path} -name txn_version.txt | wc -l"] + ).strip() + ) + + +def plant_txn_version_files(table): + """Write a valid non-transactional `txn_version.txt` onto every active part of `table`. + + Transactions are off by default, so parts carry no such file and a bare count of them cannot + tell whether the conversion removed anything. The content is the form `VersionInfo` itself + emits, so a surviving file still loads. + """ + nil = "00000000-0000-0000-0000-000000000000" + content = ( + "version: 1\\n" + "storing_version: 1\\n" + f"creation_tid: (1, 1, {nil})\\n" + "creation_csn: 1\\n" + f"removal_tid: (0, 0, {nil})\\n" + "removal_csn: 0" + ) + for part_path in active_part_paths(table): + ch1.exec_in_container( + ["bash", "-c", f'printf "{content}" > "{part_path}txn_version.txt"'] + ) + + +def test_attach_as_replicated_rejects_unsafe_name(started_cluster): + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + ch1.query(f"CREATE DATABASE {database_name}") + + q(f"CREATE TABLE `{VICTIM}` ( A Int64 ) ENGINE = ReplicatedMergeTree ORDER BY A") + q(f"INSERT INTO `{VICTIM}` VALUES (1)") + # The whole scenario needs the path to be derived from the name, so assert that rather than + # assuming the config took effect. + assert ( + q( + f"SELECT zookeeper_path FROM system.replicas WHERE database = '{database_name}' AND table = '{VICTIM}'" + ).strip() + == f"/clickhouse/tables/{database_name}/{VICTIM}" + ) + assert victim_total_replicas() == "1" + + # Merges are pinned off so the set of active parts, and therefore the planted file count, is the + # same before the conversion and after the re-attach below. + q( + f"CREATE TABLE `{GHOST}` ( A Int64 ) ENGINE = MergeTree ORDER BY A" + " SETTINGS max_bytes_to_merge_at_max_space_in_pool = 1" + ) + q(f"INSERT INTO `{GHOST}` VALUES (7)") + q(f"INSERT INTO `{GHOST}` VALUES (8)") + planted = len(active_part_paths(GHOST)) + # Arming assertion: with no part to plant onto, the file count reads 0 in every arm and the + # transaction-metadata assertion below discriminates nothing. + assert planted > 0 + plant_txn_version_files(GHOST) + assert count_txn_version_files(GHOST) == planted + q(f"DETACH TABLE `{GHOST}`") + + # The conversion must be refused. Without the check it succeeds and the table takes a path + # under the victim's own subtree. + assert "BAD_ARGUMENTS" in ch1.query_and_get_error( + f"ATTACH TABLE `{GHOST}` AS REPLICATED", database=database_name + ) + + # The rejection ran before the metadata rewrite, so a plain ATTACH brings the table back as + # the MergeTree it always was. A rejection sited after the rewrite reports ReplicatedMergeTree. + q(f"ATTACH TABLE `{GHOST}`") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}'" + ).strip() + == "MergeTree" + ) + + # It also ran before the table's transaction metadata was removed. That removal is + # irreversible, so the file count is what pins the check ahead of `clearTransactionMetadata`; the + # row count below is a plain no-regression line. + assert count_txn_version_files(GHOST) == planted + assert q(f"SELECT count() FROM `{GHOST}`").strip() == "2" + + # The victim is untouched, and still accepts a metadata ALTER. A planted ghost replica + # leaves this failing with a Keeper error over the ghost's missing log_pointer. + assert victim_total_replicas() == "1" + assert ( + q( + f"SELECT groupArray(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/{database_name}/{VICTIM}/replicas'" + ).strip() + == "['node1']" + ) + q(f"ALTER TABLE `{VICTIM}` ADD COLUMN B UInt64 SETTINGS alter_sync = 2") + + # Control: a path-safe name converts fine under the very same configuration, so the + # rejection above is about the name and not about the config or the conversion route. + q("CREATE TABLE safe_name ( A Int64 ) ENGINE = MergeTree ORDER BY A") + q("INSERT INTO safe_name VALUES (1), (2)") + q("DETACH TABLE safe_name") + q("ATTACH TABLE safe_name AS REPLICATED") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = 'safe_name'" + ).strip() + == "ReplicatedMergeTree" + ) + assert q("SELECT count() FROM safe_name").strip() == "2" + + # Control: the reverse direction mints no Keeper path, so an unsafe name must not block it. + # The table has to be replicated already AND unsafely named, which conversion cannot produce, so + # it is created directly with an explicit path outside the victim's subtree. + q( + f"CREATE TABLE `{GHOST}2` ( A Int64 )" + f" ENGINE = ReplicatedMergeTree('/clickhouse/unrelated/{database_name}', 'node1') ORDER BY A" + ) + q(f"INSERT INTO `{GHOST}2` VALUES (1), (2)") + q(f"DETACH TABLE `{GHOST}2`") + q(f"ATTACH TABLE `{GHOST}2` AS NOT REPLICATED") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}2'" + ).strip() + == "MergeTree" + ) + assert q(f"SELECT count() FROM `{GHOST}2`").strip() == "2" + + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + + +def test_convert_flag_rejects_unsafe_name(started_cluster): + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + ch1.query(f"CREATE DATABASE {database_name}") + + q(f"CREATE TABLE `{GHOST}` ( A Int64 ) ENGINE = MergeTree ORDER BY A") + q(f"INSERT INTO `{GHOST}` VALUES (1), (2)") + set_convert_flags(ch1, database_name, [GHOST]) + # Read while the server is up: every helper here answers from a query, and between the stop and + # the recovery start below there is provably no server to answer one. + table_data_path = get_table_path(ch1, GHOST, database_name) + + # The flag-file route refuses the same conversion, and because it runs during startup the + # server does not come up. That is the behaviour the sibling `checkReplicaPathExists` already + # has on this route (see test_zk_path_exists.py), and the recovery is the same: delete the flag. + ch1.stop_clickhouse() + ch1.start_clickhouse(start_wait_sec=120, expected_to_fail=True) + + ch1.exec_in_container( + ["bash", "-c", f"rm {table_data_path}convert_to_replicated"] + ) + ch1.start_clickhouse() + + # The table is intact: still a MergeTree, still holding its rows. + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}'" + ).strip() + == "MergeTree" + ) + assert q(f"SELECT count() FROM `{GHOST}`").strip() == "2" + + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") diff --git a/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference new file mode 100644 index 0000000000000..d91901a6aeaf4 --- /dev/null +++ b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference @@ -0,0 +1,8 @@ +safe_name_converts ReplicatedMergeTree 2 +definition_not_rewritten 1 +as_not_replicated_allows_unsafe_name MergeTree +stored_macro_armed 1 +short_attach OK +short_attach_count 1 +restart_replica OK +restart_replica_count 1 diff --git a/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh new file mode 100755 index 0000000000000..60ade65ffbc3a --- /dev/null +++ b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-replicated-database, no-ordinary-database, no-shared-merge-tree +# Rows needing a per-copy unique database name, which a .sql file cannot interpolate. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The rejection this test guards fires only on a name-based `default_replica_path`, which the test +# config does not set (it is {uuid}-based, and a {uuid} is never path-unsafe). So the treatment row +# lives in tests/integration/test_modify_engine_on_restart/test_unsafe_name.py, which configures +# that setting, and the rows here pin the exemptions the rejection must not disturb. +CLIENT="${CLICKHOUSE_CLIENT} --server_logs_file=/dev/null" + +# Converting a table with a path-safe name still works. Guards the check against rejecting the +# ordinary case: the shipped {uuid} template resolves through the same code path. +${CLIENT} -q "CREATE TABLE c1 (c0 Int) ENGINE = MergeTree ORDER BY c0" +${CLIENT} -q "INSERT INTO c1 VALUES (1), (2)" +${CLIENT} -q "DETACH TABLE c1" +${CLIENT} -q "ATTACH TABLE c1 AS REPLICATED" +${CLIENT} -q "SELECT 'safe_name_converts', engine, (SELECT count() FROM c1) FROM system.tables WHERE database = currentDatabase() AND name = 'c1'" + +# The check only reads the definition, it must not rewrite the engine arguments. The stored +# template stays unexpanded, which is what keeps a metadata file copyable between replicas. +${CLIENT} -q "SELECT 'definition_not_rewritten', create_table_query LIKE '%{uuid}%' FROM system.tables WHERE database = currentDatabase() AND name = 'c1'" +${CLIENT} -q "DROP TABLE c1" + +# The opposite direction strips the path arguments instead of minting one, so an unsafe name is +# irrelevant to it. The path is given explicitly here, so the CREATE itself is legal. +${CLIENT} -q "CREATE TABLE \`c3/unsafe\` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04853/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/c3', 'r1') ORDER BY c0" +${CLIENT} -q "DETACH TABLE \`c3/unsafe\`" +${CLIENT} -q "ATTACH TABLE \`c3/unsafe\` AS NOT REPLICATED" +${CLIENT} -q "SELECT 'as_not_replicated_allows_unsafe_name', engine FROM system.tables WHERE database = currentDatabase() AND name = 'c3/unsafe'" +${CLIENT} -q "DROP TABLE \`c3/unsafe\`" + +# A table whose STORED path re-expands to a path-unsafe value keeps loading, both through a +# short ATTACH and through SYSTEM RESTART REPLICA. Two arming details: +# * only a CONFIGURED macro survives into metadata unexpanded; a direct {database} is unfolded at +# CREATE, leaving nothing to re-substitute. +# * the DATABASE is renamed rather than the table, because RenamingRestrictions refuses to rename +# a table whose path carries an implicit macro. +# Report the exemption from the client's exit status, not from a grep for one error code: the table +# is registered in the catalog before `startup()` runs on both routes, so a failure there leaves the +# table present and the count assertions below cannot see it. +run_exempt() { + local label=$1 query=$2 out rc + out=$(${CLIENT} -q "$query" 2>&1) + rc=$? + if [ "$rc" = "0" ]; then + echo "$label OK" + else + echo "$label FAILED rc=$rc: $out" + fi +} + +LEGACY_DB="${CLICKHOUSE_DATABASE}_legacy" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${LEGACY_DB}/d\` SYNC" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${LEGACY_DB}\` SYNC" +${CLIENT} -q "CREATE DATABASE \`${LEGACY_DB}\`" +${CLIENT} -q "CREATE TABLE \`${LEGACY_DB}\`.t (c0 Int) ENGINE = ReplicatedMergeTree('{default_path_test}04853legacy', 'r2') ORDER BY c0" +${CLIENT} -q "SELECT 'stored_macro_armed', create_table_query LIKE '%{default_path_test}%' FROM system.tables WHERE database = '${LEGACY_DB}' AND name = 't'" +${CLIENT} -q "RENAME DATABASE \`${LEGACY_DB}\` TO \`${LEGACY_DB}/d\`" +${CLIENT} -q "DETACH TABLE \`${LEGACY_DB}/d\`.t" +run_exempt short_attach "ATTACH TABLE \`${LEGACY_DB}/d\`.t" +${CLIENT} -q "SELECT 'short_attach_count', count() FROM system.tables WHERE database = '${LEGACY_DB}/d' AND name = 't'" +run_exempt restart_replica "SYSTEM RESTART REPLICA \`${LEGACY_DB}/d\`.t" +${CLIENT} -q "SELECT 'restart_replica_count', count() FROM system.tables WHERE database = '${LEGACY_DB}/d' AND name = 't'" +# Re-resolve the path under the original name before dropping: the stored path re-expands {database}, +# so under the new name the table points at a different znode tree than the CREATE made, and dropping +# it there would leave the original tree behind. +${CLIENT} -q "RENAME DATABASE \`${LEGACY_DB}/d\` TO \`${LEGACY_DB}\`" +${CLIENT} -q "DETACH TABLE \`${LEGACY_DB}\`.t" +${CLIENT} -q "ATTACH TABLE \`${LEGACY_DB}\`.t" +${CLIENT} -q "DROP DATABASE \`${LEGACY_DB}\` SYNC" From 5f81d96e4040c2aee45c61c26d776ea5d38049c7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 11:16:24 +0000 Subject: [PATCH 047/296] Backport #120245 to 26.8: Fix wrong count() after a metadata-only MODIFY COLUMN to Nullable --- src/Storages/MergeTree/MergeTreeData.cpp | 13 ++++++++++++ ...filter_after_metadata_only_alter.reference | 6 ++++++ ...rsity_filter_after_metadata_only_alter.sql | 20 +++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.reference create mode 100644 tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.sql diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0fe87551a1a4d..592d37ed175b2 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7384,12 +7384,25 @@ MergeTreeData::getColumnDefaultnessStats(const String & column_name, ContextPtr return std::nullopt; } + auto metadata_snapshot = getInMemoryMetadataPtr(query_context, /*bypass_metadata_cache=*/ false); + auto column_in_metadata = metadata_snapshot->getColumns().tryGetPhysical(column_name); + if (!column_in_metadata) + return std::nullopt; + ColumnDefaultnessStats aggregate; for (const auto & part : getActivePartsForColumnDefaultnessStats(query_context)) { if (part->isEmpty()) continue; + /// A metadata-only `MODIFY COLUMN` (e.g. `UInt64` -> `Nullable(UInt64)`) does not rewrite the part, + /// so its `num_defaults` counts defaults of the old type while reads return the new type. + if (part->getColumnsDescription().tryGetPhysical(column_name) != column_in_metadata) + { + LOG_DEBUG(log, "No defaultness stats for column {}: type in part {} differs from the type in metadata", column_name, part->name); + return std::nullopt; + } + const auto & infos = part->getSerializationInfos(); auto it = infos.find(column_name); if (it == infos.end()) diff --git a/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.reference b/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.reference new file mode 100644 index 0000000000000..c1b0a0d45e864 --- /dev/null +++ b/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.reference @@ -0,0 +1,6 @@ +0 +100 +0 +100 +0 +100 diff --git a/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.sql b/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.sql new file mode 100644 index 0000000000000..bce113e7f2149 --- /dev/null +++ b/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.sql @@ -0,0 +1,20 @@ +-- A metadata-only widening to Nullable does not rewrite the part, so the recorded number of +-- default values still counts zeros (or empty strings), not NULLs. + +CREATE TABLE t (k UInt64, v UInt64, s String) ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; +INSERT INTO t SELECT number, number % 10, if(number % 10 = 0, '', 'x') FROM numbers(100); + +ALTER TABLE t MODIFY COLUMN v Nullable(UInt64) SETTINGS mutations_sync = 2; +ALTER TABLE t MODIFY COLUMN s Nullable(String) SETTINGS mutations_sync = 2; + +SELECT count() FROM t WHERE v IS NULL; +SELECT count() FROM t WHERE v IS NOT NULL; +SELECT count() FROM t WHERE s IS NULL; +SELECT count() FROM t WHERE s IS NOT NULL; + +-- The stats are usable again once the part is rewritten with the new type. +OPTIMIZE TABLE t FINAL; +SELECT count() FROM t WHERE v IS NULL; +SELECT count() FROM t WHERE v IS NOT NULL; + +DROP TABLE t; From ad4bbf121760671e783a95025a6c27862f3d02a0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 12:48:04 +0000 Subject: [PATCH 048/296] Backport #119346 to 26.8: Bump `mongo-c-driver` to 2.5.4 and `mongo-cxx-driver` to r4.6.0 --- .gitmodules | 2 +- contrib/mongo-c-driver | 2 +- contrib/mongo-c-driver-cmake/CMakeLists.txt | 14 +++++++------- contrib/mongo-cxx-driver | 2 +- contrib/mongo-cxx-driver-cmake/CMakeLists.txt | 19 +++++++++++++++++++ 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/.gitmodules b/.gitmodules index b7e34d05214d7..2f3dd2930a0a0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -329,7 +329,7 @@ url = https://github.com/ClickHouse/double-conversion.git [submodule "contrib/mongo-cxx-driver"] path = contrib/mongo-cxx-driver - url = https://github.com/mongodb/mongo-cxx-driver.git + url = https://github.com/ClickHouse/mongo-cxx-driver.git [submodule "contrib/mongo-c-driver"] path = contrib/mongo-c-driver url = https://github.com/ClickHouse/mongo-c-driver.git diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index 6b6b676bdbd46..a00f0b91bd24f 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit 6b6b676bdbd46fdb954ed1535893020ef91cce5b +Subproject commit a00f0b91bd24f8ddbe62ec3864cb3704385670f5 diff --git a/contrib/mongo-c-driver-cmake/CMakeLists.txt b/contrib/mongo-c-driver-cmake/CMakeLists.txt index 74cd35eef8aa7..a79ef97c8fe2c 100644 --- a/contrib/mongo-c-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-c-driver-cmake/CMakeLists.txt @@ -5,13 +5,13 @@ if(NOT USE_MONGODB) endif() set(libbson_VERSION_MAJOR 2) -set(libbson_VERSION_MINOR 3) -set(libbson_VERSION_PATCH 0) -set(libbson_VERSION 2.3.0) +set(libbson_VERSION_MINOR 5) +set(libbson_VERSION_PATCH 4) +set(libbson_VERSION 2.5.4) set(libmongoc_VERSION_MAJOR 2) -set(libmongoc_VERSION_MINOR 3) -set(libmongoc_VERSION_PATCH 0) -set(libmongoc_VERSION 2.3.0) +set(libmongoc_VERSION_MINOR 5) +set(libmongoc_VERSION_PATCH 4) +set(libmongoc_VERSION 2.5.4) set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src") set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src") @@ -135,7 +135,7 @@ endif() set(LIBMONGOC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libmongoc/src") set(LIBMONGOC_BINARY_DIR "${LIBBSON_BINARY_ROOT}/libmongoc/src") file(GLOB_RECURSE LIBMONGOC_SOURCES "${LIBMONGOC_SOURCE_DIR}/*.c") -set(UTF8PROC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/utf8proc-2.8.0") +set(UTF8PROC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/utf8proc-2.11.3") set(UTF8PROC_SOURCES "${UTF8PROC_SOURCE_DIR}/utf8proc.c") set(UTHASH_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/uthash") diff --git a/contrib/mongo-cxx-driver b/contrib/mongo-cxx-driver index 9b0c13260590b..fa9ee6ed008e8 160000 --- a/contrib/mongo-cxx-driver +++ b/contrib/mongo-cxx-driver @@ -1 +1 @@ -Subproject commit 9b0c13260590bb6485fa57c20f61098870b99ab6 +Subproject commit fa9ee6ed008e84b9562a7a55d70681de8a6bc1c6 diff --git a/contrib/mongo-cxx-driver-cmake/CMakeLists.txt b/contrib/mongo-cxx-driver-cmake/CMakeLists.txt index 6fd3cc70adb2c..a859c5d8488a9 100644 --- a/contrib/mongo-cxx-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-cxx-driver-cmake/CMakeLists.txt @@ -54,10 +54,17 @@ set(BSONCXX_SOURCES set(BSONCXX_POLY_USE_IMPLS ON) +# ClickHouse uses the `v_noabi` API, whose symbols are gated behind the "unstable ABI" switch since r4.4.0. +set(BSONCXX_ENABLE_UNSTABLE_ABI ON) + configure_file( ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/config.hpp.in ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/config/config.hpp ) +configure_file( + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.hpp.in + ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.hpp +) configure_file( ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/version.hpp.in ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/config/version.hpp @@ -152,6 +159,7 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/bulk_write.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/change_stream.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_bulk_write.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_encryption.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_session.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/collection.cpp @@ -203,6 +211,9 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/insert_one_result.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/instance.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/logger.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_callback.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_callback_params.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_credential.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/pipeline.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/pool.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/range_options.cpp @@ -216,6 +227,7 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/search_indexes.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/server_api.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/server_error.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/string_options.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/text_options.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/tls.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/transaction_options.cpp @@ -336,10 +348,17 @@ endif() set(BSONCXX_STATIC 1) set(MONGOCXX_STATIC 1) +# ClickHouse uses the `v_noabi` API, whose symbols are gated behind the "unstable ABI" switch since r4.4.0. +set(MONGOCXX_ENABLE_UNSTABLE_ABI ON) + configure_file( ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/config.hpp.in ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/config/config.hpp ) +configure_file( + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.hpp.in + ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.hpp +) configure_file( ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/version.hpp.in ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/config/version.hpp From 17a9ab3309bb1dee1d80ffcca1840b8e42309bd5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 13:21:12 +0000 Subject: [PATCH 049/296] Backport #107860 to 26.8: validate array sizes in analysisOfVariance and lttb aggregate states --- src/AggregateFunctions/Moments.h | 21 ++++++++++++++++--- src/IO/ReadHelpers.h | 6 +++--- ...lysisOfVariance_deserialize_size.reference | 0 ...57_analysisOfVariance_deserialize_size.sql | 10 +++++++++ 4 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.reference create mode 100644 tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql diff --git a/src/AggregateFunctions/Moments.h b/src/AggregateFunctions/Moments.h index ead359f40e7ef..7e77f0f7ee4ea 100644 --- a/src/AggregateFunctions/Moments.h +++ b/src/AggregateFunctions/Moments.h @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; } /// Zeroes the value's bit pattern when the flag is not set. Unlike multiplying by the @@ -906,9 +907,23 @@ struct AnalysisOfVarianceMoments void read(ReadBuffer & buf) { - readVectorBinary(xs1, buf); - readVectorBinary(xs2, buf); - readVectorBinary(ns, buf); + /// add and merge cap the group count at MAX_GROUPS_NUMBER via resizeIfNeeded, but read + /// trusts the serialized lengths. Enforce the same bound here, before the vectors are + /// resized, so a crafted state cannot exceed the aggregate's own invariant. + readVectorBinary(xs1, buf, MAX_GROUPS_NUMBER); + readVectorBinary(xs2, buf, MAX_GROUPS_NUMBER); + readVectorBinary(ns, buf, MAX_GROUPS_NUMBER); + + /// The three vectors hold one entry per group and must stay equal in length. + /// The finalize path iterates up to xs1.size() and indexes xs2 and ns with the + /// same position, so a state with mismatched lengths reads out of bounds. + if (xs1.size() != xs2.size() || xs1.size() != ns.size()) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Sizes of nested arrays in analysisOfVariance state do not match: {}, {}, {}", + xs1.size(), + xs2.size(), + ns.size()); } Float64 getMeanAll() const diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index cf24492e4bd0c..2d216684a75b0 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -187,16 +187,16 @@ inline void readIPv6Binary(IPv6 & ip, ReadBuffer & buf) } template -void readVectorBinary(V & v, ReadBuffer & buf) +void readVectorBinary(V & v, ReadBuffer & buf, size_t max_size = DEFAULT_MAX_STRING_SIZE) { using T = typename V::value_type; size_t size = 0; readVarUInt(size, buf); - if (size > DEFAULT_MAX_STRING_SIZE) + if (size > max_size) throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, - "Too large array size (maximum: {})", DEFAULT_MAX_STRING_SIZE); + "Too large array size (maximum: {})", max_size); v.resize(size); diff --git a/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.reference b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql new file mode 100644 index 0000000000000..5474dc68c5192 --- /dev/null +++ b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql @@ -0,0 +1,10 @@ +-- analysisOfVariance keeps three parallel arrays in its aggregate state (one entry per group). +-- The finalize path iterates one array and indexes the others at the same position, so a state +-- deserialized from raw bytes with mismatched array lengths reads out of bounds. Such a state is +-- reachable from any user via CAST of a String to AggregateFunction. + +-- Normal aggregation builds equal-length arrays and must keep working. +SELECT analysisOfVariance(number, number % 3) FROM numbers(30) FORMAT Null; + +-- xs1 and xs2 have two groups, ns has one -> ns is read past its end. +SELECT finalizeAggregation(CAST(unhex('02000000000000f03f000000000000f03f02000000000000f03f000000000000f03f010500000000000000') AS AggregateFunction(analysisOfVariance, Float64, UInt8))); -- { serverError INCORRECT_DATA } From a12115fdef493707b204866d88cfb9919aec77cb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 14:59:02 +0000 Subject: [PATCH 050/296] Update autogenerated version to 26.8.10.6 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 0f528e8c5392b..d14f61b49bdc0 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54522) +SET(VERSION_REVISION 54523) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 10) -SET(VERSION_GITHASH 661c91e962f04611bad10ac83a0aca1a2aa0eb65) -SET(VERSION_DESCRIBE v26.8.10.1-lts) -SET(VERSION_STRING 26.8.10.1) +SET(VERSION_PATCH 11) +SET(VERSION_GITHASH bedf2ab54b8a0c34afbf2d907eaf933324f43cd6) +SET(VERSION_DESCRIBE v26.8.11.1-lts) +SET(VERSION_STRING 26.8.11.1) # end of autochange From 169b398e9629f17f5d9fbc6981250c83f1b347eb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 15:07:36 +0000 Subject: [PATCH 051/296] Backport #117699 to 26.8: Reject the Detached serialization kind in data received from a client --- .../interfaces/specs/NativeFormat.mdx | 11 +- src/Client/Connection.cpp | 5 +- src/Columns/ColumnBLOB.h | 2 + src/Columns/ColumnSparse.cpp | 5 +- src/Columns/IColumn.h | 10 +- src/Columns/tests/gtest_column_blob.cpp | 59 ++++ .../Serializations/ISerialization.cpp | 2 +- src/DataTypes/Serializations/ISerialization.h | 25 ++ .../Serializations/SerializationInfo.cpp | 53 ++- .../Serializations/SerializationInfo.h | 7 +- .../Serializations/SerializationInfoTuple.cpp | 9 +- .../Serializations/SerializationInfoTuple.h | 2 +- .../tests/gtest_serialization_info.cpp | 44 +++ src/Formats/NativeReader.cpp | 10 +- src/Formats/NativeReader.h | 13 +- ...alization_oob_index_native_protocol.python | 8 +- ...ization_from_client_native_protocol.python | 315 ++++++++++++++++++ ...tion_from_client_native_protocol.reference | 9 + ...rialization_from_client_native_protocol.sh | 11 + 19 files changed, 578 insertions(+), 22 deletions(-) create mode 100644 src/Columns/tests/gtest_column_blob.cpp create mode 100755 tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python create mode 100644 tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference create mode 100755 tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh diff --git a/docs/reference/interfaces/specs/NativeFormat.mdx b/docs/reference/interfaces/specs/NativeFormat.mdx index 4bc0faf1b9f93..9a5cf1bfe6876 100644 --- a/docs/reference/interfaces/specs/NativeFormat.mdx +++ b/docs/reference/interfaces/specs/NativeFormat.mdx @@ -196,7 +196,14 @@ The byte values differ from the compact codes: `REPLICATED` is `0x03` in this ne The `count` is the full stack length **including the leading `DEFAULT` entry** that begins every stack. The compact codes already cover every one- and two-entry stack, so a `COMBINATION` always has a `count` of at least three. -**Recursive `kind_stack` for `Tuple` columns.** The `kind_stack` payload above is the byte (or `COMBINATION` sequence) for one column's own serialization info. A `Tuple` carries a `SerializationInfoTuple`, which first writes the tuple's *own* kind-stack payload and then writes one full kind-stack payload for *each* element, in order; a decoder reads back the same recursive structure. So for `Tuple(A, B, C)` the field-4 bytes are `[tuple_kind][A_kind][B_kind][C_kind]`, and each element payload is itself recursive if that element is again a composite. The `has_custom_serialization` byte (field 3) is set whenever the tuple's own info *or any element's* info is non-default, so a `Tuple` whose only special element is sparse, replicated, or detached still triggers the kind-stack payload. A decoder that reads only the single leading enum byte for a `Tuple` will stop too early and misread the remaining element-kind bytes as column data. +**Kind stack validity.** A decoder rejects a malformed stack with `INCORRECT_DATA` rather than building the serialization it describes: + +- the stack starts with `DEFAULT`; +- the stack is a subsequence of `DEFAULT`, `SPARSE`, `REPLICATED`, `DETACHED` — the order in which the kinds wrap each other. `SPARSE` can sit inside `REPLICATED` but not the other way round, and `DETACHED` is always last because a `ColumnBLOB` holds the serialized form of everything below it. This also means no kind occurs twice, which bounds `count` by the number of kinds. + +Note that this order is not the order of the kind byte values above. Any other stack describes a column layout no writer builds and that no materialization step unwraps back to a full column of the declared type — a `ColumnSparse` whose values column is itself sparse or replicated, for instance, stays wrapped. + +**Recursive `kind_stack` for `Tuple` columns.** The `kind_stack` payload above is the byte (or `COMBINATION` sequence) for one column's own serialization info. A `Tuple` carries a `SerializationInfoTuple`, which first writes the tuple's *own* kind-stack payload and then writes one full kind-stack payload for *each* element, in order; a decoder reads back the same recursive structure. So for `Tuple(A, B, C)` the field-4 bytes are `[tuple_kind][A_kind][B_kind][C_kind]`, and each element payload is itself recursive if that element is again a composite. The `has_custom_serialization` byte (field 3) is set whenever the tuple's own info *or any element's* info is non-default, so a `Tuple` whose only special element is sparse or replicated still triggers the kind-stack payload. A decoder that reads only the single leading enum byte for a `Tuple` will stop too early and misread the remaining element-kind bytes as column data. **Sparse wire format.** When `kind_stack = 0x01`, the column `data` is two streams written back-to-back in the single shared TCP stream: @@ -230,6 +237,8 @@ A decoder reconstructs a dense column by selecting `elements[indexes[i]]` for ea If the wrapped column was sparse, its stack is `{DEFAULT, SPARSE, DETACHED}`, which serializes as `DETACHED_OVER_SPARSE`. A client decoding such a column reads the blob length and bytes, then decompresses the blob to recover the inner column payload (see the [`ColumnBLOB` note](#compression-negotiation) under compression). +`DETACHED` is directional: it may appear only in result blocks sent by a server, and only for a whole top-level column — never as one of the element kinds of a `Tuple`. Unlike the other kinds, it makes the runtime column a `ColumnBLOB` instead of the column of the declared type, so a receiver accepts it only when its own pipeline converts the blob back. A server therefore rejects `DETACHED` in any block a client sends — external table data, scalars and insert data alike — with `INCORRECT_DATA`. + ### Block variants {#block-variants} All Data-family packets share the same Block wire format. The variants differ only in their column and row counts: diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index a2f9e9dbd1762..37b5593026278 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1692,7 +1693,9 @@ void Connection::initBlockInput() if (!block_in) { initMaybeCompressedInput(); - block_in = std::make_unique(*maybe_compressed_in, server_revision, format_settings); + /// The server may send marshalled result blocks; their consumers convert them back. + block_in = std::make_unique( + *maybe_compressed_in, server_revision, format_settings, ISerialization::KindSet::all()); } } diff --git a/src/Columns/ColumnBLOB.h b/src/Columns/ColumnBLOB.h index cd1ab67703e40..2778db7aff5de 100644 --- a/src/Columns/ColumnBLOB.h +++ b/src/Columns/ColumnBLOB.h @@ -112,6 +112,8 @@ class ColumnBLOB final : public COWHelper, ColumnBLOB> return from_blob_task(blob); } + ColumnPtr convertToFullColumnIfDetached() const override { return convertFrom(); } + /// Creates serialized and compressed blob from the source column. static void toBLOB( BLOB & blob, diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index e749675070d6a..5df6b0586425c 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -1012,8 +1012,9 @@ ColumnPtr removeSpecialRepresentations(const ColumnPtr & column) if (!column) return column; - /// We can have only Replicated(Sparse) but not Sparse(Replicated). - return recursiveRemoveSparse(column->convertToFullColumnIfReplicated()); + /// Order matters: the BLOB holds the serialized form of everything below it, and we can have + /// only Replicated(Sparse) but not Sparse(Replicated). + return recursiveRemoveSparse(column->convertToFullColumnIfDetached()->convertToFullColumnIfReplicated()); } } diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 44a7409b5abef..bf36f6191c98b 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -132,14 +132,20 @@ class IColumn : public COW /// If column is ColumnReplicated, transforms it to full column. [[nodiscard]] virtual Ptr convertToFullColumnIfReplicated() const { return getPtr(); } - /// Recursively strip internal representation wrappers (Const, Replicated, Sparse) + /// If column isn't ColumnBLOB, return itself. + /// If column is ColumnBLOB, deserializes the BLOB back into the column it holds. + [[nodiscard]] virtual Ptr convertToFullColumnIfDetached() const { return getPtr(); } + + /// Recursively strip internal representation wrappers (Const, Detached, Replicated, Sparse) /// from this column and all its subcolumns. Does NOT strip LowCardinality — that is /// a semantic type, not a representation wrapper. Callers that also need LowCardinality /// removed should chain ->convertToFullColumnIfLowCardinality() for top-level removal, /// or use recursiveRemoveLowCardinality for recursive removal. [[nodiscard]] virtual Ptr convertToFullIfWrapped() const { - Ptr converted = convertToFullColumnIfConst() + /// Detached goes first: the BLOB holds the serialized form of everything below it. + Ptr converted = convertToFullColumnIfDetached() + ->convertToFullColumnIfConst() ->convertToFullColumnIfReplicated() ->convertToFullColumnIfSparse(); diff --git a/src/Columns/tests/gtest_column_blob.cpp b/src/Columns/tests/gtest_column_blob.cpp new file mode 100644 index 0000000000000..e9b34cfecac9c --- /dev/null +++ b/src/Columns/tests/gtest_column_blob.cpp @@ -0,0 +1,59 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// The real task deserializes the BLOB; handing back `inner` exercises the same unwrapping plumbing. +ColumnPtr makeBLOB(const ColumnPtr & inner) +{ + auto blob = ColumnBLOB::create(inner->cloneEmpty()); + blob->setFromBLOBTask([inner](const ColumnBLOB::BLOB &) { return inner; }); + return blob; +} + +} + +/// `ColumnBLOB` is a transport representation that must be stripped by the generic wrapper-removal +/// helpers: their callers go on to `assert_cast` the column to the one the declared type produces. +TEST(ColumnBLOB, ConvertToFullIfWrappedRemovesBLOB) +{ + auto inner = ColumnUInt64::create(); + inner->insertValue(42); + ColumnPtr inner_ptr = std::move(inner); + + auto full = makeBLOB(inner_ptr)->convertToFullIfWrapped(); + + EXPECT_EQ(typeid_cast(full.get()), nullptr); + EXPECT_EQ(full.get(), inner_ptr.get()); +} + +TEST(ColumnBLOB, RemoveSpecialRepresentationsRemovesBLOB) +{ + auto inner = ColumnUInt64::create(); + inner->insertValue(42); + ColumnPtr inner_ptr = std::move(inner); + + auto full = removeSpecialRepresentations(makeBLOB(inner_ptr)); + + EXPECT_EQ(typeid_cast(full.get()), nullptr); + EXPECT_EQ(full.get(), inner_ptr.get()); +} + +/// A BLOB can wrap a sparse column (kind stack `{Default, Sparse, Detached}`), so removing it has to +/// leave the column it uncovers to the rest of the chain. +TEST(ColumnBLOB, RemovalContinuesIntoTheUncoveredColumn) +{ + auto values = ColumnUInt64::create(); + ColumnPtr sparse = ColumnSparse::create(std::move(values)); + + auto full = removeSpecialRepresentations(makeBLOB(sparse)); + + EXPECT_EQ(typeid_cast(full.get()), nullptr); + EXPECT_EQ(typeid_cast(full.get()), nullptr); +} diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 12aca2a07c32a..7194ada0ee5c0 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -97,7 +97,7 @@ ISerialization::KindStack ISerialization::getKindStack(const IColumn & column) return {Kind::DEFAULT}; } -static String kindToString(ISerialization::Kind kind) +String ISerialization::kindToString(ISerialization::Kind kind) { switch (kind) { diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index abfe3cdce0b32..9ec18aafc98d7 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -94,6 +94,30 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro /// - etc using KindStack = std::vector; + /// The kind of a column in the Native format is chosen by the peer that sends the data, so a + /// reader declares with such a set which kinds the peer is allowed to select. + class KindSet + { + public: + constexpr KindSet(std::initializer_list kinds) /// NOLINT(google-explicit-constructor) + { + for (auto kind : kinds) + bits |= maskOf(kind); + } + + static constexpr KindSet all() { return KindSet(~UInt32(0)); } + + constexpr bool contains(Kind kind) const { return (bits & maskOf(kind)) != 0; } + constexpr KindSet without(Kind kind) const { return KindSet(bits & ~maskOf(kind)); } + + private: + explicit constexpr KindSet(UInt32 bits_) : bits(bits_) { } + + static constexpr UInt32 maskOf(Kind kind) { return UInt32(1) << static_cast(kind); } + + UInt32 bits = 0; + }; + virtual KindStack getKindStack() const { return {Kind::DEFAULT}; } SerializationPtr getPtr() const { return shared_from_this(); } @@ -104,6 +128,7 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro virtual MutableColumnPtr wrapColumnForDeserialization(MutableColumnPtr column) const { return column; } static KindStack getKindStack(const IColumn & column); + static String kindToString(Kind kind); static String kindStackToString(const KindStack & kind); static KindStack stringToKindStack(const String & str); /// Check if provided kind stack contains specific kind. diff --git a/src/DataTypes/Serializations/SerializationInfo.cpp b/src/DataTypes/Serializations/SerializationInfo.cpp index 49caa07fcd5e5..4a83ab59a5ab2 100644 --- a/src/DataTypes/Serializations/SerializationInfo.cpp +++ b/src/DataTypes/Serializations/SerializationInfo.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include #include #include @@ -20,6 +23,7 @@ namespace DB namespace ErrorCodes { extern const int CORRUPTED_DATA; + extern const int INCORRECT_DATA; } namespace @@ -254,7 +258,46 @@ void SerializationInfo::serialializeKindStackBinary(WriteBuffer & out) const } } -void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in) +/// The order in which the kinds wrap each other, innermost first: `ColumnSparse` can sit inside +/// `ColumnReplicated` but not the other way round (see `removeSpecialRepresentations`), and nothing +/// wraps a `ColumnBLOB`. Not the order of the enum, whose values are part of the Native format. +static constexpr std::array canonical_kind_order +{ + ISerialization::Kind::DEFAULT, + ISerialization::Kind::SPARSE, + ISerialization::Kind::REPLICATED, + ISerialization::Kind::DETACHED, +}; + +void SerializationInfo::checkKindStack(ISerialization::KindSet allowed_kinds) const +{ + if (kind_stack.empty() || kind_stack.front() != ISerialization::Kind::DEFAULT) + throw Exception(ErrorCodes::INCORRECT_DATA, "Serialization kind stack must start with Default"); + + /// A stack describes nested wrappers, so it must be a subsequence of the canonical order — and + /// therefore free of repeats. Any other stack is a layout no writer builds and nothing unwraps. + auto expected = canonical_kind_order.begin(); + + for (auto kind : kind_stack) + { + if (!allowed_kinds.contains(kind)) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Unexpected serialization kind {} in the received data", + ISerialization::kindToString(kind)); + + expected = std::find(expected, canonical_kind_order.end(), kind); + if (expected == canonical_kind_order.end()) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Serialization kind {} is out of order in a kind stack", + ISerialization::kindToString(kind)); + + ++expected; + } +} + +void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in, ISerialization::KindSet allowed_kinds) { UInt8 type = 0; readBinary(type, in); @@ -283,6 +326,12 @@ void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in) { size_t num_kinds = 0; readVarUInt(num_kinds, in); + /// Refuse an impossible peer-declared count before reading that many kinds; + /// `checkKindStack` rejects the same stacks afterwards by their shape. + if (num_kinds > magic_enum::enum_count()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Too many serialization kinds in a kind stack: {}", num_kinds); + + kind_stack.clear(); for (size_t i = 0; i != num_kinds; ++i) { UInt8 kind = 0; @@ -296,6 +345,8 @@ void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in) break; } } + + checkKindStack(allowed_kinds); } void SerializationInfo::writeJSONFields(WriteBuffer & out, const String * name) const diff --git a/src/DataTypes/Serializations/SerializationInfo.h b/src/DataTypes/Serializations/SerializationInfo.h index 977a25a2a8de7..f120bd62d778a 100644 --- a/src/DataTypes/Serializations/SerializationInfo.h +++ b/src/DataTypes/Serializations/SerializationInfo.h @@ -76,7 +76,9 @@ class SerializationInfo const SerializationInfoSettings & new_settings) const; virtual void serialializeKindStackBinary(WriteBuffer & out) const; - virtual void deserializeFromKindsBinary(ReadBuffer & in); + + /// Rejects a kind outside of `allowed_kinds` as invalid data. + virtual void deserializeFromKindsBinary(ReadBuffer & in, ISerialization::KindSet allowed_kinds); virtual void writeJSON(WriteBuffer & out, const String * name) const; virtual void toJSON(Poco::JSON::Object & object) const; @@ -93,6 +95,9 @@ class SerializationInfo protected: virtual void writeJSONFields(WriteBuffer & out, const String * name) const; + /// Rejects a kind stack that no writer can produce, or that selects a kind the reader does not accept. + void checkKindStack(ISerialization::KindSet allowed_kinds) const; + const SerializationInfoSettings settings; ISerialization::KindStack kind_stack; diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.cpp b/src/DataTypes/Serializations/SerializationInfoTuple.cpp index 35c4d20d80e49..f572c95c98008 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.cpp +++ b/src/DataTypes/Serializations/SerializationInfoTuple.cpp @@ -161,11 +161,14 @@ void SerializationInfoTuple::serialializeKindStackBinary(WriteBuffer & out) cons elem->serialializeKindStackBinary(out); } -void SerializationInfoTuple::deserializeFromKindsBinary(ReadBuffer & in) +void SerializationInfoTuple::deserializeFromKindsBinary(ReadBuffer & in, ISerialization::KindSet allowed_kinds) { - SerializationInfo::deserializeFromKindsBinary(in); + SerializationInfo::deserializeFromKindsBinary(in, allowed_kinds); + + /// A detached blob always covers a whole column, never a single tuple element. + auto elements_allowed_kinds = allowed_kinds.without(ISerialization::Kind::DETACHED); for (const auto & elem : elems) - elem->deserializeFromKindsBinary(in); + elem->deserializeFromKindsBinary(in, elements_allowed_kinds); } void SerializationInfoTuple::writeJSONFields(WriteBuffer & out, const String * name) const diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.h b/src/DataTypes/Serializations/SerializationInfoTuple.h index ea826527d1c16..9c5a1401260b0 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.h +++ b/src/DataTypes/Serializations/SerializationInfoTuple.h @@ -27,7 +27,7 @@ class SerializationInfoTuple : public SerializationInfo const Settings & new_settings) const override; void serialializeKindStackBinary(WriteBuffer & out) const override; - void deserializeFromKindsBinary(ReadBuffer & in) override; + void deserializeFromKindsBinary(ReadBuffer & in, ISerialization::KindSet allowed_kinds) override; void toJSON(Poco::JSON::Object & object) const override; void fromJSON(const Poco::JSON::Object & object) override; diff --git a/src/DataTypes/Serializations/tests/gtest_serialization_info.cpp b/src/DataTypes/Serializations/tests/gtest_serialization_info.cpp index 10fa60a14d59c..bb648614bb188 100644 --- a/src/DataTypes/Serializations/tests/gtest_serialization_info.cpp +++ b/src/DataTypes/Serializations/tests/gtest_serialization_info.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -312,4 +313,47 @@ TEST(SerializationInfoJSON, ChooseKindStackZeroRows) EXPECT_EQ(kind_stack, expected); } +/// Only a reader that accepts `Kind::DETACHED` at all reaches this check, and no such reader is exposed +/// to a client, so it cannot be covered by a functional test. +TEST(SerializationInfoBinary, RejectsDetachedThatIsNotOutermost) +{ + /// COMBINATION encoding of {Default, Detached, Sparse}. + const char kinds[] = {5, 3, 0, 2, 1}; + ReadBufferFromMemory in(kinds, sizeof(kinds)); + + SerializationInfo info({ISerialization::Kind::DEFAULT}, defaultSettings()); + EXPECT_THROW(info.deserializeFromKindsBinary(in, ISerialization::KindSet::all()), Exception); +} + +TEST(SerializationInfoBinary, AcceptsDetachedOverSparse) +{ + /// COMBINATION encoding of {Default, Sparse, Detached}. + const char kinds[] = {5, 3, 0, 1, 2}; + ReadBufferFromMemory in(kinds, sizeof(kinds)); + + SerializationInfo info({ISerialization::Kind::DEFAULT}, defaultSettings()); + info.deserializeFromKindsBinary(in, ISerialization::KindSet::all()); + + ISerialization::KindStack expected{ISerialization::Kind::DEFAULT, ISerialization::Kind::SPARSE, ISerialization::Kind::DETACHED}; + EXPECT_EQ(info.getKindStack(), expected); +} + +/// The full stack a writer can build, with every kind in its canonical position. +TEST(SerializationInfoBinary, AcceptsDetachedOverReplicatedOverSparse) +{ + /// COMBINATION encoding of {Default, Sparse, Replicated, Detached}. + const char kinds[] = {5, 4, 0, 1, 3, 2}; + ReadBufferFromMemory in(kinds, sizeof(kinds)); + + SerializationInfo info({ISerialization::Kind::DEFAULT}, defaultSettings()); + info.deserializeFromKindsBinary(in, ISerialization::KindSet::all()); + + ISerialization::KindStack expected{ + ISerialization::Kind::DEFAULT, + ISerialization::Kind::SPARSE, + ISerialization::Kind::REPLICATED, + ISerialization::Kind::DETACHED}; + EXPECT_EQ(info.getKindStack(), expected); +} + } diff --git a/src/Formats/NativeReader.cpp b/src/Formats/NativeReader.cpp index 9d26ddb52a4f0..53f7f911f0ba2 100644 --- a/src/Formats/NativeReader.cpp +++ b/src/Formats/NativeReader.cpp @@ -36,8 +36,12 @@ namespace ErrorCodes } -NativeReader::NativeReader(ReadBuffer & istr_, UInt64 server_revision_, std::optional format_settings_) - : istr(istr_), server_revision(server_revision_), format_settings(format_settings_) +NativeReader::NativeReader( + ReadBuffer & istr_, + UInt64 server_revision_, + std::optional format_settings_, + ISerialization::KindSet allowed_kinds_) + : istr(istr_), server_revision(server_revision_), format_settings(format_settings_), allowed_kinds(allowed_kinds_) { } @@ -225,7 +229,7 @@ Block NativeReader::read() UInt8 has_custom = 0; readBinary(has_custom, istr); if (has_custom) - info->deserializeFromKindsBinary(istr); + info->deserializeFromKindsBinary(istr, allowed_kinds); serialization = column.type->getSerialization(*info); auto new_column = column.type->createColumn(*serialization); diff --git a/src/Formats/NativeReader.h b/src/Formats/NativeReader.h index ddf4b2142564c..390efaed15925 100644 --- a/src/Formats/NativeReader.h +++ b/src/Formats/NativeReader.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -24,8 +25,17 @@ class CompressedReadBufferFromFile; class NativeReader { public: + /// Kinds accepted from the peer unless the caller declares its own set. `Detached` is excluded + /// because it yields a column that does not match the declared type, which few callers handle. + static constexpr ISerialization::KindSet default_allowed_kinds + = ISerialization::KindSet::all().without(ISerialization::Kind::DETACHED); + /// If a non-zero server_revision is specified, additional block information may be expected and read. - NativeReader(ReadBuffer & istr_, UInt64 server_revision_, std::optional format_settings_ = std::nullopt); + NativeReader( + ReadBuffer & istr_, + UInt64 server_revision_, + std::optional format_settings_ = std::nullopt, + ISerialization::KindSet allowed_kinds_ = default_allowed_kinds); /// For cases when data structure (header) is known in advance. /// NOTE We may use header for data validation and/or type conversions. It is not implemented. @@ -62,6 +72,7 @@ class NativeReader UInt64 server_revision; std::optional format_settings = std::nullopt; BlockMissingValues * block_missing_values = nullptr; + ISerialization::KindSet allowed_kinds = default_allowed_kinds; bool use_index = false; IndexForNativeFormat::Blocks::const_iterator index_block_it; diff --git a/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python b/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python index 4ecd4becd513e..048ef838a2e0a 100644 --- a/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python +++ b/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python @@ -436,10 +436,8 @@ def indexRowCountMismatchScenario(): def sendSparseOverReplicatedBlock(s): - # A Sparse-over-Replicated kind stack makes SerializationSparse hand its values column - # (a fresh ColumnSparse always seeds one default value) to the nested Replicated - # deserialization, which then hits the "non-empty column" check. That check is decided - # by the wire-supplied kind stack and must be INCORRECT_DATA, not LOGICAL_ERROR. + # Sparse can sit inside Replicated but not the other way round, so this kind stack is rejected as + # out of order. A wire-supplied kind stack must produce INCORRECT_DATA, not LOGICAL_ERROR. END_OF_GRANULE_FLAG = 1 << 62 limit = 4 @@ -474,7 +472,7 @@ def sparseOverReplicatedScenario(): s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) connectAndStartInsert(s) sendSparseOverReplicatedBlock(s) - expectIncorrectData(s, "Reading into non-empty column ColumnReplicated") + expectIncorrectData(s, "is out of order") print("sparse over replicated: caught expected exception INCORRECT_DATA") diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python new file mode 100755 index 0000000000000..cd620dfafefab --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Regression test for a post-auth type confusion over the native TCP protocol. +# +# The serialization kind of every column of a block is chosen by the peer that sends the +# block. `Kind::DETACHED` is an internal transport representation for parallel blocks +# marshalling: it produces a `ColumnBLOB` holding raw peer-supplied bytes while the block +# keeps the declared type of the column. A client that selects it for an external table +# therefore puts a column that is not a `ColumnMap` into a block declared as `Map`, and +# every later `assert_cast` (a plain `static_cast` in release builds) +# reinterprets attacker-controlled bytes. +# +# `NativeReader` must accept `DETACHED` only from a peer whose pipeline unmarshalls blobs, +# which never includes a client, and must reject kind stacks that no writer can produce. +# All scenarios below expect INCORRECT_DATA (code 117) and a server that stays alive. +# +# The client speaks the raw native protocol pinned to revision 54482 +# (DBMS_MIN_REVISION_WITH_REPLICATED_SERIALIZATION), so the set of exchanged, +# revision-gated fields is fixed and does not drift. + +import os +import socket +import struct +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "detached serialization poc" + +REVISION = 54482 + +# KindStackBinarySerializationType +KIND_STACK_DEFAULT = 0 +KIND_STACK_DETACHED = 2 +KIND_STACK_DETACHED_OVER_SPARSE = 3 +KIND_STACK_COMBINATION = 5 + +# ISerialization::Kind +KIND_DEFAULT = 0 +KIND_SPARSE = 1 +KIND_DETACHED = 2 +KIND_REPLICATED = 3 + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) # database + writeStringBinary("default", ba) # user + writeStringBinary("", ba) # password + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version (>= 54471) + readStringBinary(s) # timezone (>= 54058) + readStringBinary(s) # display name (>= 54372) + readVarUInt(s) # version patch (>= 54401) + readStringBinary(s) # proto_send chunked (>= 54470) + readStringBinary(s) # proto_recv chunked (>= 54470) + for _ in range(readVarUInt(s)): # password complexity rules (>= 54461) + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce, UInt64 (>= 54462) + # Server settings in STRINGS_WITH_FLAGS format: (name, flags, value)* terminated + # by an empty name (>= 54474). + while True: + if readStringBinary(s) == "": # setting name + break + readVarUInt(s) # flags + readStringBinary(s) # value + readVarUInt(s) # query plan serialization version (>= 54477) + readVarUInt(s) # cluster function protocol version (>= 54479) + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key (>= 54458) + writeStringBinary("notchunked", ba) # proto_send chunked (>= 54470) + writeStringBinary("notchunked", ba) # proto_recv chunked (>= 54470) + writeVarUInt(0, ba) # parallel replicas protocol version (>= 54471) + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds (>= 54449) + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key (>= 54060) + writeVarUInt(0, ba) # distributed_depth (>= 54448) + writeVarUInt(1, ba) # client version patch (>= 54401) + ba.append(0) # opentelemetry: no trace id (>= 54442) + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator (>= 54453) + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number (>= 54475) + writeVarUInt(0, ba) # script line number (>= 54475) + ba.append(0) # jwt: none (>= 54476) + + +def sendQuery(s, query): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings + writeStringBinary("", ba) # interserver externally granted roles (>= 54472) + writeStringBinary("", ba) # interserver secret (>= 54441) + writeVarUInt(2, ba) # stage = Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator (>= 54459) + s.sendall(ba) + + +def serializeBlockInfo(ba): + writeVarUInt(1, ba) # field 1 + ba.append(0) # is_overflows = false + writeVarUInt(2, ba) # field 2 + ba.extend(struct.pack(" inside mapConcat. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + receiveHello(s) + sendAddendum(s) + sendQuery(s, "SELECT mapConcat(x) FROM e") + s.sendall(block) + expectIncorrectData(s, must_contain) + print("{}: caught expected exception INCORRECT_DATA".format(name)) + + +def main(): + # The reported vector: a Map column whose runtime representation becomes a ColumnBLOB. + runScenario( + "detached", + externalTableBlock("Map(UInt64, UInt64)", bytes([KIND_STACK_DETACHED])), + "Unexpected serialization kind Detached", + ) + + # The same, reached through the dedicated Detached-over-Sparse encoding. + runScenario( + "detached over sparse", + externalTableBlock("UInt64", bytes([KIND_STACK_DETACHED_OVER_SPARSE])), + "Unexpected serialization kind Detached", + ) + + # The same, reached through the generic kind stack encoding. + runScenario( + "detached in a combination", + externalTableBlock("Map(UInt64, UInt64)", combinationKindStack([KIND_DEFAULT, KIND_DETACHED])), + "Unexpected serialization kind Detached", + ) + + # Kind stacks are per tuple element, so a blob can also be requested for a nested column. + tuple_block = externalTableBlock("Tuple(UInt64, UInt64)", bytes([KIND_STACK_DEFAULT])) + tuple_block += bytes([KIND_STACK_DETACHED]) # element 0 + tuple_block += bytes([KIND_STACK_DEFAULT]) # element 1 + runScenario("detached tuple element", tuple_block, "Unexpected serialization kind Detached") + + # A repeated kind nests a column inside its own layout: a ColumnSparse whose values column is + # itself sparse never materializes back to a full column of the declared type. + runScenario( + "repeated kind", + externalTableBlock("UInt64", combinationKindStack([KIND_DEFAULT, KIND_SPARSE, KIND_SPARSE])), + "is out of order", + ) + + # Sparse can sit inside Replicated but not the other way round, and ColumnSparse over + # ColumnReplicated does not materialize back to a full column of the declared type either. + runScenario( + "kinds in the wrong order", + externalTableBlock("UInt64", combinationKindStack([KIND_DEFAULT, KIND_REPLICATED, KIND_SPARSE])), + "is out of order", + ) + + # Every kind stack starts with Default. + runScenario( + "kind stack without default", + externalTableBlock("UInt64", combinationKindStack([KIND_SPARSE])), + "must start with Default", + ) + + # A huge kind stack must be rejected before it is materialized into nested serializations. + huge = bytearray() + huge.append(KIND_STACK_COMBINATION) + writeVarUInt(1000000, huge) + runScenario("too many kinds", externalTableBlock("UInt64", bytes(huge)), "Too many serialization kinds") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference new file mode 100644 index 0000000000000..f362e10f25139 --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference @@ -0,0 +1,9 @@ +detached: caught expected exception INCORRECT_DATA +detached over sparse: caught expected exception INCORRECT_DATA +detached in a combination: caught expected exception INCORRECT_DATA +detached tuple element: caught expected exception INCORRECT_DATA +repeated kind: caught expected exception INCORRECT_DATA +kinds in the wrong order: caught expected exception INCORRECT_DATA +kind stack without default: caught expected exception INCORRECT_DATA +too many kinds: caught expected exception INCORRECT_DATA +server alive 45 diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh new file mode 100755 index 0000000000000..55eae4a886457 --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# We should have correct env vars from shell_config.sh to run this test +python3 "$CUR_DIR"/04936_detached_serialization_from_client_native_protocol.python + +# The kinds a client may legitimately select still work, and the server is still alive. +$CLICKHOUSE_CLIENT -q "SELECT 'server alive', sum(number) FROM numbers(10)" From 2bacbce4a28a6ae5dc2c9d46847dd78d08704468 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 16:01:11 +0000 Subject: [PATCH 052/296] Backport #116335 to 26.8: Check for timeouts and KILL QUERY while covering a polygon with H3 cells --- .../functions/regular-functions/geo/h3.mdx | 14 +- src/Functions/h3GeometryToCells.cpp | 432 ++++++++++++++++++ src/Functions/h3GeometryToCells.h | 27 ++ src/Functions/h3PolygonToCells.cpp | 178 +------- .../h3PolygonToCellsWithContainment.cpp | 153 +------ .../03754_h3_polygon_to_cells_const.reference | 18 +- ...ls_with_containment_cancellation.reference | 3 + ..._to_cells_with_containment_cancellation.sh | 40 ++ ...h3_polygon_to_cells_cancellation.reference | 8 + .../05043_h3_polygon_to_cells_cancellation.sh | 52 +++ ...gon_to_cells_sliver_cancellation.reference | 2 + ...h3_polygon_to_cells_sliver_cancellation.sh | 34 ++ 12 files changed, 643 insertions(+), 318 deletions(-) create mode 100644 src/Functions/h3GeometryToCells.cpp create mode 100644 src/Functions/h3GeometryToCells.h create mode 100644 tests/queries/0_stateless/05042_h3_polygon_to_cells_with_containment_cancellation.reference create mode 100755 tests/queries/0_stateless/05042_h3_polygon_to_cells_with_containment_cancellation.sh create mode 100644 tests/queries/0_stateless/05043_h3_polygon_to_cells_cancellation.reference create mode 100755 tests/queries/0_stateless/05043_h3_polygon_to_cells_cancellation.sh create mode 100644 tests/queries/0_stateless/05046_h3_polygon_to_cells_sliver_cancellation.reference create mode 100755 tests/queries/0_stateless/05046_h3_polygon_to_cells_sliver_cancellation.sh diff --git a/docs/reference/functions/regular-functions/geo/h3.mdx b/docs/reference/functions/regular-functions/geo/h3.mdx index 6130d22faba51..77016b0de76fb 100644 --- a/docs/reference/functions/regular-functions/geo/h3.mdx +++ b/docs/reference/functions/regular-functions/geo/h3.mdx @@ -304,9 +304,11 @@ SELECT arrayJoin(h3kRing(644325529233966508, 1)) AS h3index; └────────────────────┘ ``` -## h3PolygonToCells +## h3PolygonToCells {#h3polygontocells} -Returns the hexagons (at specified resolution) contained by the provided geometry, either ring or (multi-)polygon. +Returns the hexagons (at specified resolution) contained by the provided geometry, either ring or (multi-)polygon. A hexagon is contained when the geometry contains its center, which is containment mode `0` of [h3PolygonToCellsWithContainment](#h3polygontocellswithcontainment). + +Every vertex of the geometry must be on the sphere: longitude in the range `[-180, 180]` and latitude in `[-90, 90]` degrees. The order of the returned cells is not guaranteed, so sort the array when the order matters. **Syntax** @@ -334,12 +336,12 @@ SELECT h3PolygonToCells([(-122.4089866999972145,37.813318999983238),(-122.354473 ``` text title="Response" ┌────────────h3index─┐ -│ 608692970769612799 │ -│ 608692971927240703 │ │ 608692970585063423 │ -│ 608692970819944447 │ │ 608692970719281151 │ │ 608692970752835583 │ +│ 608692970769612799 │ +│ 608692970819944447 │ +│ 608692971927240703 │ │ 608692972027903999 │ └────────────────────┘ ``` @@ -348,6 +350,8 @@ SELECT h3PolygonToCells([(-122.4089866999972145,37.813318999983238),(-122.354473 Returns the hexagons (at specified resolution) covering the provided geometry, either ring or (multi-)polygon, using H3 experimental containment modes. +Every vertex of the geometry must be on the sphere: longitude in the range `[-180, 180]` and latitude in `[-90, 90]` degrees. The order of the returned cells is not guaranteed, so sort the array when the order matters. + This function maps `flags` to H3 containment modes: - `0` — `CONTAINMENT_CENTER` diff --git a/src/Functions/h3GeometryToCells.cpp b/src/Functions/h3GeometryToCells.cpp new file mode 100644 index 0000000000000..24d7ea0f3fd21 --- /dev/null +++ b/src/Functions/h3GeometryToCells.cpp @@ -0,0 +1,432 @@ +#include + +#if USE_H3 + +#include +#include +#include + +#include +#include +/// H3's polygon internals: private to the library, and not declared for C++. The candidate walk below +/// is H3's own, so it needs the same primitives. +extern "C" +{ +#include +#include +#include +#include +#include +#include +} + +static constexpr size_t MAX_ARRAY_SIZE = 1 << 30; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int TOO_LARGE_ARRAY_SIZE; + extern const int INCORRECT_DATA; +} + +namespace +{ + +/// Number of cells the size estimate aims to fit in the polygon's bounding box, `MAX_SIZE_CELL_THRESHOLD` +/// in H3's `polyfill.c`. +constexpr double MAX_SIZE_CELL_THRESHOLD = 10; +/// The coordinate range `latLngToCell` accepts, `VALID_RANGE_BBOX` in H3's `polyfill.c`. +constexpr BBox VALID_RANGE_BBOX = {M_PI_2, -M_PI_2, M_PI, -M_PI}; + +SphericalPointInRadians toRadianPoint(const SphericalPoint & degree_point) +{ + Float64 lon = get<0>(degree_point) * M_PI / 180.0; + Float64 lat = get<1>(degree_point) * M_PI / 180.0; + return SphericalPointInRadians(lon, lat); +} + +/// A vertex off the sphere makes H3's bounding box, and the area it derives from it, meaningless: the size +/// estimate then counts every cell of a polygon covering much of the globe, at the requested resolution. +/// NaN and infinity are already rejected when the geometry is converted; this is the range they leave open. +void validateCoordinates(const SphericalPoint & degree_point, std::string_view function_name) +{ + const Float64 lon = get<0>(degree_point); + const Float64 lat = get<1>(degree_point); + + if (!std::isfinite(lon) || !std::isfinite(lat) || std::abs(lon) > 180.0 || std::abs(lat) > 90.0) + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The vertex ({}, {}) of the geometry passed to function {} is out of bounds " + "(longitude must be -180..180 and latitude -90..90 degrees)", + toString(lon), + toString(lat), + function_name); +} + +void checkH3Error(H3Error error, std::string_view function_name) +{ + if (error != E_SUCCESS) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Failed to compute H3 polygon to cells in function {}: {}", + function_name, describeH3Error(error)); +} + +/// The next cell in the sequence of all cells to check, `nextCell` in H3's `polyfill.c`. +H3Index nextCellInSequence(H3Index cell) +{ + int res = H3_GET_RESOLUTION(cell); + while (true) + { + /// A base cell has no parent, so continue with the next one. + if (res == 0) + return baseCellNumToCell(H3_GET_BASE_CELL(cell) + 1); + + H3Index parent = cell; + H3_SET_RESOLUTION(parent, res - 1); + H3_SET_INDEX_DIGIT(parent, res, H3_DIGIT_MASK); + + /// Not the last sibling: continue with the next one, skipping the missing child of a pentagon. + const int digit = static_cast(H3_GET_INDEX_DIGIT(cell, res)); + if (digit < INVALID_DIGIT - 1) + { + H3_SET_INDEX_DIGIT(cell, res, digit + ((isPentagon(parent) && digit == CENTER_DIGIT) ? 2 : 1)); + return cell; + } + + --res; + cell = parent; + } +} + +/// The candidate walk of `iterStepPolygonCompact` in H3's `polyfill.c`, over the same H3 primitives, with a +/// cancellation checkpoint per candidate. H3 wraps the walk in one call, which can reject millions of +/// candidates before it returns a cell, leaving no place to observe `max_execution_time` or `KILL QUERY`. +/// Keep in sync with H3 when the submodule is bumped. +class CompactCellScan +{ +public: + CompactCellScan( + const GeoPolygon * polygon_, + const BBox * bboxes_, + int res_, + UInt32 flags_, + std::string_view function_name_, + CancellationBudget & budget_, + size_t units_per_candidate_) + : polygon(polygon_) + , bboxes(bboxes_) + , res(res_) + , mode(static_cast(FLAG_GET_CONTAINMENT_MODE(flags_))) + , function_name(function_name_) + , budget(budget_) + , units_per_candidate(units_per_candidate_) + { + } + + /// The next cell covering the polygon, coarser than the target resolution when all of its children are + /// covered, or `H3_NULL` once the polygon is exhausted. + H3Index next() + { + if (cell == H3_NULL) + return H3_NULL; + + /// The first candidate is the cell the walk starts at; after that, the one after the last output. + if (started) + cell = nextCellInSequence(cell); + else + started = true; + + /// Nothing covers a polygon without vertices. + if (polygon->geoloop.numVerts == 0) + cell = H3_NULL; + + while (cell) + { + budget.chargeUnits(units_per_candidate); + + const int cell_res = H3_GET_RESOLUTION(cell); + + /// Target resolution: a fine-grained check. + if (cell_res == res && matchesContainmentMode(cell, cell_res)) + return cell; + + /// Coarser cell: check the bounding box of all of its children. + if (cell_res < res) + { + BBox bbox; + checkH3Error(cellToBBox(cell, &bbox, true), function_name); + + if (bboxOverlapsBBox(&bboxes[0], &bbox)) + { + /// Quick check for possible containment, then the expensive one on the polygon. + if (bboxContainsBBox(&bboxes[0], &bbox)) + { + CellBoundary bbox_boundary = bboxToCellBoundary(&bbox); + if (cellBoundaryInsidePolygon(polygon, bboxes, &bbox_boundary, &bbox)) + return cell; + } + + /// An intersecting bounding box means every child has to be tested. + H3Index child = H3_NULL; + checkH3Error(cellToCenterChild(cell, cell_res + 1, &child), function_name); + cell = child; + continue; + } + } + + cell = nextCellInSequence(cell); + } + + return H3_NULL; + } + +private: + bool matchesContainmentMode(H3Index candidate, int candidate_res) + { + if (mode == CONTAINMENT_CENTER || mode == CONTAINMENT_OVERLAPPING || mode == CONTAINMENT_OVERLAPPING_BBOX) + { + LatLng center; + checkH3Error(cellToLatLng(candidate, ¢er), function_name); + if (pointInsidePolygon(polygon, bboxes, ¢er)) + return true; + } + + if (mode == CONTAINMENT_OVERLAPPING || mode == CONTAINMENT_OVERLAPPING_BBOX) + { + /// The polygon may be wholly contained by the cell, which its first vertex tells us. That + /// vertex exists because the caller rejects a polygon without vertices. + const LatLng first_vertex = polygon->geoloop.verts[0]; + /// Out-of-range coordinates yield false positives with `latLngToCell`. + if (bboxContains(&VALID_RANGE_BBOX, &first_vertex)) + { + H3Index polygon_cell = H3_NULL; + checkH3Error(latLngToCell(&first_vertex, candidate_res, &polygon_cell), function_name); + if (polygon_cell == candidate) + return true; + } + } + + if (mode == CONTAINMENT_FULL || mode == CONTAINMENT_OVERLAPPING || mode == CONTAINMENT_OVERLAPPING_BBOX) + { + CellBoundary boundary; + checkH3Error(cellToBoundary(candidate, &boundary), function_name); + BBox bbox; + checkH3Error(cellToBBox(candidate, &bbox, false), function_name); + + if ((mode == CONTAINMENT_FULL || mode == CONTAINMENT_OVERLAPPING_BBOX) + && cellBoundaryInsidePolygon(polygon, bboxes, &boundary, &bbox)) + return true; + + /// Center point inclusion was checked above, so overlap only needs line intersection. + if ((mode == CONTAINMENT_OVERLAPPING || mode == CONTAINMENT_OVERLAPPING_BBOX) + && cellBoundaryCrossesPolygon(polygon, bboxes, &boundary, &bbox)) + return true; + } + + if (mode == CONTAINMENT_OVERLAPPING_BBOX) + { + /// A bounding box containing all the cell's children, so that this works for the size estimate. + BBox bbox; + checkH3Error(cellToBBox(candidate, &bbox, true), function_name); + + if (bboxOverlapsBBox(&bboxes[0], &bbox)) + { + CellBoundary bbox_boundary = bboxToCellBoundary(&bbox); + if (bboxContainsBBox(&bbox, &bboxes[0]) + || pointInsidePolygon(polygon, bboxes, &bbox_boundary.verts[0]) + || cellBoundaryCrossesPolygon(polygon, bboxes, &bbox_boundary, &bbox)) + return true; + } + } + + return false; + } + + const GeoPolygon * polygon; + const BBox * bboxes; + const int res; + const ContainmentMode mode; + const std::string_view function_name; + CancellationBudget & budget; + const size_t units_per_candidate; + + H3Index cell = baseCellNumToCell(0); + bool started = false; +}; + +/// `maxPolygonToCellsSizeExperimental` in H3's `polyfill.c`, over the walk above so that it is checkpointed +/// too: a rough upper bound from the polygon's bounding box, counted at a resolution coarse enough for the +/// box to hold about `MAX_SIZE_CELL_THRESHOLD` cells. +Int64 estimateCellCount( + const GeoPolygon * polygon, + const BBox * bboxes, + int res, + std::string_view function_name, + CancellationBudget & budget, + size_t units_per_candidate) +{ + if (polygon->geoloop.numVerts == 0) + return 0; + + /// A (very) rough area of the polygon's bounding box. + const BBox & polygon_bbox = bboxes[0]; + const double polygon_bbox_area_km2 = bboxHeightRads(&polygon_bbox) * bboxWidthRads(&polygon_bbox) + / cos(fmin(fabs(polygon_bbox.north), fabs(polygon_bbox.south))) * EARTH_RADIUS_KM * EARTH_RADIUS_KM; + + /// All this needs is a general order of magnitude of the number of cells that would fit. + int estimate_res = res; + while (estimate_res > 0) + { + double average_cell_area_km2 = 0; + checkH3Error(getHexagonAreaAvgKm2(estimate_res - 1, &average_cell_area_km2), function_name); + if (polygon_bbox_area_km2 / average_cell_area_km2 <= MAX_SIZE_CELL_THRESHOLD) + break; + --estimate_res; + } + + /// Ignore the requested containment mode and use the faster overlapping-bbox one. + CompactCellScan scan( + polygon, bboxes, estimate_res, CONTAINMENT_OVERLAPPING_BBOX, function_name, budget, units_per_candidate); + + Int64 estimate = 0; + while (H3Index cell = scan.next()) + { + Int64 children_size = 0; + checkH3Error(cellToChildrenSize(cell, res, &children_size), function_name); + estimate += children_size; + } + + return estimate; +} + +LatLng toH3LatLng(const SphericalPointInRadians & point) +{ + LatLng result; + result.lat = point.get<1>(); + result.lng = point.get<0>(); + return result; +} + +class GeoPolygonContainer +{ +private: + VectorWithMemoryTracking mainLoopVerts; + VectorWithMemoryTracking> holeVerts; + + mutable GeoLoop mutableMainLoop{}; + mutable GeoPolygon mutablePolygon{}; + mutable VectorWithMemoryTracking mutableHoles; + +public: + explicit GeoPolygonContainer(VectorWithMemoryTracking && mainLoop, + VectorWithMemoryTracking> && holes = {}) + : mainLoopVerts(std::move(mainLoop)), holeVerts(std::move(holes)) {} + + const GeoPolygon * unwrap() const + { + mutableMainLoop = {static_cast(mainLoopVerts.size()), + const_cast(mainLoopVerts.data())}; + + mutableHoles.clear(); + mutableHoles.reserve(holeVerts.size()); + for (const auto & hole : holeVerts) + mutableHoles.push_back({static_cast(hole.size()), + const_cast(hole.data())}); + + mutablePolygon = {mutableMainLoop, + static_cast(mutableHoles.size()), + mutableHoles.data()}; + return &mutablePolygon; + } +}; + +} + +void appendH3Cells( + const SphericalMultiPolygon & multi_polygon, + UInt8 resolution, + UInt32 flags, + std::string_view function_name, + CancellationBudget & budget, + ColumnUInt64 & dst_data) +{ + const size_t row_start_offset = dst_data.size(); + for (const auto & polygon : multi_polygon) + { + VectorWithMemoryTracking exterior; + exterior.reserve(polygon.outer().size()); + for (const auto & point : polygon.outer()) + { + validateCoordinates(point, function_name); + exterior.push_back(toH3LatLng(toRadianPoint(point))); + } + + size_t vertex_count = exterior.size(); + + VectorWithMemoryTracking> holes; + holes.reserve(polygon.inners().size()); + for (const auto & inner : polygon.inners()) + { + VectorWithMemoryTracking hole; + hole.reserve(inner.size()); + for (const auto & point : inner) + { + validateCoordinates(point, function_name); + hole.push_back(toH3LatLng(toRadianPoint(point))); + } + vertex_count += hole.size(); + holes.emplace_back(std::move(hole)); + } + + GeoPolygonContainer polygon_wrapper(std::move(exterior), std::move(holes)); + const GeoPolygon * geo_polygon = polygon_wrapper.unwrap(); + + checkH3Error(validatePolygonFlags(flags), function_name); + + /// Bounding boxes of the polygon and of each of its holes, which every candidate check needs. + VectorWithMemoryTracking bboxes(geo_polygon->numHoles + 1); + bboxesFromGeoPolygon(geo_polygon, bboxes.data()); + + /// A candidate at the target resolution is tested against every vertex of the polygon. + const size_t units_per_candidate = 1 + vertex_count; + + /// Cheap bbox-based estimate: rejects a polygon that cannot fit before enumerating it. Rough in + /// both directions, hence the exact check per cell below. + const size_t row_size_so_far = dst_data.size() - row_start_offset; + const size_t estimate = static_cast( + estimateCellCount(geo_polygon, bboxes.data(), resolution, function_name, budget, units_per_candidate)); + if (estimate > MAX_ARRAY_SIZE || row_size_so_far + estimate > MAX_ARRAY_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "The result of function {} (array of {} elements) will be too large with resolution = {}", + function_name, row_size_so_far + estimate, toString(resolution)); + + CompactCellScan scan( + geo_polygon, bboxes.data(), resolution, flags, function_name, budget, units_per_candidate); + + while (H3Index compact_cell = scan.next()) + { + /// A cell coarser than the requested resolution stands for all of its children. + IterCellsChildren children = iterInitParent(compact_cell, resolution); + for (; children.h; iterStepChild(&children)) + { + /// The candidates rejected on the way here are charged by the walk; a child of a covered + /// cell only costs the index arithmetic that produced it. + budget.charge(sizeof(H3Index)); + + if (dst_data.size() - row_start_offset >= MAX_ARRAY_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "The result of function {} (array of {} elements) will be too large with resolution = {}", + function_name, dst_data.size() - row_start_offset + 1, toString(resolution)); + + /// `PODArray::push_back` grows geometrically; `IColumn::reserve` would size it exactly. + dst_data.getData().push_back(children.h); + } + } + } +} + +} + +#endif diff --git a/src/Functions/h3GeometryToCells.h b/src/Functions/h3GeometryToCells.h new file mode 100644 index 0000000000000..d0224ad6b20e6 --- /dev/null +++ b/src/Functions/h3GeometryToCells.h @@ -0,0 +1,27 @@ +#pragma once + +#include "config.h" + +#if USE_H3 + +#include +#include +#include + +namespace DB +{ + +/// Appends the H3 cells covering `multi_polygon` to `dst_data`, charging `budget` for every candidate cell +/// examined so that it can check for a timeout or `KILL QUERY` mid-row, including while the search rejects +/// candidates without producing any. `flags` is an H3 containment mode, 0 being CONTAINMENT_CENTER. +void appendH3Cells( + const SphericalMultiPolygon & multi_polygon, + UInt8 resolution, + UInt32 flags, + std::string_view function_name, + CancellationBudget & budget, + ColumnUInt64 & dst_data); + +} + +#endif diff --git a/src/Functions/h3PolygonToCells.cpp b/src/Functions/h3PolygonToCells.cpp index 40b0b97481419..3b8aabeca6e3b 100644 --- a/src/Functions/h3PolygonToCells.cpp +++ b/src/Functions/h3PolygonToCells.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -13,16 +14,11 @@ #include #include +#include #include -#include -#include -#include -#include +#include #include -#include - -static constexpr size_t MAX_ARRAY_SIZE = 1 << 30; namespace DB { @@ -30,31 +26,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; - extern const int TOO_LARGE_ARRAY_SIZE; extern const int ILLEGAL_COLUMN; - extern const int QUERY_WAS_CANCELLED; - extern const int INCORRECT_DATA; -} - -namespace -{ - -SphericalPointInRadians toRadianPoint(const SphericalPoint & degree_point) -{ - Float64 lon = get<0>(degree_point) * M_PI / 180.0; - Float64 lat = get<1>(degree_point) * M_PI / 180.0; - - return SphericalPointInRadians(lon, lat); -} - -LatLng toH3LatLng(const SphericalPointInRadians & point) -{ - LatLng result; - result.lat = point.get<1>(); - result.lng = point.get<0>(); - return result; -} - } /// Takes a geometry (Ring, Polygon or MultiPolygon) and returns an array of H3 hexagons that cover this geometry. @@ -77,11 +49,11 @@ class FunctionH3PolygonToCells final : public IFunction ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - /// Resolved from the executing thread rather than captured: this instance can be stored in table - /// metadata and then run by any later query. - QueryStatusPtr process_list_element; - if (auto query_context = CurrentThread::tryGetQueryContext()) - process_list_element = query_context->getProcessListElementSafe(); + /// One polygon can expand to hundreds of millions of cells, so the executor's between-blocks check + /// cannot bound this call. Resolved from the executing thread rather than captured: this instance + /// can be stored in table metadata and then run by any later query. + const std::function check_cancellation = makeCancellationCheck(name); + CancellationBudget budget(check_cancellation); const bool is_const_geometry = isColumnConst(*arguments[0].column); @@ -107,14 +79,13 @@ class FunctionH3PolygonToCells final : public IFunction arguments[1].column->getName(), getName()); const auto & data_resolution = col_resolution->getData(); + const String function_name = getName(); auto dst_data_column = ColumnUInt64::create(); auto dst_offsets_column = ColumnArray::ColumnOffsets::create(input_rows_count); auto & dst_data = *dst_data_column; auto & dst_offsets = dst_offsets_column->getData(); - size_t current_offset = 0; - callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { using TypeConverter = std::decay_t; @@ -155,9 +126,6 @@ class FunctionH3PolygonToCells final : public IFunction if (is_const_geometry) const_multi_polygon = to_multi_polygon(std::move(geometries[0])); - /// Reuse buffer across rows to avoid repeated allocations - VectorWithMemoryTracking hindex_vec; - for (size_t row = 0; row < input_rows_count; ++row) { UInt8 resolution = data_resolution[row]; @@ -173,140 +141,20 @@ class FunctionH3PolygonToCells final : public IFunction row_multi_polygon = to_multi_polygon(std::move(geometries[row])); const SphericalMultiPolygon & multi_polygon = is_const_geometry ? const_multi_polygon : row_multi_polygon; - if (process_list_element && process_list_element->isKilled()) - throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); - - const size_t row_start_offset = current_offset; - for (const auto & polygon : multi_polygon) - { - VectorWithMemoryTracking exterior; - exterior.reserve(polygon.outer().size()); - for (const auto & point : polygon.outer()) - exterior.push_back(toH3LatLng(toRadianPoint(point))); - - VectorWithMemoryTracking> holes; - holes.reserve(polygon.inners().size()); - for (const auto & inner : polygon.inners()) - { - VectorWithMemoryTracking hole; - hole.reserve(inner.size()); - for (const auto & point : inner) - hole.push_back(toH3LatLng(toRadianPoint(point))); - - holes.emplace_back(std::move(hole)); - } - - GeoPolygonContainer polygon_wrapper(std::move(exterior), std::move(holes)); - - int64_t polygon_size = 0; - H3Error size_err = maxPolygonToCellsSize(polygon_wrapper.unwrap(), resolution, 0, &polygon_size); - if (size_err != E_SUCCESS) - throw Exception( - ErrorCodes::INCORRECT_DATA, - "Failed to estimate H3 polygon to cells size in function {}: {}", - getName(), describeH3Error(size_err)); - - const size_t vec_size = static_cast(polygon_size); - const size_t row_size_so_far = current_offset - row_start_offset; - if (vec_size > MAX_ARRAY_SIZE || row_size_so_far + vec_size > MAX_ARRAY_SIZE) - throw Exception( - ErrorCodes::TOO_LARGE_ARRAY_SIZE, - "The result of function {} (array of {} elements) will be too large with resolution argument = {}", - getName(), row_size_so_far + vec_size, toString(resolution)); - - hindex_vec.assign(vec_size, 0); - H3Error fill_err = polygonToCells(polygon_wrapper.unwrap(), resolution, 0, hindex_vec.data()); - if (fill_err != E_SUCCESS) - throw Exception( - ErrorCodes::INCORRECT_DATA, - "Failed to compute H3 polygon to cells in function {}: {}", - getName(), describeH3Error(fill_err)); - - /// Go through PODArray::reserve: it grows capacity geometrically, IColumn::reserve sizes it exactly. - dst_data.getData().reserve(dst_data.size() + vec_size); - for (auto hindex : hindex_vec) - { - if (hindex != 0) - { - ++current_offset; - dst_data.insert(hindex); - } - } - } - - if (current_offset - row_start_offset > MAX_ARRAY_SIZE) - throw Exception( - ErrorCodes::TOO_LARGE_ARRAY_SIZE, - "The result of function {} (array of {} elements) will be too large with resolution argument = {}", - getName(), current_offset - row_start_offset, toString(resolution)); - - dst_offsets[row] = current_offset; + /// CONTAINMENT_CENTER: the rule `polygonToCells` documents for itself. + appendH3Cells(multi_polygon, resolution, 0, function_name, budget, dst_data); + dst_offsets[row] = dst_data.size(); } }); return ColumnArray::create(std::move(dst_data_column), std::move(dst_offsets_column)); } - -private: - class GeoPolygonContainer - { - private: - // Store the polygon data - VectorWithMemoryTracking mainLoopVerts; - VectorWithMemoryTracking> holeVerts; - - // Temporary storage for C-style structs - mutable GeoLoop mutableMainLoop{}; - mutable GeoPolygon mutablePolygon{}; - mutable VectorWithMemoryTracking mutableHoles; - - public: - // Constructor to create from C++ data - explicit GeoPolygonContainer( - VectorWithMemoryTracking && mainLoop, - VectorWithMemoryTracking> && holes = {}) - : mainLoopVerts(std::move(mainLoop)), holeVerts(std::move(holes)) {} - - // Method to get C-style GeoPolygon pointer - const GeoPolygon * unwrap() const - { - // Prepare main loop - mutableMainLoop = { - static_cast(mainLoopVerts.size()), - const_cast(mainLoopVerts.data()) - }; - - // Prepare holes - mutableHoles.clear(); - mutableHoles.reserve(holeVerts.size()); - for (const auto& hole : holeVerts) - { - mutableHoles.push_back({ - static_cast(hole.size()), - const_cast(hole.data()) - }); - } - - // Prepare full polygon - mutablePolygon = { - mutableMainLoop, - static_cast(mutableHoles.size()), - mutableHoles.data() - }; - - return &mutablePolygon; - } - - // Additional utility methods - size_t size() const { return mainLoopVerts.size(); } - bool empty() const { return mainLoopVerts.empty(); } - }; }; REGISTER_FUNCTION(H3PolygonToCells) { factory.registerFunction(FunctionDocumentation{ - .description="Returns the hexagons (at specified resolution) contained by the provided geometry, either ring or (multi-)polygon.", + .description="Returns the hexagons (at specified resolution) contained by the provided geometry, either ring or (multi-)polygon. Every vertex must be on the sphere: longitude in -180..180 and latitude in -90..90 degrees. The order of the returned cells is not guaranteed.", .syntax = "h3PolygonToCells(geometry, resolution)", .introduced_in = {25, 11}, .category = FunctionDocumentation::Category::Geo}); diff --git a/src/Functions/h3PolygonToCellsWithContainment.cpp b/src/Functions/h3PolygonToCellsWithContainment.cpp index 27969c18ab83c..a2164d6033769 100644 --- a/src/Functions/h3PolygonToCellsWithContainment.cpp +++ b/src/Functions/h3PolygonToCellsWithContainment.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -16,48 +17,26 @@ #include #include #include +#include #include -#include -#include +#include #include -#include -#include #include #include -static constexpr size_t MAX_ARRAY_SIZE = 1 << 30; - namespace DB { namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; - extern const int TOO_LARGE_ARRAY_SIZE; extern const int ILLEGAL_COLUMN; - extern const int QUERY_WAS_CANCELLED; - extern const int INCORRECT_DATA; } namespace { -SphericalPointInRadians toRadianPoint(const SphericalPoint & degree_point) -{ - Float64 lon = get<0>(degree_point) * M_PI / 180.0; - Float64 lat = get<1>(degree_point) * M_PI / 180.0; - return SphericalPointInRadians(lon, lat); -} - -LatLng toH3LatLng(const SphericalPointInRadians & point) -{ - LatLng result; - result.lat = point.get<1>(); - result.lng = point.get<0>(); - return result; -} - [[noreturn]] void throwInvalidContainmentFlag(Int64 flags, std::string_view function_name) { throw Exception( @@ -103,11 +82,11 @@ class Functionh3PolygonToCellsWithContainment : public IFunction ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - /// Resolved from the executing thread rather than captured: this instance can be stored in table - /// metadata and then run by any later query. - QueryStatusPtr process_list_element; - if (auto query_context = CurrentThread::tryGetQueryContext()) - process_list_element = query_context->getProcessListElementSafe(); + /// One polygon can expand to hundreds of millions of cells, so the executor's between-blocks check + /// cannot bound this call. Resolved from the executing thread rather than captured: this instance + /// can be stored in table metadata and then run by any later query. + const std::function check_cancellation = makeCancellationCheck(name); + CancellationBudget budget(check_cancellation); const bool is_const_geometry = isColumnConst(*arguments[0].column); @@ -144,7 +123,7 @@ class Functionh3PolygonToCellsWithContainment : public IFunction toString(MAX_H3_RES)); } - /// H3 polygonToCellsExperimental expects uint32_t flags. + /// H3's containment mode is a `uint32_t` flag mask. /// Fast path: UInt8 literals (0..3) and UInt32 columns; otherwise cast to Int64 and validate 0..3. /// All flag values are validated before geometry conversion. auto col_flags_materialized = arguments[2].column->convertToFullColumnIfConst(); @@ -189,9 +168,6 @@ class Functionh3PolygonToCellsWithContainment : public IFunction auto & dst_data = *dst_data_column; auto & dst_offsets = dst_offsets_column->getData(); - size_t current_offset = 0; - - callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { using TypeConverter = std::decay_t; @@ -230,8 +206,6 @@ class Functionh3PolygonToCellsWithContainment : public IFunction if (is_const_geometry) const_multi_polygon = to_multi_polygon(std::move(geometries[0])); - VectorWithMemoryTracking hindex_vec; - for (size_t row = 0; row < input_rows_count; ++row) { const UInt8 resolution = data_resolution[row]; @@ -247,114 +221,13 @@ class Functionh3PolygonToCellsWithContainment : public IFunction const SphericalMultiPolygon & multi_polygon = is_const_geometry ? const_multi_polygon : row_multi_polygon; - if (process_list_element && process_list_element->isKilled()) - throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); - - const size_t row_start_offset = current_offset; - for (const auto & polygon : multi_polygon) - { - VectorWithMemoryTracking exterior; - exterior.reserve(polygon.outer().size()); - for (const auto & point : polygon.outer()) - exterior.push_back(toH3LatLng(toRadianPoint(point))); - - VectorWithMemoryTracking> holes; - holes.reserve(polygon.inners().size()); - for (const auto & inner : polygon.inners()) - { - VectorWithMemoryTracking hole; - hole.reserve(inner.size()); - for (const auto & point : inner) - hole.push_back(toH3LatLng(toRadianPoint(point))); - holes.emplace_back(std::move(hole)); - } - - GeoPolygonContainer polygon_wrapper(std::move(exterior), std::move(holes)); - - int64_t polygon_size = 0; - H3Error size_err = maxPolygonToCellsSizeExperimental( - polygon_wrapper.unwrap(), resolution, flags, &polygon_size); - if (size_err != E_SUCCESS) - throw Exception(ErrorCodes::INCORRECT_DATA, - "Failed to estimate H3 polygon to cells size in function {}: {}", - getName(), describeH3Error(size_err)); - - const size_t vec_size = static_cast(polygon_size); - const size_t row_size_so_far = current_offset - row_start_offset; - if (vec_size > MAX_ARRAY_SIZE || row_size_so_far + vec_size > MAX_ARRAY_SIZE) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, - "The result of function {} (array of {} elements) will be too large with resolution = {}", - getName(), row_size_so_far + vec_size, toString(resolution)); - - hindex_vec.assign(vec_size, 0); - H3Error fill_err = polygonToCellsExperimental( - polygon_wrapper.unwrap(), - resolution, - flags, - static_cast(vec_size), - hindex_vec.data()); - if (fill_err != E_SUCCESS) - throw Exception(ErrorCodes::INCORRECT_DATA, - "Failed to compute H3 polygon to cells in function {}: {}", - getName(), describeH3Error(fill_err)); - - /// Go through PODArray::reserve: it grows capacity geometrically, IColumn::reserve sizes it exactly. - dst_data.getData().reserve(dst_data.size() + vec_size); - for (auto hindex : hindex_vec) - { - if (hindex != 0) - { - ++current_offset; - dst_data.insert(hindex); - } - } - } - - if (current_offset - row_start_offset > MAX_ARRAY_SIZE) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, - "The result of function {} (array of {} elements) will be too large with resolution = {}", - getName(), current_offset - row_start_offset, toString(resolution)); - - dst_offsets[row] = current_offset; + appendH3Cells(multi_polygon, resolution, flags, function_name, budget, dst_data); + dst_offsets[row] = dst_data.size(); } }); return ColumnArray::create(std::move(dst_data_column), std::move(dst_offsets_column)); } - -private: - class GeoPolygonContainer - { - private: - VectorWithMemoryTracking mainLoopVerts; - VectorWithMemoryTracking> holeVerts; - - mutable GeoLoop mutableMainLoop{}; - mutable GeoPolygon mutablePolygon{}; - mutable VectorWithMemoryTracking mutableHoles; - - public: - explicit GeoPolygonContainer(VectorWithMemoryTracking && mainLoop, - VectorWithMemoryTracking> && holes = {}) - : mainLoopVerts(std::move(mainLoop)), holeVerts(std::move(holes)) {} - - const GeoPolygon * unwrap() const - { - mutableMainLoop = {static_cast(mainLoopVerts.size()), - const_cast(mainLoopVerts.data())}; - - mutableHoles.clear(); - mutableHoles.reserve(holeVerts.size()); - for (const auto & hole : holeVerts) - mutableHoles.push_back({static_cast(hole.size()), - const_cast(hole.data())}); - - mutablePolygon = {mutableMainLoop, - static_cast(mutableHoles.size()), - mutableHoles.data()}; - return &mutablePolygon; - } - }; }; REGISTER_FUNCTION(h3PolygonToCellsWithContainment) @@ -365,7 +238,9 @@ REGISTER_FUNCTION(h3PolygonToCellsWithContainment) "using H3's experimental algorithm with selectable containment mode. " "Flags: 0=CONTAINMENT_CENTER, 1=CONTAINMENT_FULL, 2=CONTAINMENT_OVERLAPPING, " "3=CONTAINMENT_OVERLAPPING_BBOX. The flags argument is passed to the H3 API as UInt32; " - "use integer literals (0..3) or toUInt32. Other native integer types are converted with an accurate cast. See H3 docs.", + "use integer literals (0..3) or toUInt32. Other native integer types are converted with an accurate cast. " + "Every vertex must be on the sphere: longitude in -180..180 and latitude in -90..90 degrees. " + "The order of the returned cells is not guaranteed. See H3 docs.", .syntax = "h3PolygonToCellsWithContainment(geometry, resolution, flags)", .introduced_in = {26, 6}, .category = FunctionDocumentation::Category::Geo}); diff --git a/tests/queries/0_stateless/03754_h3_polygon_to_cells_const.reference b/tests/queries/0_stateless/03754_h3_polygon_to_cells_const.reference index 9ef0d61ba3565..2e477ec1feac8 100644 --- a/tests/queries/0_stateless/03754_h3_polygon_to_cells_const.reference +++ b/tests/queries/0_stateless/03754_h3_polygon_to_cells_const.reference @@ -1,21 +1,21 @@ -[608692970769612799,608692971927240703,608692970585063423,608692970819944447,608692970719281151,608692970752835583,608692972027903999] 1 -[608692970769612799,608692971927240703,608692970585063423,608692970819944447,608692970719281151,608692970752835583,608692972027903999] 1 -[608692970769612799,608692971927240703,608692970585063423,608692970819944447,608692970719281151,608692970752835583,608692972027903999] 1 -[608692970769612799,608692971927240703,608692970585063423,608692970819944447,608692970719281151,608692970752835583,608692972027903999] 1 +[608692970585063423,608692970719281151,608692970752835583,608692970769612799,608692970819944447,608692971927240703,608692972027903999] 1 +[608692970585063423,608692970719281151,608692970752835583,608692970769612799,608692970819944447,608692971927240703,608692972027903999] 1 +[608692970585063423,608692970719281151,608692970752835583,608692970769612799,608692970819944447,608692971927240703,608692972027903999] 1 +[608692970585063423,608692970719281151,608692970752835583,608692970769612799,608692970819944447,608692971927240703,608692972027903999] 1 0 [] 1 [] 2 [] 3 [] 4 [] 5 [] -6 [604189372417310719,604189371209351167,604189371075133439] -7 [608692970769612799,608692971927240703,608692970585063423,608692970819944447,608692970719281151,608692970752835583,608692972027903999] -[608692970769612799,608692971927240703,608692970585063423,608692970819944447,608692970719281151,608692970752835583,608692972027903999] 1 +6 [604189371075133439,604189371209351167,604189372417310719] +7 [608692970585063423,608692970719281151,608692970752835583,608692970769612799,608692970819944447,608692971927240703,608692972027903999] +[608692970585063423,608692970719281151,608692970752835583,608692970769612799,608692970819944447,608692971927240703,608692972027903999] 1 0 [] 1 [] 2 [] 3 [] 4 [] 5 [] -6 [604189372417310719,604189371209351167,604189371075133439] -7 [608692970769612799,608692971927240703,608692970585063423,608692970819944447,608692970719281151,608692970752835583,608692972027903999] +6 [604189371075133439,604189371209351167,604189372417310719] +7 [608692970585063423,608692970719281151,608692970752835583,608692970769612799,608692970819944447,608692971927240703,608692972027903999] diff --git a/tests/queries/0_stateless/05042_h3_polygon_to_cells_with_containment_cancellation.reference b/tests/queries/0_stateless/05042_h3_polygon_to_cells_with_containment_cancellation.reference new file mode 100644 index 0000000000000..a290b1c58163c --- /dev/null +++ b/tests/queries/0_stateless/05042_h3_polygon_to_cells_with_containment_cancellation.reference @@ -0,0 +1,3 @@ +out of bounds (longitude must be -180..180 and latitude -90..90 degrees) +stopped at the deadline +292 1 diff --git a/tests/queries/0_stateless/05042_h3_polygon_to_cells_with_containment_cancellation.sh b/tests/queries/0_stateless/05042_h3_polygon_to_cells_with_containment_cancellation.sh new file mode 100755 index 0000000000000..ed18397917fd0 --- /dev/null +++ b/tests/queries/0_stateless/05042_h3_polygon_to_cells_with_containment_cancellation.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the H3 library is not built in fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A vertex off the sphere used to reach H3 as it came, and its size estimate then counted every cell of a +# polygon covering much of the globe: about 45 minutes in one call, observing neither the deadline nor +# `KILL QUERY`. The oracle is the outer `timeout`. + +timeout 30 ${CLICKHOUSE_CLIENT} --query " + SELECT sum(length(h3PolygonToCellsWithContainment([(materialize(-122.40898669999721), -1.), (2., 100.0001), (-1., -1.7976931348623157e308)], 9, 0))) FROM numbers(200) + " 2>&1 | grep -o -m 1 -F 'out of bounds (longitude must be -180..180 and latitude -90..90 degrees)' + +# Valid vertices can still ask for more work than the deadline allows: every candidate cell is tested +# against every vertex, so the comb below - 20000 vertices inside half a degree - costs 7.5 s in a single +# row. The cells are now enumerated one at a time, so the deadline is observed mid-row. The oracle is the +# outer `timeout`, which is why the query must not depend on how fast the build is. + +COMB="arrayConcat([(0., 0.)], arrayMap(i -> (0.5 + 0.4 * (i % 2), i * 0.00002), range(20000)), [(0., 0.4)])" + +if timeout 30 ${CLICKHOUSE_CLIENT} --max_execution_time 1 --query " + SELECT length(h3PolygonToCellsWithContainment(${COMB}, 10, 0)) + " 2>&1 | grep -q -F 'TIMEOUT_EXCEEDED' +then + echo 'stopped at the deadline' +else + echo 'still running after 30 seconds' +fi + +# The checkpoint must not change what the function returns. + +${CLICKHOUSE_CLIENT} --query " + SELECT + length(h3PolygonToCellsWithContainment([(-122.4089866999972145, 37.813318999983238), (-122.3544736999993603, 37.7198061999978478), (-122.4798767000009008, 37.8151571999998453)], 9, 0)), + arraySort(h3PolygonToCellsWithContainment([(-122.4089866999972145, 37.813318999983238), (-122.3544736999993603, 37.7198061999978478), (-122.4798767000009008, 37.8151571999998453)], 7, 2)) + = arraySort(h3PolygonToCellsWithContainment([(materialize(-122.4089866999972145), 37.813318999983238), (-122.3544736999993603, 37.7198061999978478), (-122.4798767000009008, 37.8151571999998453)], 7, 2)) +" diff --git a/tests/queries/0_stateless/05043_h3_polygon_to_cells_cancellation.reference b/tests/queries/0_stateless/05043_h3_polygon_to_cells_cancellation.reference new file mode 100644 index 0000000000000..a8426675f1ba0 --- /dev/null +++ b/tests/queries/0_stateless/05043_h3_polygon_to_cells_cancellation.reference @@ -0,0 +1,8 @@ +stopped at the deadline +out of bounds (longitude must be -180..180 and latitude -90..90 degrees) +Point's component must not be NaN +Point's component must not be infinite +5 1 1 +7 1 1 +9 1 1 +11 1 1 diff --git a/tests/queries/0_stateless/05043_h3_polygon_to_cells_cancellation.sh b/tests/queries/0_stateless/05043_h3_polygon_to_cells_cancellation.sh new file mode 100755 index 0000000000000..f6dfaed04d30e --- /dev/null +++ b/tests/queries/0_stateless/05043_h3_polygon_to_cells_cancellation.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the H3 library is not built in fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every candidate cell is tested against every vertex, so the comb below - 20000 valid vertices inside half +# a degree - took 29 s in a single row, with nothing observing the deadline until the row was done. The +# oracle is the outer `timeout`: a `TIMEOUT_EXCEEDED` arriving 29 s after a 1 s deadline still fails, and +# the margin holds on a sanitizer build because the unbounded version grows with it. + +COMB="arrayConcat([(0., 0.)], arrayMap(i -> (0.5 + 0.4 * (i % 2), i * 0.00002), range(20000)), [(0., 0.4)])" + +if timeout 30 ${CLICKHOUSE_CLIENT} --max_execution_time 1 --query " + SELECT length(h3PolygonToCells(${COMB}, 10)) + " 2>&1 | grep -q -F 'TIMEOUT_EXCEEDED' +then + echo 'stopped at the deadline' +else + echo 'still running after 30 seconds' +fi + +# A vertex off the sphere is rejected instead of reaching H3, which derives a meaningless bounding box. + +${CLICKHOUSE_CLIENT} --query " + SELECT length(h3PolygonToCells([(-122.40898669999721, -1.), (2., 100.0001), (-1., -1.7976931348623157e308)], 9)) + " 2>&1 | grep -o -m 1 -F 'out of bounds (longitude must be -180..180 and latitude -90..90 degrees)' + +# A NaN vertex never reaches that check: converting the geometry rejects it first, together with infinity. + +${CLICKHOUSE_CLIENT} --query "SELECT length(h3PolygonToCells([(nan, 0.), (1., 0.), (1., 1.)], 9))" 2>&1 | + grep -o -m 1 -F "Point's component must not be NaN" + +${CLICKHOUSE_CLIENT} --query "SELECT length(h3PolygonToCells([(inf, 0.), (1., 0.), (1., 1.)], 9))" 2>&1 | + grep -o -m 1 -F "Point's component must not be infinite" + +# The cells covering a geometry are those whose center it contains, which is containment mode 0, so the two +# functions agree. + +${CLICKHOUSE_CLIENT} --query " + WITH + [(-122.4089866999972145, 37.813318999983238), (-122.3544736999993603, 37.7198061999978478), (-122.4798767000009008, 37.8151571999998453)] AS sf, + [(55.66824, 12.595493), (55.667901, 12.593991), (55.667474, 12.595117), (55.66824, 12.595493)] AS cph + SELECT + resolution, + arraySort(h3PolygonToCells(sf, resolution)) = arraySort(h3PolygonToCellsWithContainment(sf, resolution, 0)), + arraySort(h3PolygonToCells(cph, resolution)) = arraySort(h3PolygonToCellsWithContainment(cph, resolution, 0)) + FROM (SELECT arrayJoin([toUInt8(5), 7, 9, 11]) AS resolution) + ORDER BY resolution +" diff --git a/tests/queries/0_stateless/05046_h3_polygon_to_cells_sliver_cancellation.reference b/tests/queries/0_stateless/05046_h3_polygon_to_cells_sliver_cancellation.reference new file mode 100644 index 0000000000000..7350d390d4329 --- /dev/null +++ b/tests/queries/0_stateless/05046_h3_polygon_to_cells_sliver_cancellation.reference @@ -0,0 +1,2 @@ +stopped at the deadline +stopped at the deadline diff --git a/tests/queries/0_stateless/05046_h3_polygon_to_cells_sliver_cancellation.sh b/tests/queries/0_stateless/05046_h3_polygon_to_cells_sliver_cancellation.sh new file mode 100755 index 0000000000000..3d8cca9ac7693 --- /dev/null +++ b/tests/queries/0_stateless/05046_h3_polygon_to_cells_sliver_cancellation.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the H3 library is not built in fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A needle-thin polygon whose bounding box covers much of the globe. The search descends into every cell of +# that box and the needle contains the center of almost none of them, so a checkpoint per returned cell +# observes nothing and the query used to run for minutes past the deadline. The oracle is the outer +# `timeout`, whose margin holds on a sanitizer build because the unchecked version grows with it too. + +NEEDLE="[(-100., -60.), (100., 60.), (-99.9999999, -60.)]" + +if timeout 30 ${CLICKHOUSE_CLIENT} --max_execution_time 1 --query " + SELECT length(h3PolygonToCells(${NEEDLE}, 8)) + " 2>&1 | grep -q -F 'TIMEOUT_EXCEEDED' +then + echo 'stopped at the deadline' +else + echo 'still running after 30 seconds' +fi + +# The same shape with a containment mode, where a candidate cell is also tested for overlap. + +if timeout 30 ${CLICKHOUSE_CLIENT} --max_execution_time 1 --query " + SELECT length(h3PolygonToCellsWithContainment(${NEEDLE}, 8, 1)) + " 2>&1 | grep -q -F 'TIMEOUT_EXCEEDED' +then + echo 'stopped at the deadline' +else + echo 'still running after 30 seconds' +fi From c64d3c6c3aa995fae818db86a032c2f0a75cd0b7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 17:51:06 +0000 Subject: [PATCH 053/296] Backport #120764 to 26.8: Do not throw from `ReplxxLineReader` destructor when the terminal is gone --- src/Client/ReplxxLineReader.cpp | 30 ++++++-- src/Server/SSH/SSHPtyHandler.cpp | 31 +++++++- tests/integration/test_ssh/test.py | 115 +++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 7 deletions(-) diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 583883b5bd871..37af946759846 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "config.h" // USE_SKIM @@ -737,12 +738,31 @@ bool ReplxxLineReader::hintChosen() ReplxxLineReader::~ReplxxLineReader() { - if (history_file_fd >= 0 && close(history_file_fd)) - rx.print("Close of history file failed: %s\n", errnoToString().c_str()); + /// `Replxx::print` may fail with `std::runtime_error("write failed")` when e.g. the pty of the embedded + /// SSH client is gone already. A destructor is implicitly `noexcept`, so letting anything escape from + /// here would `std::terminate` the whole process. + try + { + if (history_file_fd >= 0 && close(history_file_fd)) + rx.print("Close of history file failed: %s\n", errnoToString().c_str()); - /// Reset cursor blinking - if (overwrite_mode) - rx.print("%s", "\033[0 q"); + /// Reset cursor blinking + if (overwrite_mode) + rx.print("%s", "\033[0 q"); + } + catch (...) + { + /// The reporting path must not be able to escape either: the `const char *` overload of + /// `tryLogCurrentException` builds a `String` for the logger name and calls `getLogger` + /// before it reaches its own `try`, so under memory pressure it can throw as well. + try + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + catch (...) // NOLINT(bugprone-empty-catch) Ok: reporting failed, nothing more to do + { + } + } } LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt) diff --git a/src/Server/SSH/SSHPtyHandler.cpp b/src/Server/SSH/SSHPtyHandler.cpp index 02ecb110d3c78..8edc1d4771000 100644 --- a/src/Server/SSH/SSHPtyHandler.cpp +++ b/src/Server/SSH/SSHPtyHandler.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -528,6 +529,18 @@ void SSHPtyHandler::run() } bool fds_set = false; + /// `ssh_event_add_fd` allocates a wrapper that is released only by `ssh_event_remove_fd`: + /// `ssh_event_free` does not know about it. The fds therefore have to be removed on every exit + /// path, including when `poll` throws because the connection with the client is already gone. + int registered_out_fd = -1; + int registered_err_fd = -1; + SCOPE_EXIT({ + if (registered_out_fd != -1) + event.removeFd(registered_out_fd); + if (registered_err_fd != -1) + event.removeFd(registered_err_fd); + }); + do { /* Poll the main event which takes care of the session, the channel and @@ -544,10 +557,16 @@ void SSHPtyHandler::run() /* If stdout valid, add stdout to be monitored by the poll event. */ if (sdata.channel_callback->client_input_output.out != -1) + { event.addFd(sdata.channel_callback->client_input_output.out, POLLIN, process_stdout, sdata.channel_callback->channel.getCChannelPtr()); + registered_out_fd = sdata.channel_callback->client_input_output.out; + } if (sdata.channel_callback->client_input_output.err != -1) + { event.addFd(sdata.channel_callback->client_input_output.err, POLLIN, process_stderr, sdata.channel_callback->channel.getCChannelPtr()); + registered_err_fd = sdata.channel_callback->client_input_output.err; + } } while (sdata.channel_callback->channel.isOpen() && !sdata.channel_callback->hasClientFinished() && !server.isCancelled()); @@ -558,8 +577,16 @@ void SSHPtyHandler::run() sdata.channel_callback->channel.isOpen(), sdata.channel_callback->hasClientFinished(), server.isCancelled() ); - event.removeFd(sdata.channel_callback->client_input_output.out); - event.removeFd(sdata.channel_callback->client_input_output.err); + if (registered_out_fd != -1) + { + event.removeFd(registered_out_fd); + registered_out_fd = -1; + } + if (registered_err_fd != -1) + { + event.removeFd(registered_err_fd); + registered_err_fd = -1; + } /// Drain any remaining data from stdout/stderr pipes before closing the channel. /// The client may have finished writing to the pipes before the event loop had a chance diff --git a/tests/integration/test_ssh/test.py b/tests/integration/test_ssh/test.py index 5fa570bfe6c08..350aa0023645f 100644 --- a/tests/integration/test_ssh/test.py +++ b/tests/integration/test_ssh/test.py @@ -1,6 +1,7 @@ import os import re import socket +import struct import subprocess import time @@ -495,3 +496,117 @@ def test_interactive_tab_completion_respects_session_user(started_cluster): instance.query("DROP USER IF EXISTS completer") instance.query("DROP TABLE IF EXISTS default.visible_completion_target") instance.query("DROP TABLE IF EXISTS default.hidden_completion_target") + + +def test_interactive_session_torn_down_with_a_dead_pty(started_cluster): + """Losing the pty while the embedded client is shutting down must not kill the server. + + `ReplxxLineReader::~ReplxxLineReader` writes an escape sequence to the + terminal to reset cursor blinking when overwrite mode was ever enabled. + `Replxx::print` throws `std::runtime_error("write failed")` when that write + does not go through, and a destructor is implicitly `noexcept`, so the + exception used to `std::terminate` the whole server process. + + Reproduce it the way a real disconnect does: turn overwrite mode on with + the `Insert` key, then drop the TCP connection with a RST so that the + server's side of the pty is gone by the time the line reader is destroyed. + """ + # The daemon watchdog restarts the server after `std::terminate`, so "the + # server answers queries again" is not evidence of anything, and neither is + # a growing `uptime()`: on a fresh cluster the uptime before the disconnect + # is only a few seconds, so a restarted server reaches a larger value well + # within the sampling window below. Pin the process identity instead: the + # watchdog restarts the server by forking a new child, so the set of + # `clickhouse-server` pids in the container changes and cannot recover. + # The pattern is anchored at argv0 so that the shell running `pgrep` (whose + # own command line contains the pattern) does not match itself. + def server_pids(): + return instance.exec_in_container( + ["bash", "-c", "pgrep -f '^[^ ]*clickhouse(-| )server' | sort -n"], + nothrow=True, + ).split() + + pids_before = server_pids() + assert pids_before, "no `clickhouse-server` process in the container" + uptime_before = float(instance.query("SELECT uptime()").strip()) + + pkey = paramiko.Ed25519Key.from_private_key_file(f"{SCRIPT_DIR}/keys/lucy_ed25519") + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + hostname=instance.ip_address, + port=9022, + username="lucy", + pkey=pkey, + timeout=30, + ) + try: + channel = client.invoke_shell(term="xterm", width=80, height=24) + channel.settimeout(20) + output = _read_channel_until(channel, timeout=20, marker=":) ") + assert ":) " in output, f"no prompt from the embedded client: {output!r}" + + # `Insert` toggles overwrite mode, which is what makes the destructor + # print the "reset cursor blinking" sequence in the first place. Wait + # for the raw `\033[5 q` ("blinking cursor") escape the key handler + # prints: it is the only observable proof that the server really + # consumed the key and that `overwrite_mode` became true. Without it + # the destructor writes nothing and the test would pass even unfixed. + channel.sendall("\x1b[2~") + raw = b"" + deadline = time.time() + 10 + while time.time() < deadline and b"\x1b[5 q" not in raw: + if channel.recv_ready(): + raw += channel.recv(65536) + else: + time.sleep(0.05) + assert ( + b"\x1b[5 q" in raw + ), f"overwrite mode was not enabled, the destructor would write nothing: {raw!r}" + + # Abort the connection with a RST instead of a graceful shutdown, so + # writes on the server side fail rather than being silently discarded. + sock = client.get_transport().sock + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + sock.close() + finally: + client.close() + + # The session teardown is asynchronous, so keep sampling for a while. A + # sample that fails only means the teardown is still in flight, but the + # *last* sample of the window must succeed and must still show the very + # same process: the server that survived the disconnect, not a fresh one. + last_failure = None + last_uptime = None + deadline = time.time() + 30 + while time.time() < deadline: + try: + uptime = float(instance.query("SELECT uptime()").strip()) + except Exception as e: # down or restarting — the next sample decides + last_failure = e + last_uptime = None + time.sleep(0.5) + continue + last_uptime = uptime + pids = server_pids() + assert pids == pids_before, ( + "the server process was replaced after the SSH disconnect, i.e. it " + "died while tearing the session down and was started again " + f"(pids {pids} != {pids_before})" + ) + assert uptime >= uptime_before, ( + "the server restarted after the SSH disconnect, i.e. it died while " + f"tearing the session down (uptime {uptime} < {uptime_before})" + ) + time.sleep(0.5) + + assert last_uptime is not None, ( + "the server did not answer a query at the end of the 30 s window after " + f"the SSH disconnect, i.e. it died while tearing the session down: {last_failure}" + ) + + # `from_host=True` also greps the rotated logs, so a restart cannot hide the + # fatal line by rotating it out of the active `clickhouse-server.log`. + assert not instance.contains_in_log( + "std::terminate", from_host=True + ), "the server called `std::terminate` while tearing down the SSH session" From 03c5fd680390bbc0d2f68dc66b682f25c0c53ea5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 21 Sep 2026 20:27:35 +0000 Subject: [PATCH 054/296] Backport #120202 to 26.8: Retry `poll` on `EINTR` in the vendored `nats-io` client --- contrib/nats-io | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/nats-io b/contrib/nats-io index 3e3a3f10572ed..b88704dfeb55e 160000 --- a/contrib/nats-io +++ b/contrib/nats-io @@ -1 +1 @@ -Subproject commit 3e3a3f10572ed9ff1b7ad17873881877cb62f301 +Subproject commit b88704dfeb55e0f1dee165b9c1daccb90d5816d4 From 314d8c4c278dc987fc2444cd81f76424a88ee990 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 05:47:42 +0000 Subject: [PATCH 055/296] Backport #120182 to 26.8: Fix wrong results for `plus`/`minus` keys with a non-integer operand --- src/Functions/FunctionHelpers.cpp | 23 ++++ src/Functions/FunctionHelpers.h | 6 ++ .../Optimizations/actionsDAGUtils.cpp | 15 +-- ...3_injective_plus_minus_type_gate.reference | 95 +++++++++++++++++ .../04833_injective_plus_minus_type_gate.sql | 97 +++++++++++++++++ ...us_type_gate_sets_shards_windows.reference | 97 +++++++++++++++++ ...us_minus_type_gate_sets_shards_windows.sql | 97 +++++++++++++++++ ..._plus_minus_type_gate_plan_shape.reference | 98 +++++++++++++++++ ...ective_plus_minus_type_gate_plan_shape.sql | 100 ++++++++++++++++++ 9 files changed, 622 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference create mode 100644 tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql create mode 100644 tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference create mode 100644 tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql create mode 100644 tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference create mode 100644 tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index f5e46079539c9..5eaa0bf567a10 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -498,6 +498,29 @@ bool allArgumentColumnsAreConstant(const ColumnsWithTypeAndName & args) return true; } +bool plusMinusWithConstantsIsInjective( + const ColumnWithTypeAndName & left, const ColumnWithTypeAndName & right, const DataTypePtr & return_type) +{ + /// Two varying operands are not injective (`x + y` maps many pairs to one sum), and with both + /// fixed there is no varying argument to be injective in. + const bool left_is_const = left.column && isColumnConst(*left.column); + const bool right_is_const = right.column && isColumnConst(*right.column); + if (left_is_const == right_is_const) + return false; + + auto is_integer_type = [](const DataTypePtr & type) + { return type && isInteger(*removeNullable(recursiveRemoveLowCardinality(type))); }; + + if (!is_integer_type(left.type) || !is_integer_type(right.type) || !is_integer_type(return_type)) + return false; + + /// A NULL among the varying argument's values maps to NULL one-to-one, so only the fixed operand + /// matters. Its `ColumnConst` nests a column of size 1, so the value is readable even when the + /// constant itself was materialized with size 0, as query-plan constants are. + const ColumnWithTypeAndName & constant = left_is_const ? left : right; + return !constant.column->onlyNull(); +} + bool convertLowCardinalityColumnsToFull(ColumnsWithTypeAndName & args) { bool converted = false; diff --git a/src/Functions/FunctionHelpers.h b/src/Functions/FunctionHelpers.h index 8a77b9a3eda2f..9260ae9861d02 100644 --- a/src/Functions/FunctionHelpers.h +++ b/src/Functions/FunctionHelpers.h @@ -217,6 +217,12 @@ bool isLowCardinalityType(const IDataType & type); bool hasLowCardinalityTypes(const ColumnsWithTypeAndName & args); /// Returns true if all of the arguments have constant columns. bool allArgumentColumnsAreConstant(const ColumnsWithTypeAndName & args); +/// Whether `plus`/`minus` is injective in its varying argument, given the other one fixed. Only +/// integer arithmetic qualifies: integer wrap-around is a bijection, while every other operand class +/// collapses distinct arguments somewhere - an `Interval` at end-of-month days and DST transitions, a +/// float or `Decimal` by rounding or rescaling, a narrower date constant, a NULL constant. +bool plusMinusWithConstantsIsInjective( + const ColumnWithTypeAndName & left, const ColumnWithTypeAndName & right, const DataTypePtr & return_type); bool convertLowCardinalityColumnsToFull(ColumnsWithTypeAndName & args); void checkFunctionArgumentSizes(const ColumnsWithTypeAndName & arguments, size_t input_rows_count); diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp index fa04c059cff62..a4462d6004c3d 100644 --- a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -555,12 +556,14 @@ bool isInjectiveFunction(const ActionsDAG::Node * node) if (node->function_base->isInjective({})) return true; - size_t fixed_args = 0; - for (const auto & child : node->children) - if (child->type == ActionsDAG::ActionType::COLUMN) - ++fixed_args; - static const std::vector injective = {"plus", "minus", "negate", "tuple"}; - return (fixed_args + 1 >= node->children.size()) && (std::ranges::find(injective, node->function_base->getName()) != injective.end()); + const auto & name = node->function_base->getName(); + if (node->children.size() != 2 || (name != "plus" && name != "minus")) + return false; + + const auto & left = *node->children[0]; + const auto & right = *node->children[1]; + return plusMinusWithConstantsIsInjective( + {left.column, left.result_type, left.result_name}, {right.column, right.result_type, right.result_name}, node->result_type); } NodeSet removeInjectiveFunctionsFromResultsRecursively(const ActionsDAG & actions) diff --git a/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference new file mode 100644 index 0000000000000..48531a4780d35 --- /dev/null +++ b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference @@ -0,0 +1,95 @@ +-- { echo } + +-- --------------------------------------------------------------------------- +-- Correctness: per-partition evaluation must agree with the merged evaluation. Each arm +-- prints the forced count and then the merged count; they must be equal. The two settings +-- apply to the outer query, so they reach the aggregation under test. +-- --------------------------------------------------------------------------- + +-- Date + INTERVAL MONTH collapses the 29th, 30th and 31st of a month into one key. 2001 and +-- 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_month; +CREATE TABLE t_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS force_distinct_partitions_independently = 1; +2 +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS allow_distinct_partitions_independently = 0; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 0; +2 +DROP TABLE t_month; +-- the minus direction collapses the same way, going back into a shorter month +DROP TABLE IF EXISTS t_month_minus; +CREATE TABLE t_month_minus (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month_minus SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-03-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +DROP TABLE t_month_minus; +-- DateTime + INTERVAL DAY collapses across a DST spring-forward, so the interval kind alone +-- does not decide safety +DROP TABLE IF EXISTS t_dst; +CREATE TABLE t_dst (t DateTime('Europe/Moscow'), x UInt32) ENGINE = MergeTree ORDER BY t PARTITION BY t; +INSERT INTO t_dst SELECT toDateTime('2010-03-27 00:00:00', 'Europe/Moscow') + (intDiv(number, 100) * 1800) AS t, number FROM numbers_mt(600); +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +4 +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +4 +DROP TABLE t_dst; +-- a Float64 addend collapses by mantissa rounding; a float column reaches the optimizer +-- through a non-float partition key +DROP TABLE IF EXISTS t_float; +CREATE TABLE t_float (f Float64, x UInt32) ENGINE = MergeTree ORDER BY f PARTITION BY toUInt64(f - 1e16); +INSERT INTO t_float SELECT 1e16 + intDiv(number, 100) AS f, number FROM numbers_mt(600); +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +-- the collapse the arm above depends on: fewer distinct sums than distinct addends +SELECT uniqExact(f), uniqExact(f + 1.0) FROM t_float; +3 2 +DROP TABLE t_float; +-- a constant of a narrower date type narrows the result: every multiple of 65536 maps to +-- 1970-01-01. A NULL constant maps every value to NULL. +DROP TABLE IF EXISTS t_narrow; +CREATE TABLE t_narrow (x UInt32, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_narrow SELECT intDiv(number, 100) * 65536 AS x, number FROM numbers_mt(600); +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +1 +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +1 +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS force_distinct_partitions_independently = 1; +1 +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS allow_distinct_partitions_independently = 0; +1 +SELECT uniqExact(x), uniqExact(x + toDate(0)) FROM t_narrow; +6 1 +DROP TABLE t_narrow; +-- a Decimal constant rescales the varying operand: every multiple of 2^32 maps to one +-- Decimal(9, 1) +DROP TABLE IF EXISTS t_decimal; +CREATE TABLE t_decimal (x UInt64, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_decimal SELECT intDiv(number, 100) * 4294967296 AS x, number FROM numbers_mt(400); +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +1 +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +1 +SELECT uniqExact(x), uniqExact(x + toDecimal32(0, 1)) FROM t_decimal; +4 1 +DROP TABLE t_decimal; +-- two varying operands are not injective either: many pairs share one sum +DROP TABLE IF EXISTS t_two_cols; +CREATE TABLE t_two_cols (x UInt32, y UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_two_cols SELECT intDiv(number, 100) AS x, number % 100 AS y FROM numbers_mt(1000); +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +109 +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +109 +DROP TABLE t_two_cols; diff --git a/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql new file mode 100644 index 0000000000000..cf5290884f8db --- /dev/null +++ b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql @@ -0,0 +1,97 @@ +-- Tags: no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +-- The values and plans below are the analyzer's, so pin it. +SET enable_analyzer = 1; + +-- { echo } + +-- --------------------------------------------------------------------------- +-- Correctness: per-partition evaluation must agree with the merged evaluation. Each arm +-- prints the forced count and then the merged count; they must be equal. The two settings +-- apply to the outer query, so they reach the aggregation under test. +-- --------------------------------------------------------------------------- + +-- Date + INTERVAL MONTH collapses the 29th, 30th and 31st of a month into one key. 2001 and +-- 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_month; +CREATE TABLE t_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS force_distinct_partitions_independently = 1; +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS allow_distinct_partitions_independently = 0; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 1; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 0; +DROP TABLE t_month; + +-- the minus direction collapses the same way, going back into a shorter month +DROP TABLE IF EXISTS t_month_minus; +CREATE TABLE t_month_minus (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month_minus SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-03-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_month_minus; + +-- DateTime + INTERVAL DAY collapses across a DST spring-forward, so the interval kind alone +-- does not decide safety +DROP TABLE IF EXISTS t_dst; +CREATE TABLE t_dst (t DateTime('Europe/Moscow'), x UInt32) ENGINE = MergeTree ORDER BY t PARTITION BY t; +INSERT INTO t_dst SELECT toDateTime('2010-03-27 00:00:00', 'Europe/Moscow') + (intDiv(number, 100) * 1800) AS t, number FROM numbers_mt(600); +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_dst; + +-- a Float64 addend collapses by mantissa rounding; a float column reaches the optimizer +-- through a non-float partition key +DROP TABLE IF EXISTS t_float; +CREATE TABLE t_float (f Float64, x UInt32) ENGINE = MergeTree ORDER BY f PARTITION BY toUInt64(f - 1e16); +INSERT INTO t_float SELECT 1e16 + intDiv(number, 100) AS f, number FROM numbers_mt(600); +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +-- the collapse the arm above depends on: fewer distinct sums than distinct addends +SELECT uniqExact(f), uniqExact(f + 1.0) FROM t_float; +DROP TABLE t_float; + +-- a constant of a narrower date type narrows the result: every multiple of 65536 maps to +-- 1970-01-01. A NULL constant maps every value to NULL. +DROP TABLE IF EXISTS t_narrow; +CREATE TABLE t_narrow (x UInt32, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_narrow SELECT intDiv(number, 100) * 65536 AS x, number FROM numbers_mt(600); +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS force_distinct_partitions_independently = 1; +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS allow_distinct_partitions_independently = 0; +SELECT uniqExact(x), uniqExact(x + toDate(0)) FROM t_narrow; +DROP TABLE t_narrow; + +-- a Decimal constant rescales the varying operand: every multiple of 2^32 maps to one +-- Decimal(9, 1) +DROP TABLE IF EXISTS t_decimal; +CREATE TABLE t_decimal (x UInt64, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_decimal SELECT intDiv(number, 100) * 4294967296 AS x, number FROM numbers_mt(400); +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT uniqExact(x), uniqExact(x + toDecimal32(0, 1)) FROM t_decimal; +DROP TABLE t_decimal; + +-- two varying operands are not injective either: many pairs share one sum +DROP TABLE IF EXISTS t_two_cols; +CREATE TABLE t_two_cols (x UInt32, y UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_two_cols SELECT intDiv(number, 100) AS x, number % 100 AS y FROM numbers_mt(1000); +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_two_cols; diff --git a/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference new file mode 100644 index 0000000000000..d93a3042cfa81 --- /dev/null +++ b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference @@ -0,0 +1,97 @@ +-- { echo } + +-- per-partition set building reads the same predicate. The set fill deduplicates across +-- partitions anyway, so the merged answer stays correct and only the plan shape shows the +-- decline; the bare-key arm is the control that the fixture reaches the optimization. +-- 2001 and 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_set_month; +CREATE TABLE t_set_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +SYSTEM STOP MERGES t_set_month; +INSERT INTO t_set_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d + INTERVAL 1 MONTH FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +Pre-distinct: 1 +Read each partition through separate port: 1 +DROP TABLE t_set_month; +-- an integer key keeps per-partition set building; the partition key is a function of the set's +-- own output column, which the interval arm above cannot use because its key is the collapsing one +DROP TABLE IF EXISTS t_set_int; +CREATE TABLE t_set_int (a UInt32, b UInt32) ENGINE = MergeTree ORDER BY tuple() PARTITION BY a % 8; +SYSTEM STOP MERGES t_set_int; +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +Pre-distinct: 1 +Read each partition through separate port: 1 +SELECT (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 0) = (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1); +1 +DROP TABLE t_set_int; +-- --------------------------------------------------------------------------- +-- The distributed sharding-key consumer reaches the same predicate through its own rejection +-- loop and its own direct call, so it gets its own arms. Dropping the merge step is only +-- correct when the group key determines the shard: a key that collapses distinct shard-key +-- values leaves each shard's partial groups unmerged, so the same key is returned twice. +-- Each view filters itself by shardNum() so the two shards hold the disjoint rows the +-- declared key implies - a declared key alone does not redistribute rows on a read, and +-- without the filter every shard holds every row and even a sound merge drop doubles the +-- answer. The first arm of each pair counts merge steps (1 = kept, 0 = dropped) and the +-- second compares the answer against the unoptimized one; the integer pair is the control +-- that the optimization still fires where it is sound. +-- --------------------------------------------------------------------------- + +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY s ORDER BY s; +1 10 +2 20 +SELECT count() FROM (EXPLAIN SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +1 +SELECT (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +1 +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY s ORDER BY s; +1 15 +2 15 +SELECT count() FROM (EXPLAIN SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +0 +SELECT (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +1 +-- --------------------------------------------------------------------------- +-- The window consumer reaches the same predicate through the stream-disjointness +-- propagation, at two sites: the per-partition read request and the scatter skip above it. +-- INTERVAL MONTH collapses the 29th, 30th and 31st into one key, so one logical window +-- partition spans the table partitions those days live in and must not be evaluated per +-- table partition. The default arm carries no setting: the cost heuristic accepts this +-- fixture, so the answer has to be right without opting out. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_win_month; +CREATE TABLE t_win_month (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_win_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d FROM numbers(60); +-- the collapse the arms below depend on: more table partitions than window keys +SELECT uniqExact(_partition_id), uniqExact(d + INTERVAL 1 MONTH) FROM t_win_month; +6 2 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS force_window_partitions_independently = 1; +30 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +30 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c; +30 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +0 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +0 +DROP TABLE t_win_month; +-- the integer control: an injective addend keeps both sites firing, so the arms above +-- attribute to the operand type and not to the window shape or the fixture +DROP TABLE IF EXISTS t_win_int; +CREATE TABLE t_win_int (x UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x % 8; +INSERT INTO t_win_int SELECT number % 8 FROM numbers_mt(800); +SELECT uniqExact(_partition_id), uniqExact(x + 1) FROM t_win_int; +8 8 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS force_window_partitions_independently = 1; +100 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +100 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +1 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +1 +DROP TABLE t_win_int; diff --git a/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql new file mode 100644 index 0000000000000..509e8a3e3d5e7 --- /dev/null +++ b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql @@ -0,0 +1,97 @@ +-- Tags: distributed, no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +SET explain_query_plan_default = 'legacy'; +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +-- The values and plans below are the analyzer's, so pin it. +SET enable_analyzer = 1; + +-- { echo } + +-- per-partition set building reads the same predicate. The set fill deduplicates across +-- partitions anyway, so the merged answer stays correct and only the plan shape shows the +-- decline; the bare-key arm is the control that the fixture reaches the optimization. +-- 2001 and 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_set_month; +CREATE TABLE t_set_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +SYSTEM STOP MERGES t_set_month; +INSERT INTO t_set_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d + INTERVAL 1 MONTH FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_set_month; + +-- an integer key keeps per-partition set building; the partition key is a function of the set's +-- own output column, which the interval arm above cannot use because its key is the collapsing one +DROP TABLE IF EXISTS t_set_int; +CREATE TABLE t_set_int (a UInt32, b UInt32) ENGINE = MergeTree ORDER BY tuple() PARTITION BY a % 8; +SYSTEM STOP MERGES t_set_int; +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 0) = (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1); +DROP TABLE t_set_int; + +-- --------------------------------------------------------------------------- +-- The distributed sharding-key consumer reaches the same predicate through its own rejection +-- loop and its own direct call, so it gets its own arms. Dropping the merge step is only +-- correct when the group key determines the shard: a key that collapses distinct shard-key +-- values leaves each shard's partial groups unmerged, so the same key is returned twice. +-- Each view filters itself by shardNum() so the two shards hold the disjoint rows the +-- declared key implies - a declared key alone does not redistribute rows on a read, and +-- without the filter every shard holds every row and even a sound merge drop doubles the +-- answer. The first arm of each pair counts merge steps (1 = kept, 0 = dropped) and the +-- second compares the answer against the unoptimized one; the integer pair is the control +-- that the optimization still fires where it is sound. +-- --------------------------------------------------------------------------- + +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY s ORDER BY s; +SELECT count() FROM (EXPLAIN SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +SELECT (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY s ORDER BY s; +SELECT count() FROM (EXPLAIN SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +SELECT (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); + +-- --------------------------------------------------------------------------- +-- The window consumer reaches the same predicate through the stream-disjointness +-- propagation, at two sites: the per-partition read request and the scatter skip above it. +-- INTERVAL MONTH collapses the 29th, 30th and 31st into one key, so one logical window +-- partition spans the table partitions those days live in and must not be evaluated per +-- table partition. The default arm carries no setting: the cost heuristic accepts this +-- fixture, so the answer has to be right without opting out. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_win_month; +CREATE TABLE t_win_month (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_win_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d FROM numbers(60); +-- the collapse the arms below depend on: more table partitions than window keys +SELECT uniqExact(_partition_id), uniqExact(d + INTERVAL 1 MONTH) FROM t_win_month; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS force_window_partitions_independently = 1; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +DROP TABLE t_win_month; + +-- the integer control: an injective addend keeps both sites firing, so the arms above +-- attribute to the operand type and not to the window shape or the fixture +DROP TABLE IF EXISTS t_win_int; +CREATE TABLE t_win_int (x UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x % 8; +INSERT INTO t_win_int SELECT number % 8 FROM numbers_mt(800); +SELECT uniqExact(_partition_id), uniqExact(x + 1) FROM t_win_int; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS force_window_partitions_independently = 1; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +DROP TABLE t_win_int; diff --git a/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference new file mode 100644 index 0000000000000..d695e73641ae3 --- /dev/null +++ b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference @@ -0,0 +1,98 @@ +-- { echo } + +-- --------------------------------------------------------------------------- +-- Preservation: integer arithmetic with an integer constant stays injective, so the +-- optimization must keep firing. A correctness-only test would not notice this. Each arm +-- prints two plan lines; an arm printing nothing has lost the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_int; +CREATE TABLE t_int (a UInt32) ENGINE = MergeTree ORDER BY a PARTITION BY intDiv(a, 2) * 2 + 1; +INSERT INTO t_int SELECT number FROM numbers_mt(32); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) + 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- subtraction widens to a signed result, which is a different branch of the integer test +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) - 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- the constant may be either operand, so both positions have to be recognized +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 7 - intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 1 + intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- negate and tuple are not named by the query-plan consumer: they answer isInjective themselves, so +-- the optimization has to keep firing for them. +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT -intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT tuple(intDiv(a, 2)) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_int; +-- wide integers are exact, so the gate must not be narrowed to native widths +DROP TABLE IF EXISTS t_int128; +CREATE TABLE t_int128 (a Int128) ENGINE = MergeTree ORDER BY a PARTITION BY a % 8; +INSERT INTO t_int128 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT a + 7 AS a1 FROM t_int128 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_int128; +DROP TABLE IF EXISTS t_uint256; +CREATE TABLE t_uint256 (b UInt256) ENGINE = MergeTree ORDER BY b PARTITION BY b % 8; +INSERT INTO t_uint256 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT b + 7 AS b1 FROM t_uint256 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_uint256; +-- both type-wrapper orders must be stripped before the integer test. Contrast with the NULL +-- constant arm above: a Nullable column keeps the optimization, a NULL constant loses it. +DROP TABLE IF EXISTS t_nullable; +CREATE TABLE t_nullable (n Nullable(UInt32)) ENGINE = MergeTree ORDER BY n PARTITION BY n % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT n + 1 AS n1 FROM t_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_nullable; +DROP TABLE IF EXISTS t_lc; +CREATE TABLE t_lc (l LowCardinality(UInt32)) ENGINE = MergeTree ORDER BY l PARTITION BY l % 8; +INSERT INTO t_lc SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT l + 1 AS l1 FROM t_lc SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_lc; +DROP TABLE IF EXISTS t_lc_nullable; +CREATE TABLE t_lc_nullable (ln LowCardinality(Nullable(UInt32))) ENGINE = MergeTree ORDER BY ln PARTITION BY ln % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_lc_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT ln + 1 AS ln1 FROM t_lc_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_lc_nullable; +-- --------------------------------------------------------------------------- +-- Accepted narrowings. These three expressions are injective, but every type they would +-- admit also admits a non-injective case above that cannot be told apart: an Interval on +-- Date is not separable from one on DateTime by kind, a Date operand re-admits the +-- narrowing constant, and a Decimal operand re-admits the rescaling constant. Each arm +-- prints nothing; the bare-key arm after it is the control showing the fixture does reach +-- the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_date; +CREATE TABLE t_date (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY toDayOfWeek(d); +INSERT INTO t_date SELECT toDate('2001-01-01') + number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + INTERVAL 1 DAY AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + 1 AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toDayOfWeek(d) AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_date; +DROP TABLE IF EXISTS t_dec; +CREATE TABLE t_dec (dec Decimal64(2)) ENGINE = MergeTree ORDER BY dec PARTITION BY toUInt32(dec) % 8; +INSERT INTO t_dec SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT dec + 1 AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toUInt32(dec) AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_dec; diff --git a/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql new file mode 100644 index 0000000000000..cb93bc55fbcc5 --- /dev/null +++ b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql @@ -0,0 +1,100 @@ +-- Tags: no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +SET explain_query_plan_default = 'legacy'; +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +SET allow_suspicious_low_cardinality_types = 1; +-- The EXPLAIN QUERY TREE arms below are analyzer-only; old-analyzer jobs would error on them. +SET enable_analyzer = 1; + +-- { echo } + +-- --------------------------------------------------------------------------- +-- Preservation: integer arithmetic with an integer constant stays injective, so the +-- optimization must keep firing. A correctness-only test would not notice this. Each arm +-- prints two plan lines; an arm printing nothing has lost the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_int; +CREATE TABLE t_int (a UInt32) ENGINE = MergeTree ORDER BY a PARTITION BY intDiv(a, 2) * 2 + 1; +INSERT INTO t_int SELECT number FROM numbers_mt(32); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) + 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- subtraction widens to a signed result, which is a different branch of the integer test +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) - 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- the constant may be either operand, so both positions have to be recognized +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 7 - intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 1 + intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- negate and tuple are not named by the query-plan consumer: they answer isInjective themselves, so +-- the optimization has to keep firing for them. +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT -intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT tuple(intDiv(a, 2)) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_int; + +-- wide integers are exact, so the gate must not be narrowed to native widths +DROP TABLE IF EXISTS t_int128; +CREATE TABLE t_int128 (a Int128) ENGINE = MergeTree ORDER BY a PARTITION BY a % 8; +INSERT INTO t_int128 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT a + 7 AS a1 FROM t_int128 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_int128; + +DROP TABLE IF EXISTS t_uint256; +CREATE TABLE t_uint256 (b UInt256) ENGINE = MergeTree ORDER BY b PARTITION BY b % 8; +INSERT INTO t_uint256 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT b + 7 AS b1 FROM t_uint256 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_uint256; + +-- both type-wrapper orders must be stripped before the integer test. Contrast with the NULL +-- constant arm above: a Nullable column keeps the optimization, a NULL constant loses it. +DROP TABLE IF EXISTS t_nullable; +CREATE TABLE t_nullable (n Nullable(UInt32)) ENGINE = MergeTree ORDER BY n PARTITION BY n % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT n + 1 AS n1 FROM t_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_nullable; + +DROP TABLE IF EXISTS t_lc; +CREATE TABLE t_lc (l LowCardinality(UInt32)) ENGINE = MergeTree ORDER BY l PARTITION BY l % 8; +INSERT INTO t_lc SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT l + 1 AS l1 FROM t_lc SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_lc; + +DROP TABLE IF EXISTS t_lc_nullable; +CREATE TABLE t_lc_nullable (ln LowCardinality(Nullable(UInt32))) ENGINE = MergeTree ORDER BY ln PARTITION BY ln % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_lc_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT ln + 1 AS ln1 FROM t_lc_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_lc_nullable; + +-- --------------------------------------------------------------------------- +-- Accepted narrowings. These three expressions are injective, but every type they would +-- admit also admits a non-injective case above that cannot be told apart: an Interval on +-- Date is not separable from one on DateTime by kind, a Date operand re-admits the +-- narrowing constant, and a Decimal operand re-admits the rescaling constant. Each arm +-- prints nothing; the bare-key arm after it is the control showing the fixture does reach +-- the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_date; +CREATE TABLE t_date (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY toDayOfWeek(d); +INSERT INTO t_date SELECT toDate('2001-01-01') + number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + INTERVAL 1 DAY AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + 1 AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toDayOfWeek(d) AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_date; + +DROP TABLE IF EXISTS t_dec; +CREATE TABLE t_dec (dec Decimal64(2)) ENGINE = MergeTree ORDER BY dec PARTITION BY toUInt32(dec) % 8; +INSERT INTO t_dec SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT dec + 1 AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toUInt32(dec) AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_dec; From bb7091ad5ed09ac6151489008e575f43b3a80960 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 06:27:27 +0000 Subject: [PATCH 056/296] Backport #120972 to 26.8: MySQL dictionary source: accept `enable_local_infile` only in the configuration file --- src/Dictionaries/MySQLDictionarySource.cpp | 27 ++++++ .../dictionaries/mysql_dict_local_infile.xml | 30 +++++++ .../test_dictionaries_mysql/test.py | 72 +++++++++++++++ ...l_dictionary_enable_local_infile.reference | 12 +++ ...29_mysql_dictionary_enable_local_infile.sh | 90 +++++++++++++++++++ 5 files changed, 231 insertions(+) create mode 100644 tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml create mode 100644 tests/queries/0_stateless/05229_mysql_dictionary_enable_local_infile.reference create mode 100755 tests/queries/0_stateless/05229_mysql_dictionary_enable_local_infile.sh diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 62965ab14b05a..e18fdb935deef 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -88,6 +88,26 @@ static void checkNoSSLPaths(const Poco::Util::AbstractConfiguration & config, co key, contents_key); } } + +/// `enable_local_infile` sets `MYSQL_OPT_LOCAL_INFILE`, which lets the MySQL endpoint ask the client +/// for the contents of a file of its choosing, read with the server's own privileges: the option is +/// off by default because it is insecure (`mysqlxx/Connection.h`). Same reasoning as above. +/// `fallback_prefix` is the parent prefix a `` inherits the value from, resolved in the same +/// order as `Pool::Pool`, so what is checked is the value the connection will actually use. +static void checkNoLocalInfile( + const Poco::Util::AbstractConfiguration & config, + const std::string & prefix, + const std::string & fallback_prefix = {}) +{ + const bool inherited + = !fallback_prefix.empty() && config.getBool(fallback_prefix + ".enable_local_infile", false); + + if (config.getBool(prefix + ".enable_local_infile", inherited)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`enable_local_infile` cannot be enabled in a dictionary created with a DDL query. " + "It is only accepted in a dictionary defined in a server configuration file"); +} #endif void registerDictionarySourceMysql(DictionarySourceFactory & factory); @@ -197,6 +217,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) { const auto replica_prefix = settings_config_prefix + "." + replica_key; checkNoSSLPaths(config, replica_prefix); + checkNoLocalInfile(config, replica_prefix, settings_config_prefix); global_context->getRemoteHostFilter().checkHostAndPort( config.getString(replica_prefix + ".host"), toString(config.getInt(replica_prefix + ".port", 3306))); @@ -205,6 +226,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) } else { + checkNoLocalInfile(config, settings_config_prefix); global_context->getRemoteHostFilter().checkHostAndPort( config.getString(settings_config_prefix + ".host"), toString(config.getInt(settings_config_prefix + ".port", 3306))); @@ -300,6 +322,7 @@ Setting fields: | `fail_on_connection_loss` | Controls behavior of the server on connection loss. If `true`, an exception is thrown immediately if the connection between client and server was lost. If `false`, the server retries to fetch data at least three times before reporting an error. Note that retrying leads to increased response times. Default value: `false`. | | `query` | The custom query. Optional. | | `enable_compression` | Enables zlib compression for the MySQL protocol connection. When set to `1`, ClickHouse requests protocol-level compression from the MySQL server. Can also be set per-replica inside ``. Default value: `0`. | +| `enable_local_infile` | Allows the MySQL server to ask ClickHouse for the contents of a local file (`LOAD DATA LOCAL INFILE`). Only a dictionary defined in a server configuration file may enable it; a `CREATE DICTIONARY` query may leave it unset or set it to `0`. Can also be set per-replica inside ``, and a `` that does not set it inherits the value above. Default value: `0`. | | `ssl_ca_pem` | Contents of the CA certificate that the MySQL server certificate is verified against. Optional. | | `ssl_cert_pem` | Contents of the client certificate, for certificate-based authentication. Optional. | | `ssl_key_pem` | Contents of the private key belonging to `ssl_cert_pem`. Optional. | @@ -313,6 +336,10 @@ The `table` or `where` fields cannot be used together with the `query` field. An `ssl_ca`, `ssl_cert` and `ssl_key` name files that the server opens with its own privileges, so they are only accepted for a dictionary defined in a server configuration file, or through a named collection defined there. A `CREATE DICTIONARY` query that specifies the TLS credentials directly must pass their contents instead, in `ssl_ca_pem`, `ssl_cert_pem` and `ssl_key_pem`. Those values are masked in logs and in `SHOW` queries, the same way passwords are. + +`enable_local_infile` lets the MySQL server ask ClickHouse for the contents of a file of its choosing, which the server reads with its own privileges, so it is only accepted for a dictionary defined in a server configuration file. A `CREATE DICTIONARY` query that enables it, at the source or at one of its `` entries, is rejected with `BAD_ARGUMENTS`. + + There is no explicit parameter `secure`. When establishing an SSL-connection security is mandatory. diff --git a/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml b/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml new file mode 100644 index 0000000000000..9d5a3fe658f37 --- /dev/null +++ b/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml @@ -0,0 +1,30 @@ + + + dict_local_infile + + + test + mysql80 + 3306 + root + ClickHouse_MySQL_P@ssw0rd + dict_local_infile_table
+ 1 + + + + + + + + id + + + value + String + + + + 0 + + diff --git a/tests/integration/test_dictionaries_mysql/test.py b/tests/integration/test_dictionaries_mysql/test.py index d2bd9de404019..b7be44a3ef57d 100644 --- a/tests/integration/test_dictionaries_mysql/test.py +++ b/tests/integration/test_dictionaries_mysql/test.py @@ -19,6 +19,7 @@ "configs/dictionaries/mysql_dict_compression.xml", "configs/dictionaries/mysql_dict_compression_wire.xml", "configs/dictionaries/mysql_dict_no_compression_wire.xml", + "configs/dictionaries/mysql_dict_local_infile.xml", ] CONFIG_FILES = [ "configs/remote_servers.xml", @@ -1032,3 +1033,74 @@ def reload(name): mysql_connection, "DROP TABLE IF EXISTS test.inherit_pool_test;" ) mysql_connection.close() + + +def test_enable_local_infile_xml_dict(started_cluster): + """`enable_local_infile` is rejected in a dictionary created with a DDL query, but a dictionary + defined in a server configuration file is written by an operator and keeps working.""" + mysql_connection = get_mysql_conn(started_cluster) + + try: + execute_mysql_query( + mysql_connection, "DROP TABLE IF EXISTS test.dict_local_infile_table;" + ) + execute_mysql_query( + mysql_connection, + "CREATE TABLE test.dict_local_infile_table (id INT NOT NULL, value TEXT, PRIMARY KEY(id));", + ) + execute_mysql_query( + mysql_connection, + "INSERT INTO test.dict_local_infile_table VALUES (1, 'local_infile');", + ) + + # The dictionary is lazily loaded, so nothing has touched the source yet; reload it now that + # its table exists. + for _ in range(10): + try: + instance.query("SYSTEM RELOAD DICTIONARY dict_local_infile") + break + except Exception: + time.sleep(0.5) + + # Loading the dictionary means the source was instantiated and connected with the option on, + # which is what the guard must not prevent for this route. + value = instance.query( + "SELECT dictGet('dict_local_infile', 'value', toUInt64(1))" + ).strip() + last_exception = instance.query( + "SELECT last_exception FROM system.dictionaries WHERE name = 'dict_local_infile'" + ).strip() + assert value == "local_infile", ( + " was rejected in the XML dict config: " + f"{last_exception!r}" + ) + + # The same option from a DDL query is rejected, at the source and at a replica alike. The + # endpoint here is the working MySQL server, so a rejection cannot be a connection failure + # in disguise. + credentials = f"USER 'root' PASSWORD '{mysql_pass}' DB 'test' TABLE 'dict_local_infile_table'" + for source in ( + f"HOST 'mysql80' PORT 3306 {credentials} ENABLE_LOCAL_INFILE 1", + f"{credentials} REPLICA(PRIORITY 1 HOST 'mysql80' PORT 3306 ENABLE_LOCAL_INFILE 1)", + ): + instance.query("DROP DICTIONARY IF EXISTS dict_local_infile_ddl") + instance.query( + f""" + CREATE DICTIONARY dict_local_infile_ddl (id UInt64, value String) + PRIMARY KEY id + SOURCE(MYSQL({source})) + LAYOUT(FLAT()) + LIFETIME(0) + """ + ) + with pytest.raises(Exception) as exc: + instance.query("SYSTEM RELOAD DICTIONARY dict_local_infile_ddl") + assert "cannot be enabled in a dictionary created with a DDL query" in str( + exc.value + ), f"Unexpected error for {source!r}: {exc.value}" + finally: + instance.query("DROP DICTIONARY IF EXISTS dict_local_infile_ddl") + execute_mysql_query( + mysql_connection, "DROP TABLE IF EXISTS test.dict_local_infile_table;" + ) + mysql_connection.close() diff --git a/tests/queries/0_stateless/05229_mysql_dictionary_enable_local_infile.reference b/tests/queries/0_stateless/05229_mysql_dictionary_enable_local_infile.reference new file mode 100644 index 0000000000000..b2e6ae32e0372 --- /dev/null +++ b/tests/queries/0_stateless/05229_mysql_dictionary_enable_local_infile.reference @@ -0,0 +1,12 @@ +--- enabled at the source +OK +--- enabled at a replica +OK +--- enabled at a later replica, disabled at the first +OK +--- inherited by a replica from the source +OK +--- disabled at the source +OK +--- disabled at every replica, enabled at the source +OK diff --git a/tests/queries/0_stateless/05229_mysql_dictionary_enable_local_infile.sh b/tests/queries/0_stateless/05229_mysql_dictionary_enable_local_infile.sh new file mode 100755 index 0000000000000..38f1d99828889 --- /dev/null +++ b/tests/queries/0_stateless/05229_mysql_dictionary_enable_local_infile.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the MySQL integration is not available in the fast test build. + +# `enable_local_infile` sets `MYSQL_OPT_LOCAL_INFILE` on the connection, which lets the MySQL endpoint +# ask the client for the contents of a file of its choosing: the server opens it with its own +# privileges. A dictionary created with a DDL query may not enable it, whether at the source itself or +# at one of its replicas. A dictionary defined in a server configuration file is written by an operator +# and keeps working; that route is covered by tests/integration/test_dictionaries_mysql. +# +# No MySQL server is needed: the rejection happens while the source is instantiated, before any +# connection. A source is instantiated when the dictionary is loaded, so the reload is what surfaces +# the rejection if the `CREATE` itself did not. +# +# The arms that must NOT be rejected point at `127.0.0.1` port 1, where nothing listens, so they end in +# a connection failure. That failure is asserted too: it is what proves the arm reached an actual +# connection attempt instead of being rejected for some unrelated reason, which is what keeps the +# "no rejection" oracle from passing vacuously. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +REJECTED="cannot be enabled in a dictionary created with a DDL query" +CONNECTION_FAILED="ALL_CONNECTION_TRIES_FAILED" + +create_and_load() { + local name="$1" + local source="$2" + $CLICKHOUSE_CLIENT --query "DROP DICTIONARY IF EXISTS ${name}" + $CLICKHOUSE_CLIENT --query " + CREATE DICTIONARY ${name} (id UInt32, name String) PRIMARY KEY id + SOURCE(MYSQL(${source})) + LAYOUT(FLAT()) LIFETIME(0)" + $CLICKHOUSE_CLIENT --query "SYSTEM RELOAD DICTIONARY ${name}" + $CLICKHOUSE_CLIENT --query "DROP DICTIONARY IF EXISTS ${name}" +} + +expect_rejected() { + echo "--- $1" + if create_and_load "$2" "$3" 2>&1 | grep -q "$REJECTED"; then + echo "OK" + else + echo "FAIL: expected an error matching: $REJECTED" + fi +} + +expect_not_rejected() { + echo "--- $1" + local output + output=$(create_and_load "$2" "$3" 2>&1) + if echo "$output" | grep -q "$REJECTED"; then + echo "FAIL: unexpected error matching: $REJECTED" + elif ! echo "$output" | grep -q "$CONNECTION_FAILED"; then + echo "FAIL: expected the source to reach a connection attempt" + else + echo "OK" + fi +} + +expect_rejected "enabled at the source" dict_infile_source \ + "HOST '127.0.0.1' PORT 1 USER 'u' PASSWORD 'p' DB 'd' TABLE 't' ENABLE_LOCAL_INFILE 1" + +expect_rejected "enabled at a replica" dict_infile_replica \ + "USER 'u' PASSWORD 'p' DB 'd' TABLE 't' + REPLICA(PRIORITY 1 HOST '127.0.0.1' PORT 1 ENABLE_LOCAL_INFILE 1)" + +# Every is checked, not just the first: the pool resolves the option per replica and +# fails over between them, so a later replica enabling it is enough to reach MYSQL_OPT_LOCAL_INFILE. +expect_rejected "enabled at a later replica, disabled at the first" dict_infile_replica_second \ + "USER 'u' PASSWORD 'p' DB 'd' TABLE 't' PORT 1 + REPLICA(PRIORITY 1 HOST '127.0.0.1' ENABLE_LOCAL_INFILE 0) + REPLICA(PRIORITY 2 HOST '127.0.0.1' ENABLE_LOCAL_INFILE 1)" + +# A replica that does not set the option inherits it from the source, so the source is where this one +# is enabled. The guard resolves the value the same way the connection pool does. +expect_rejected "inherited by a replica from the source" dict_infile_inherited \ + "USER 'u' PASSWORD 'p' DB 'd' TABLE 't' PORT 1 ENABLE_LOCAL_INFILE 1 + REPLICA(PRIORITY 1 HOST '127.0.0.1')" + +# Explicitly disabled is what the connection does by default, so it is accepted. +expect_not_rejected "disabled at the source" dict_infile_disabled \ + "HOST '127.0.0.1' PORT 1 USER 'u' PASSWORD 'p' DB 'd' TABLE 't' ENABLE_LOCAL_INFILE 0" + +# Only per-replica pools are built when the source has replicas, so a replica that turns the option off +# never enables it, whatever the source says. Two replicas, because the option is resolved per replica. +expect_not_rejected "disabled at every replica, enabled at the source" dict_infile_replicas_disabled \ + "USER 'u' PASSWORD 'p' DB 'd' TABLE 't' PORT 1 ENABLE_LOCAL_INFILE 1 + REPLICA(PRIORITY 1 HOST '127.0.0.1' ENABLE_LOCAL_INFILE 0) + REPLICA(PRIORITY 2 HOST '127.0.0.1' ENABLE_LOCAL_INFILE 0)" From 47a0b71b58f22f214c6f6d96371c61cfbb658713 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 09:40:47 +0000 Subject: [PATCH 057/296] Backport #120736 to 26.8: Update NuRaft to fix abandoned-peer rejoining --- contrib/NuRaft | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index a0eda0069d2da..884294e9dbee1 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit a0eda0069d2dac0852c453cbe7b0285eda218479 +Subproject commit 884294e9dbee183b53601bbe2c8c5753cec1a3db From a377c2249b6bda1cd8291f34a8bded599bb506f4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 10:03:20 +0000 Subject: [PATCH 058/296] Backport #112934 to 26.8: Keep Iceberg statistics for supported columns when a field type is unsupported --- .../DataLakes/Iceberg/IcebergWrites.cpp | 25 +++--- ...eberg_insert_per_file_statistics.reference | 4 +- ...4652_iceberg_insert_per_file_statistics.sh | 16 ++-- ...insert_partial_statistics_bounds.reference | 8 ++ ...ceberg_insert_partial_statistics_bounds.sh | 79 +++++++++++++++++++ 5 files changed, 110 insertions(+), 22 deletions(-) create mode 100644 tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference create mode 100755 tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index a83d8b5f3278a..27c5a81df1cb3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -296,21 +296,22 @@ std::vector dumpFieldToBytes(const Field & field, DataTypePtr type) } } -bool canWriteStatistics( +/// Retains only the bounds that can be serialized. A field left out is simply absent from the +/// manifest bounds map, which readers treat as "bound unknown" for that column. +std::vector> filterWritableStatistics( const std::vector> & statistics, const std::unordered_map & field_id_to_column_index, SharedHeader sample_block) { - if (statistics.empty()) - return false; - + std::vector> writable; + writable.reserve(statistics.size()); for (const auto & [field_id, stat] : statistics) { auto type = sample_block->getDataTypes()[field_id_to_column_index.at(field_id)]; - if (!canDumpIcebergStats(stat, type)) - return false; + if (canDumpIcebergStats(stat, type)) + writable.emplace_back(field_id, stat); } - return true; + return writable; } } @@ -610,13 +611,15 @@ void generateManifestFile( auto dump_fields = [&](size_t field_id, Field value) { return dumpFieldToBytes(value, sample_block->getDataTypes()[field_id_to_column_index.at(field_id)]); }; - auto lower_statistics = effective_statistics->getLowerBounds(); - if (canWriteStatistics(lower_statistics, field_id_to_column_index, sample_block)) + auto lower_statistics + = filterWritableStatistics(effective_statistics->getLowerBounds(), field_id_to_column_index, sample_block); + if (!lower_statistics.empty()) { set_fields(lower_statistics, Iceberg::f_lower_bounds, dump_fields); } - auto upper_statistics = effective_statistics->getUpperBounds(); - if (canWriteStatistics(upper_statistics, field_id_to_column_index, sample_block)) + auto upper_statistics + = filterWritableStatistics(effective_statistics->getUpperBounds(), field_id_to_column_index, sample_block); + if (!upper_statistics.empty()) { set_fields(upper_statistics, Iceberg::f_upper_bounds, dump_fields); } diff --git a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference index 515699bb5704e..8f2287b860fa9 100644 --- a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference +++ b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference @@ -34,8 +34,8 @@ partition_record_count_null_count: [('{\'eu\'}',1,0),('{\'eu\'}',1,1),('{\'us\'} 0 --- per-entry pairing against the referenced Parquet file --- id=1 rows=1/1 score_nulls=0/0 paired=yes lower=[(1,1),(2,10)] upper=[(1,1),(2,10)] -id=2 rows=1/1 score_nulls=1/1 paired=yes lower=[] upper=[] -id=3 rows=1/1 score_nulls=1/1 paired=yes lower=[] upper=[] +id=2 rows=1/1 score_nulls=1/1 paired=yes lower=[(1,2)] upper=[(1,2)] +id=3 rows=1/1 score_nulls=1/1 paired=yes lower=[(1,3)] upper=[(1,3)] --- per-entry bounds describe only their own file --- id=1 lower=[(1,1),(2,100)] upper=[(1,1),(2,100)] bounds_are_own_row=yes id=2 lower=[(1,2),(2,200)] upper=[(1,2),(2,200)] bounds_are_own_row=yes diff --git a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh index dc05b1e1ec7eb..63458bc946f15 100755 --- a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh +++ b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh @@ -153,11 +153,10 @@ ${CLICKHOUSE_CLIENT} --query " echo '--- per-entry pairing against the referenced Parquet file ---' for manifest in $(find "${PAIRED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not -name 'snap-*.avro' -type f | sort); do # `lower_bounds`/`upper_bounds` hold raw little-endian bytes (`dumpValue` in `IcebergWrites.cpp`), - # so they are decoded with `reinterpretAsInt32`; both key columns are `Int32` here. An entry whose - # `score` is all-NULL legitimately has NO bounds at all: `canWriteStatistics` is all-or-nothing - # across the entry's columns and `ColumnNullable::getExtremes` yields NULL extremes for an - # all-NULL column, which `canDumpIcebergStats` rejects. That is pre-existing behaviour, so it is - # asserted here rather than fixed. + # so they are decoded with `reinterpretAsInt32`; both key columns are `Int32` here. Bounds are + # per-field: an entry keeps a bound for every column whose extreme is serializable and omits the + # rest. `ColumnNullable::getExtremes` yields NULL extremes for an all-NULL column, which + # `canDumpIcebergStats` rejects, so an all-NULL `score` contributes no bound while `id` still does. ${CLICKHOUSE_CLIENT} --query " WITH entries AS ( SELECT @@ -190,10 +189,9 @@ for manifest in $(find "${PAIRED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not " done -# The bounds above are present on one entry only, because the two all-NULL files legitimately carry no -# bounds at all. This companion has no nullable column, so every entry carries bounds and the decoded -# value of each is checked against the single row its own file holds. Without it the bounds half of -# this change would be pinned on a single entry. +# Only one entry above carries a `score` bound, because the other two files hold an all-NULL `score`. +# This companion has no nullable column, so every entry carries a bound for both of its columns and the +# decoded value of each is checked against the single row its own file holds. ${CLICKHOUSE_CLIENT} --query " ${ONE_ROW_PER_FILE} CREATE TABLE bounded (id Int32, v Int32) diff --git a/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference new file mode 100644 index 0000000000000..415eb3c1f59a8 --- /dev/null +++ b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference @@ -0,0 +1,8 @@ +--- A: key Int32 + arr Array(Int32) +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] +--- B: key Int32 + opt Nullable(Int32) all NULL +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] +--- C: key Int32 + val String (control, all supported) +lower_ids=[1,2] upper_ids=[1,2] key_lower=[0] key_upper=[4] +--- D: key Int32 + f Nullable(Float64) with values +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] diff --git a/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh new file mode 100755 index 0000000000000..8acdb1edb91bc --- /dev/null +++ b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + +PREFIX="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +ROOT="${USER_FILES_PATH}/${PREFIX}" + +cleanup() +{ + for suffix in a b c d; do + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${PREFIX}_${suffix}" + done + rm -rf "${ROOT}" +} +trap cleanup EXIT + +# Prints the field ids present in lower_bounds / upper_bounds, plus the decoded Int32 bounds of +# field id 1 (the `key` column, always Int32 and always serializable here). +report() +{ + local table_dir="$1" + for manifest in $(find "${table_dir}/metadata" -maxdepth 1 -name '*.avro' \ + -not -name 'snap-*.avro' -type f | sort); do + ${CLICKHOUSE_CLIENT} --query " + SELECT + 'lower_ids=' || toString(arraySort(arrayMap(x -> x.1, tupleElement(data_file, 'lower_bounds')))), + 'upper_ids=' || toString(arraySort(arrayMap(x -> x.1, tupleElement(data_file, 'upper_bounds')))), + 'key_lower=' || toString(arrayMap(x -> reinterpretAsInt32(x.2), + arrayFilter(x -> x.1 = 1, tupleElement(data_file, 'lower_bounds')))), + 'key_upper=' || toString(arrayMap(x -> reinterpretAsInt32(x.2), + arrayFilter(x -> x.1 = 1, tupleElement(data_file, 'upper_bounds')))) + FROM file('${manifest}', Avro) + ORDER BY 1, 2, 3, 4 + " + done +} + +# Case A: an unsupported column type (Array(Int32)) must not suppress the bounds of `key`. +echo '--- A: key Int32 + arr Array(Int32)' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_a (key Int32, arr Array(Int32)) + ENGINE = IcebergLocal('${ROOT}/a/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_a SELECT number::Int32, [number, number] FROM numbers(5)" +report "${ROOT}/a" + +# Case B: an entirely-NULL Nullable column yields a Null bound, which must not suppress `key` either. +echo '--- B: key Int32 + opt Nullable(Int32) all NULL' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_b (key Int32, opt Nullable(Int32)) + ENGINE = IcebergLocal('${ROOT}/b/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_b SELECT number::Int32, NULL FROM numbers(5)" +report "${ROOT}/b" + +# Case C: control. All columns serializable, so both field ids keep their bounds exactly as before. +echo '--- C: key Int32 + val String (control, all supported)' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_c (key Int32, val String) + ENGINE = IcebergLocal('${ROOT}/c/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_c SELECT number::Int32, 'val-' || toString(number) FROM numbers(5)" +report "${ROOT}/c" + +# Case D: wrapper composition with a non-NULL value. Float64 is unsupported for bounds even though +# the byte dumper handles it, so the filter must gate on the bounds predicate and skip only `f`. +echo '--- D: key Int32 + f Nullable(Float64) with values' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_d (key Int32, f Nullable(Float64)) + ENGINE = IcebergLocal('${ROOT}/d/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_d SELECT number::Int32, number + 0.5 FROM numbers(5)" +report "${ROOT}/d" From 082700968837c479d07759aabc9e2ad890c486b0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 10:12:08 +0000 Subject: [PATCH 059/296] Backport #121392 to 26.8: Add column size statistics in metadata files --- src/Processors/Formats/IOutputFormat.h | 3 + .../Formats/Impl/ParquetBlockOutputFormat.cpp | 58 +++++++++ .../Formats/Impl/ParquetBlockOutputFormat.h | 4 + .../DataLakes/Iceberg/Compaction.cpp | 1 + .../DataLakes/Iceberg/DataFileStatistics.cpp | 60 ++++++++-- .../DataLakes/Iceberg/DataFileStatistics.h | 2 + .../DataLakes/Iceberg/IcebergWrites.cpp | 7 +- .../DataLakes/Iceberg/MultipleFileWriter.cpp | 6 + .../DataLakes/Iceberg/Mutations.cpp | 5 + ...est_writes_statistics_by_minmax_pruning.py | 113 +++++++++++++++++- ...eberg_insert_per_file_statistics.reference | 6 +- ...4652_iceberg_insert_per_file_statistics.sh | 24 ++-- 12 files changed, 262 insertions(+), 27 deletions(-) diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h index a86ca31cf01df..6edf87caef91b 100644 --- a/src/Processors/Formats/IOutputFormat.h +++ b/src/Processors/Formats/IOutputFormat.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -86,6 +87,8 @@ class IOutputFormat : public IProcessor virtual bool supportsWritingException() const { return false; } virtual void setException(const String & /*exception_message*/) {} + virtual std::unordered_map getColumnSizesOnDisk() const { return {}; } + /// A framing format (see IFramingFormat.h) multiplexes the formatted data along with auxiliary /// packets (progress, logs, profile events, exceptions) in the output stream. The format must /// have been created over the framing format's payload buffer. When set, the format notifies diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 7d5ea6dd2941e..28c3c0816d54a 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -21,6 +21,11 @@ namespace CurrentMetrics namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + using namespace Parquet; ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHeader header_, const FormatSettings & format_settings_, FormatFilterInfoPtr format_filter_info_) @@ -198,10 +203,62 @@ void ParquetBlockOutputFormat::finalizeImpl() writeFileHeader(file_state, out); } Block header = materializeBlock(getPort(PortKind::Main).getHeader()); + collectColumnSizesOnDisk(header); writeFileFooter(file_state, schema, options, out, header); chassert(out.count() - base_offset == file_state.offset); } +static size_t countSchemaLeaves(const SchemaElements & schema, size_t & index) +{ + if (index >= schema.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Parquet schema of {} elements is truncated", schema.size()); + + const auto & element = schema[index]; + ++index; + + if (!element.__isset.num_children || element.num_children == 0) + return 1; + + size_t leaves = 0; + for (Int32 i = 0; i < element.num_children; ++i) + leaves += countSchemaLeaves(schema, index); + return leaves; +} + +void ParquetBlockOutputFormat::collectColumnSizesOnDisk(const Block & header) +{ + std::vector leaves_per_column; + leaves_per_column.reserve(header.columns()); + size_t schema_index = 1; + size_t num_leaves = 0; + for (size_t i = 0; i < header.columns(); ++i) + { + leaves_per_column.push_back(countSchemaLeaves(schema, schema_index)); + num_leaves += leaves_per_column.back(); + } + + column_sizes_on_disk.clear(); + for (const auto & row_group : file_state.completed_row_groups) + { + const auto & column_chunks = row_group.row_group.columns; + if (column_chunks.size() != num_leaves) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Parquet row group has {} column chunks while the schema has {} leaf columns", + column_chunks.size(), + num_leaves); + + size_t leaf_index = 0; + for (size_t i = 0; i < header.columns(); ++i) + { + size_t column_size = 0; + for (size_t j = 0; j < leaves_per_column[i]; ++j, ++leaf_index) + column_size += static_cast(column_chunks[leaf_index].meta_data.total_compressed_size); + column_sizes_on_disk[header.getByPosition(i).name] += column_size; + } + } +} + void ParquetBlockOutputFormat::resetFormatterImpl() { if (pool) @@ -216,6 +273,7 @@ void ParquetBlockOutputFormat::resetFormatterImpl() task_queue.clear(); row_groups.clear(); file_state = {}; + column_sizes_on_disk.clear(); staging_chunks.clear(); staging_rows = 0; staging_bytes = 0; diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index 07322de00c291..c38b1e5d0ce7f 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -20,6 +20,8 @@ class ParquetBlockOutputFormat final : public IOutputFormat String getName() const override { return "ParquetBlockOutputFormat"; } + std::unordered_map getColumnSizesOnDisk() const override { return column_sizes_on_disk; } + private: struct MemoryToken { @@ -94,6 +96,7 @@ class ParquetBlockOutputFormat final : public IOutputFormat void consume(Chunk) override; void finalizeImpl() override; + void collectColumnSizesOnDisk(const Block & header); void resetFormatterImpl() override; void onCancel() noexcept override; @@ -119,6 +122,7 @@ class ParquetBlockOutputFormat final : public IOutputFormat Parquet::IcebergOptionality iceberg_optionality; Parquet::SchemaElements schema; Parquet::FileWriteState file_state; + std::unordered_map column_sizes_on_disk; size_t base_offset = 0; // initial out.count(), just for assert std::mutex mutex; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index beb714c61a235..bb21135a0aabb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -407,6 +407,7 @@ static void writeDataFiles( } output_format->flush(); output_format->finalize(); + data_file->manifest_list->statistics.addColumnSizesOnDisk(output_format->getColumnSizesOnDisk(), *sample_block); write_buffer->finalize(); auto file_bytes = write_buffer->count(); if (file_bytes == 0 && !data_file->patched_path.empty()) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp index 4e6879496fafe..95e3293aa6647 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp @@ -3,10 +3,16 @@ #include #include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + #if USE_AVRO DataFileStatistics::DataFileStatistics(Poco::JSON::Array::Ptr schema_) @@ -33,9 +39,8 @@ void DataFileStatistics::update(const Chunk & chunk) if (!chunk.hasRows()) return; size_t num_columns = chunk.getNumColumns(); - if (column_sizes.empty()) + if (null_counts.empty()) { - column_sizes.resize(num_columns, 0); null_counts.resize(num_columns, 0); for (size_t i = 0; i < num_columns; ++i) { @@ -48,7 +53,6 @@ void DataFileStatistics::update(const Chunk & chunk) for (size_t i = 0; i < num_columns; ++i) { const auto & col = chunk.getColumns()[i]; - column_sizes[i] += col->byteSize(); if (const auto * nullable_col = checkAndGetColumn(col.get())) { for (UInt8 v : nullable_col->getNullMapData()) @@ -58,23 +62,61 @@ void DataFileStatistics::update(const Chunk & chunk) } } -void DataFileStatistics::merge(const DataFileStatistics & other) +void DataFileStatistics::addColumnSizesOnDisk(const std::unordered_map & sizes_by_column_name, const Block & sample_block) { - if (other.column_sizes.empty()) + if (sizes_by_column_name.empty()) return; + if (sample_block.columns() != field_ids.size()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Iceberg data file has {} columns while its schema has {} fields", + sample_block.columns(), + field_ids.size()); + if (column_sizes.empty()) + column_sizes.resize(field_ids.size(), 0); + + for (size_t i = 0; i < field_ids.size(); ++i) + { + const auto & column_name = sample_block.getByPosition(i).name; + auto it = sizes_by_column_name.find(column_name); + if (it == sizes_by_column_name.end()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Written data file does not report the on-disk size of column {}", column_name); + column_sizes[i] += static_cast(it->second); + } +} + +void DataFileStatistics::merge(const DataFileStatistics & other) +{ + if (!other.column_sizes.empty()) + { + if (column_sizes.empty()) + { + column_sizes = other.column_sizes; + } + else + { + chassert(column_sizes.size() == other.column_sizes.size()); + for (size_t i = 0; i < column_sizes.size(); ++i) + column_sizes[i] += other.column_sizes[i]; + } + } + + if (other.null_counts.empty()) + return; + + if (null_counts.empty()) { - column_sizes = other.column_sizes; null_counts = other.null_counts; ranges = other.ranges; return; } - chassert(column_sizes.size() == other.column_sizes.size()); - for (size_t i = 0; i < column_sizes.size(); ++i) + chassert(null_counts.size() == other.null_counts.size()); + for (size_t i = 0; i < null_counts.size(); ++i) { - column_sizes[i] += other.column_sizes[i]; null_counts[i] += other.null_counts[i]; ranges[i] = uniteRanges(ranges[i], other.ranges[i]); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h index 680b283e03885..6352f2df9ad78 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -25,6 +26,7 @@ class DataFileStatistics explicit DataFileStatistics(Poco::JSON::Array::Ptr schema_); void update(const Chunk & chunk); + void addColumnSizesOnDisk(const std::unordered_map & sizes_by_column_name, const Block & sample_block); void merge(const DataFileStatistics & other); std::vector> getColumnSizes() const; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index a83d8b5f3278a..f9e3ef2fb69a0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -596,10 +596,11 @@ void generateManifestFile( } else if (effective_statistics) { - auto statistics = effective_statistics->getColumnSizes(); - set_fields(statistics, Iceberg::f_column_sizes, [](size_t, size_t value) { return static_cast(value); }); + auto column_sizes = effective_statistics->getColumnSizes(); + if (!column_sizes.empty()) + set_fields(column_sizes, Iceberg::f_column_sizes, [](size_t, size_t value) { return static_cast(value); }); - statistics = effective_statistics->getNullCounts(); + auto statistics = effective_statistics->getNullCounts(); set_fields(statistics, Iceberg::f_null_value_counts, [](size_t, size_t value) { return static_cast(value); }); std::unordered_map field_id_to_column_index; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp index 06f1ca6fcec63..1935421c3701f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp @@ -85,6 +85,12 @@ void MultipleFileWriter::finalize() { output_format->flush(); output_format->finalize(); + + auto column_sizes_on_disk = output_format->getColumnSizesOnDisk(); + if (current_file_stats) + current_file_stats->addColumnSizesOnDisk(column_sizes_on_disk, *sample_block); + stats.addColumnSizesOnDisk(column_sizes_on_disk, *sample_block); + buffer->finalize(); UInt64 file_bytes = buffer->count(); total_bytes += file_bytes; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 3bb6890d7f48b..6ffc50d493b88 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -267,6 +267,8 @@ static std::optional writeDataFiles( { delete_data_writers[partition_key]->flush(); delete_data_writers[partition_key]->finalize(); + delete_data_statistics.at(partition_key).addColumnSizesOnDisk( + delete_data_writers[partition_key]->getColumnSizesOnDisk(), getPositionDeleteFileSampleBlock()); delete_data_write_buffers[partition_key]->finalize(); { auto delete_bytes = delete_data_write_buffers[partition_key]->count(); @@ -289,6 +291,7 @@ static std::optional writeDataFiles( PullingPipelineExecutor executor(pipeline); auto header = interpreter->getUpdatedHeader(); + auto update_sample_block = getNonVirtualColumns(header, /*remove_low_cardinality=*/ true); Block block; while (executor.pull(block)) @@ -338,6 +341,8 @@ static std::optional writeDataFiles( { update_data_writers[partition_key]->flush(); update_data_writers[partition_key]->finalize(); + update_data_statistics.at(partition_key).addColumnSizesOnDisk( + update_data_writers[partition_key]->getColumnSizesOnDisk(), update_sample_block); update_data_write_buffers[partition_key]->finalize(); { auto update_bytes = update_data_write_buffers[partition_key]->count(); diff --git a/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py b/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py index 2cc7d839778a9..63ac6063258f6 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py @@ -1,11 +1,22 @@ +import glob +import os + +import avro.datafile +import avro.io +import pyarrow.parquet as pq import pytest from helpers.iceberg_utils import ( check_validity_and_get_prunned_files_general, create_iceberg_table, - get_uuid_str + default_download_directory, + get_last_snapshot, + get_uuid_str, + unescape_path, ) +TABLE_ROOT = "/var/lib/clickhouse/user_files/iceberg_data/default" + @pytest.mark.parametrize("format_version", [1, 2]) @pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) @@ -133,4 +144,102 @@ def check_validity_and_get_prunned_files(select_expression): f"SELECT * FROM {TABLE_NAME} WHERE number <= 5 ORDER BY ALL" ) == 3 - ) \ No newline at end of file + ) + + +def _read_avro(path): + with open(path, "rb") as f: + return list(avro.datafile.DataFileReader(f, avro.io.DatumReader())) + + +def _data_files_of_last_snapshot(table_path): + """The `data_file` record of every manifest entry of the newest snapshot.""" + snapshot_id = get_last_snapshot(table_path) + manifest_lists = glob.glob(f"{table_path}/metadata/snap-{snapshot_id}-*.avro") + assert len(manifest_lists) == 1, manifest_lists + + data_files = [] + for list_entry in _read_avro(manifest_lists[0]): + manifest = os.path.join( + table_path, "metadata", os.path.basename(unescape_path(list_entry["manifest_path"])) + ) + data_files.extend(entry["data_file"] for entry in _read_avro(manifest)) + return data_files + + +def _column_sizes_from_parquet(path): + """Per-column compressed size as the Parquet footer of one data file reports it, keyed by the + Iceberg field id the column carries.""" + parquet_file = pq.ParquetFile(path) + field_ids = [ + int(field.metadata[b"PARQUET:field_id"]) for field in parquet_file.schema_arrow + ] + + sizes = dict.fromkeys(field_ids, 0) + metadata = parquet_file.metadata + for row_group_index in range(metadata.num_row_groups): + row_group = metadata.row_group(row_group_index) + assert row_group.num_columns == len(field_ids) + for column_index, field_id in enumerate(field_ids): + sizes[field_id] += row_group.column(column_index).total_compressed_size + return sizes + + +@pytest.mark.parametrize("format_version", [1, 2]) +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_writes_column_sizes_are_on_disk_sizes( + started_cluster_iceberg_no_spark, format_version, storage_type +): + """`data_file.column_sizes` is the size the column occupies inside the data file, so every + entry must repeat what the Parquet footer of the file it names says. An in-memory size passes + neither check below: `s` holds 1000000 bytes of one repeated character, which compresses to a + fraction of the file, and the sum over a file cannot exceed the file itself.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + TABLE_NAME = ( + "test_writes_column_sizes_are_on_disk_sizes_" + + storage_type + + "_" + + get_uuid_str() + ) + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_no_spark, + "(id Int32, s String)", + format_version, + order_by="id", + ) + # Blocks of 1000 rows that the insert pipeline does not squash, so the writer rolls over at + # exactly 4000 rows and the commit holds three data files of known sizes. + instance.query( + f"INSERT INTO {TABLE_NAME} SELECT number, repeat('x', 100) FROM numbers(10000)", + settings={ + "iceberg_insert_max_rows_in_data_file": 4000, + "max_block_size": 1000, + "max_insert_block_size": 1000, + "min_insert_block_size_rows": 0, + "min_insert_block_size_bytes": 0, + "max_insert_threads": 1, + }, + ) + + table_path = f"{TABLE_ROOT}/{TABLE_NAME}/" + default_download_directory( + started_cluster_iceberg_no_spark, storage_type, table_path, table_path + ) + + data_files = _data_files_of_last_snapshot(table_path) + assert len(data_files) == 3 + + for data_file in data_files: + data_path = os.path.join( + table_path, "data", os.path.basename(unescape_path(data_file["file_path"])) + ) + written = {pair["key"]: pair["value"] for pair in data_file["column_sizes"]} + + assert written == _column_sizes_from_parquet(data_path) + assert sum(written.values()) <= data_file["file_size_in_bytes"] + + assert instance.query(f"SELECT count(), uniqExact(s) FROM {TABLE_NAME}") == "10000\t1\n" diff --git a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference index 515699bb5704e..09024e5f5d003 100644 --- a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference +++ b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference @@ -41,6 +41,6 @@ id=1 lower=[(1,1),(2,100)] upper=[(1,1),(2,100)] bounds_are_own_row=yes id=2 lower=[(1,2),(2,200)] upper=[(1,2),(2,200)] bounds_are_own_row=yes id=3 lower=[(1,3),(2,300)] upper=[(1,3),(2,300)] bounds_are_own_row=yes --- per-entry column_sizes describe only their own file --- -id=1 width=4 id_size=4 s_size=12 -id=2 width=8 id_size=4 s_size=16 -id=3 width=12 id_size=4 s_size=20 +id=1 width=10000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes +id=2 width=100000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes +id=3 width=1000000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes diff --git a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh index dc05b1e1ec7eb..710ebee9df7d9 100755 --- a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh +++ b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh @@ -235,16 +235,16 @@ done # The two scenarios above pin `record_count`, `null_value_counts` and the bounds to the file each # entry names, but not `column_sizes`, whose only other assertion is a cross-table `max()` inequality # that constrains no individual entry. Both of their fixtures also give every file the same per-file -# sizes, so a permutation cannot move a size value there. This scenario gives each file a `String` of -# a different width, which makes the sizes distinct and therefore permutation-sensitive. The values -# are in-memory `IColumn::byteSize` sums, not Parquet file sizes, so they are deterministic: -# `ColumnString::byteSize` is `chars.size() + offsets.size() * sizeof(offsets[0])`, and `insertData` -# appends exactly `length` bytes with no terminator. +# sizes, so a permutation cannot move a size value there. This scenario gives each file an +# incompressible `String` of a different order of magnitude, which makes the sizes distinct and +# therefore permutation-sensitive. `column_sizes` holds the post-compression size of the column chunk +# inside the Parquet file, so the exact byte count is a property of the encoder; what is asserted is +# that each entry's size follows the width of the row its own file holds and fits inside that file. ${CLICKHOUSE_CLIENT} --query " ${ONE_ROW_PER_FILE} CREATE TABLE sized (id Int32, s String) ENGINE = IcebergLocal('${SIZED_PATH}', 'Parquet') ORDER BY (id); - INSERT INTO sized SELECT number + 1, repeat('x', (number + 1) * 4) FROM numbers(3); + INSERT INTO sized SELECT number + 1, randomPrintableASCII(toUInt32(pow(10, number + 4))) FROM numbers(3); " echo '--- per-entry column_sizes describe only their own file ---' @@ -252,8 +252,9 @@ for manifest in $(find "${SIZED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not ${CLICKHOUSE_CLIENT} --query " WITH entries AS ( SELECT - replaceRegexpOne(tupleElement(data_file, 'file_path'), '^.*/', '') AS base, - CAST(tupleElement(data_file, 'column_sizes'), 'Map(Int32, Int64)') AS entry_sizes + replaceRegexpOne(tupleElement(data_file, 'file_path'), '^.*/', '') AS base, + CAST(tupleElement(data_file, 'column_sizes'), 'Map(Int32, Int64)') AS entry_sizes, + tupleElement(data_file, 'file_size_in_bytes') AS entry_file_size FROM file('${manifest}', Avro) ), files AS ( @@ -267,8 +268,11 @@ for manifest in $(find "${SIZED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not SELECT 'id=' || toString(f.own_id) || ' width=' || toString(f.own_width) - || ' id_size=' || toString(e.entry_sizes[1]) - || ' s_size=' || toString(e.entry_sizes[2]) AS entry + || ' id_size_positive=' || if(e.entry_sizes[1] > 0, 'yes', 'no') + || ' s_size_follows_own_width=' || if( + e.entry_sizes[2] > f.own_width / 2 AND e.entry_sizes[2] < f.own_width * 2, 'yes', 'no') + || ' sizes_fit_own_file=' || if( + e.entry_sizes[1] + e.entry_sizes[2] <= e.entry_file_size, 'yes', 'no') AS entry FROM entries AS e INNER JOIN files AS f ON e.base = f.base ORDER BY f.own_id FORMAT TSV; From cd83bc785c31287f3feebfd6d54e66ec32e7c895 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 12:01:46 +0000 Subject: [PATCH 060/296] Backport #116099 to 26.8: Honour `date_time_overflow_behavior` on parsing Date[Time] from text --- src/Core/FormatFactorySettings.h | 2 +- .../Serializations/SerializationDate.cpp | 32 ++-- .../Serializations/SerializationDate32.cpp | 40 +++-- .../Serializations/SerializationDateTime.cpp | 34 ++-- src/Formats/FormatSettings.h | 2 + src/Formats/JSONExtractTree.cpp | 25 ++- src/Functions/FunctionsConversion.h | 63 +++---- src/IO/ReadHelpers.cpp | 74 ++++++--- src/IO/ReadHelpers.h | 156 +++++++++++------- src/IO/parseDateTimeBestEffort.cpp | 101 ++++++------ src/IO/parseDateTimeBestEffort.h | 12 +- ...me_overflow_behavior_from_string.reference | 42 +++++ ...ate_time_overflow_behavior_from_string.sql | 105 ++++++++++++ 13 files changed, 470 insertions(+), 218 deletions(-) create mode 100644 tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference create mode 100644 tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index 5bc1e8766ec44..d48bdde8189e6 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -1613,7 +1613,7 @@ Possible values: Use the precise float parsing algorithm, which always returns the closest representable value to the input. When disabled, a faster but less accurate algorithm is used that may differ from the precise result by the least significant bits. )", 0) \ DECLARE(DateTimeOverflowBehavior, date_time_overflow_behavior, "ignore", R"( -Defines the behavior when [Date](/reference/data-types/date), [Date32](/reference/data-types/date32), [DateTime](/reference/data-types/datetime), [DateTime64](/reference/data-types/datetime64) or integers are converted into Date, Date32, DateTime or DateTime64 but the value cannot be represented in the result type. +Defines the behavior when [Date](/reference/data-types/date), [Date32](/reference/data-types/date32), [DateTime](/reference/data-types/datetime), [DateTime64](/reference/data-types/datetime64) or integers are converted into Date, Date32, DateTime or DateTime64 but the value cannot be represented in the result type. It also applies when a `Date` or `DateTime` is parsed from text, including by an input format. Possible values: diff --git a/src/DataTypes/Serializations/SerializationDate.cpp b/src/DataTypes/Serializations/SerializationDate.cpp index 0552baf4fd046..8b33f626b6714 100644 --- a/src/DataTypes/Serializations/SerializationDate.cpp +++ b/src/DataTypes/Serializations/SerializationDate.cpp @@ -38,26 +38,26 @@ void SerializationDate::deserializeWholeText(IColumn & column, ReadBuffer & istr throwUnexpectedDataAfterParsedValue(column, istr, settings, "Date"); } -bool SerializationDate::tryDeserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; - if (!tryReadDateText(x, istr, time_zone) || !istr.eof()) + if (!tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow()) || !istr.eof()) return false; assert_cast(column).getData().push_back(x); return true; } -void SerializationDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !settings.throwOnDateTimeOverflow()); assert_cast(column).getData().push_back(x); } -bool SerializationDate::tryDeserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; - if (!tryReadDateText(x, istr, time_zone)) + if (!tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow())) return false; assert_cast(column).getData().push_back(x); return true; @@ -75,19 +75,19 @@ void SerializationDate::serializeTextQuoted(const IColumn & column, size_t row_n writeChar('\'', ostr); } -void SerializationDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; assertChar('\'', istr); - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !settings.throwOnDateTimeOverflow()); assertChar('\'', istr); assert_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. } -bool SerializationDate::tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; - if (!checkChar('\'', istr) || !tryReadDateText(x, istr, time_zone) || !checkChar('\'', istr)) + if (!checkChar('\'', istr) || !tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow()) || !checkChar('\'', istr)) return false; assert_cast(column).getData().push_back(x); @@ -109,7 +109,7 @@ void SerializationDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr, return; } DayNum x; - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !format_settings.throwOnDateTimeOverflow()); assertChar('"', istr); assert_cast(column).getData().push_back(x); } @@ -120,7 +120,7 @@ bool SerializationDate::tryDeserializeTextJSON(IColumn & column, ReadBuffer & is return SerializationNumber::tryDeserializeTextJSON(column, istr, format_settings); DayNum x; - if (!tryReadDateText(x, istr, time_zone) || !checkChar('"', istr)) + if (!tryReadDateText(x, istr, time_zone, nullptr, !format_settings.throwOnDateTimeOverflow()) || !checkChar('"', istr)) return false; assert_cast(column).getData().push_back(x); return true; @@ -133,17 +133,17 @@ void SerializationDate::serializeTextCSV(const IColumn & column, size_t row_num, writeChar('"', ostr); } -void SerializationDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum value; - readCSV(value, istr, time_zone); + readCSV(value, istr, time_zone, !settings.throwOnDateTimeOverflow()); assert_cast(column).getData().push_back(value); } -bool SerializationDate::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum value; - if (!tryReadCSV(value, istr, time_zone)) + if (!tryReadCSV(value, istr, time_zone, !settings.throwOnDateTimeOverflow())) return false; assert_cast(column).getData().push_back(value); return true; diff --git a/src/DataTypes/Serializations/SerializationDate32.cpp b/src/DataTypes/Serializations/SerializationDate32.cpp index 0f17f1de8779d..085bbf971765a 100644 --- a/src/DataTypes/Serializations/SerializationDate32.cpp +++ b/src/DataTypes/Serializations/SerializationDate32.cpp @@ -10,6 +10,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_PARSE_DATE; +} + UInt128 SerializationDate32::getHash(const DateLUTImpl & time_zone_) { SipHash hash; @@ -37,26 +42,26 @@ void SerializationDate32::deserializeWholeText(IColumn & column, ReadBuffer & is throwUnexpectedDataAfterParsedValue(column, istr, settings, "Date32"); } -bool SerializationDate32::tryDeserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; - if (!tryReadDateText(x, istr, time_zone) || !istr.eof()) + if (!tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow()) || !istr.eof()) return false; assert_cast(column).getData().push_back(x); return true; } -void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !settings.throwOnDateTimeOverflow()); assert_cast(column).getData().push_back(x); } -bool SerializationDate32::tryDeserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; - if (!tryReadDateText(x, istr, time_zone)) + if (!tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow())) return false; assert_cast(column).getData().push_back(x); return true; @@ -74,19 +79,19 @@ void SerializationDate32::serializeTextQuoted(const IColumn & column, size_t row writeChar('\'', ostr); } -void SerializationDate32::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate32::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; assertChar('\'', istr); - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !settings.throwOnDateTimeOverflow()); assertChar('\'', istr); assert_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. } -bool SerializationDate32::tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; - if (!checkChar('\'', istr) || !tryReadDateText(x, istr, time_zone) || !checkChar('\'', istr)) + if (!checkChar('\'', istr) || !tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow()) || !checkChar('\'', istr)) return false; assert_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. return true; @@ -107,7 +112,7 @@ void SerializationDate32::deserializeTextJSON(IColumn & column, ReadBuffer & ist return; } ExtendedDayNum x; - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !format_settings.throwOnDateTimeOverflow()); assertChar('"', istr); assert_cast(column).getData().push_back(x); } @@ -118,7 +123,7 @@ bool SerializationDate32::tryDeserializeTextJSON(IColumn & column, ReadBuffer & return SerializationNumber::tryDeserializeTextJSON(column, istr, format_settings); ExtendedDayNum x; - if (!tryReadDateText(x, istr, time_zone) || !checkChar('"', istr)) + if (!tryReadDateText(x, istr, time_zone, nullptr, !format_settings.throwOnDateTimeOverflow()) || !checkChar('"', istr)) return false; assert_cast(column).getData().push_back(x); return true; @@ -131,18 +136,25 @@ void SerializationDate32::serializeTextCSV(const IColumn & column, size_t row_nu writeChar('"', ostr); } -void SerializationDate32::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate32::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { LocalDate value; readCSV(value, istr); + /// This one goes through `LocalDate`, which accepts a calendar-invalid date and resolves it to a default + if (settings.throwOnDateTimeOverflow() + && !tryToMakeDayNum(time_zone, value.year(), value.month(), value.day())) + throw Exception(ErrorCodes::CANNOT_PARSE_DATE, "Cannot parse date"); assert_cast(column).getData().push_back(value.getExtenedDayNum()); } -bool SerializationDate32::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { LocalDate value; if (!tryReadCSV(value, istr)) return false; + if (settings.throwOnDateTimeOverflow() + && !tryToMakeDayNum(time_zone, value.year(), value.month(), value.day())) + return false; assert_cast(column).getData().push_back(value.getExtenedDayNum()); return true; } diff --git a/src/DataTypes/Serializations/SerializationDateTime.cpp b/src/DataTypes/Serializations/SerializationDateTime.cpp index 0428077056674..16df898076f88 100644 --- a/src/DataTypes/Serializations/SerializationDateTime.cpp +++ b/src/DataTypes/Serializations/SerializationDateTime.cpp @@ -43,16 +43,19 @@ namespace inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) { + const auto overflow = settings.throwOnDateTimeOverflow() + ? DateTimeOverflow::Report + : DateTimeOverflow::Saturate; switch (settings.date_time_input_format) { case FormatSettings::DateTimeInputFormat::Basic: - readDateTimeTextImpl<>(x, istr, time_zone); + readDateTimeTextImpl<>(x, istr, time_zone, nullptr, nullptr, overflow == DateTimeOverflow::Saturate); break; case FormatSettings::DateTimeInputFormat::BestEffort: - parseDateTimeBestEffort(x, istr, time_zone, utc_time_zone); + parseDateTimeBestEffort(x, istr, time_zone, utc_time_zone, overflow); break; case FormatSettings::DateTimeInputFormat::BestEffortUS: - parseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone); + parseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone, overflow); break; } @@ -62,17 +65,20 @@ readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const D inline bool tryReadText( time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) { + const auto overflow = settings.throwOnDateTimeOverflow() + ? DateTimeOverflow::Report + : DateTimeOverflow::Saturate; bool res = false; switch (settings.date_time_input_format) { case FormatSettings::DateTimeInputFormat::Basic: - res = tryReadDateTimeText(x, istr, time_zone); + res = tryReadDateTimeText(x, istr, time_zone, nullptr, nullptr, overflow == DateTimeOverflow::Saturate); break; case FormatSettings::DateTimeInputFormat::BestEffort: - res = tryParseDateTimeBestEffort(x, istr, time_zone, utc_time_zone); + res = tryParseDateTimeBestEffort(x, istr, time_zone, utc_time_zone, overflow); break; case FormatSettings::DateTimeInputFormat::BestEffortUS: - res = tryParseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone); + res = tryParseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone, overflow); break; } @@ -181,11 +187,11 @@ void SerializationDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & } else if (settings.read_datetime_number_as_raw_value) /// Legacy: the raw value (seconds). { - readDateTimeAsRawValue(x, istr); + readDateTimeAsRawValue(x, istr, !settings.throwOnDateTimeOverflow()); } else /// Just 1504193808 or 1703363853.5 (a Unix timestamp, possibly with a sub-second part) { - readDateTimeAsNumber(x, istr); + readDateTimeAsNumber(x, istr, !settings.throwOnDateTimeOverflow()); } /// It's important to do this at the end - for exception safety. @@ -202,12 +208,12 @@ bool SerializationDateTime::tryDeserializeTextQuoted(IColumn & column, ReadBuffe } else if (settings.read_datetime_number_as_raw_value) /// Legacy: the raw value (seconds). { - if (!tryReadDateTimeAsRawValue(x, istr)) + if (!tryReadDateTimeAsRawValue(x, istr, !settings.throwOnDateTimeOverflow())) return false; } else /// Just 1504193808 or 1703363853.5 (a Unix timestamp, possibly with a sub-second part) { - if (!tryReadDateTimeAsNumber(x, istr)) + if (!tryReadDateTimeAsNumber(x, istr, !settings.throwOnDateTimeOverflow())) return false; } @@ -234,11 +240,11 @@ void SerializationDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & i } else if (settings.read_datetime_number_as_raw_value) /// Legacy: the raw value (seconds). { - readDateTimeAsRawValue(x, istr); + readDateTimeAsRawValue(x, istr, !settings.throwOnDateTimeOverflow()); } else { - readDateTimeAsNumber(x, istr); + readDateTimeAsNumber(x, istr, !settings.throwOnDateTimeOverflow()); } assert_cast(column).getData().push_back(static_cast(x)); @@ -254,12 +260,12 @@ bool SerializationDateTime::tryDeserializeTextJSON(IColumn & column, ReadBuffer } else if (settings.read_datetime_number_as_raw_value) /// Legacy: the raw value (seconds). { - if (!tryReadDateTimeAsRawValue(x, istr)) + if (!tryReadDateTimeAsRawValue(x, istr, !settings.throwOnDateTimeOverflow())) return false; } else { - if (!tryReadDateTimeAsNumber(x, istr)) + if (!tryReadDateTimeAsNumber(x, istr, !settings.throwOnDateTimeOverflow())) return false; } diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 42f62e34fb22c..99385f454d624 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -133,6 +133,8 @@ struct FormatSettings DateTimeOverflowBehavior date_time_overflow_behavior = DateTimeOverflowBehavior::Ignore; + bool throwOnDateTimeOverflow() const { return date_time_overflow_behavior == DateTimeOverflowBehavior::Throw; } + bool input_format_ipv4_default_on_conversion_error = false; bool input_format_ipv6_default_on_conversion_error = false; bool check_conversion_from_numbers_to_enum = true; diff --git a/src/Formats/JSONExtractTree.cpp b/src/Formats/JSONExtractTree.cpp index c8e2d6c27bc0c..c00dac73ec2b0 100644 --- a/src/Formats/JSONExtractTree.cpp +++ b/src/Formats/JSONExtractTree.cpp @@ -701,7 +701,7 @@ class DateNode : public JSONExtractTreeNode auto data = element.getString(); ReadBufferFromMemory buf(data); DateType date; - if (!tryReadDateText(date, buf) || !buf.eof()) + if (!tryReadDateText(date, buf, DateLUT::instance(), nullptr, !format_settings.throwOnDateTimeOverflow()) || !buf.eof()) { error = fmt::format("cannot parse Date value here: {}", data); return false; @@ -737,7 +737,7 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi time_t value = 0; if (element.isString()) { - if (!tryParse(value, element.getString(), format_settings.date_time_input_format)) + if (!tryParse(value, element.getString(), format_settings.date_time_input_format, !format_settings.throwOnDateTimeOverflow())) { error = fmt::format("cannot parse DateTime value here: {}", element.getString()); return false; @@ -756,12 +756,22 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi return false; } value = element.getInt64(); + if (format_settings.throwOnDateTimeOverflow() && (value < 0 || value > 0xFFFFFFFF)) + { + error = fmt::format("value {} is out of bounds of type DateTime", value); + return false; + } } else { /// Clamp in the unsigned domain before narrowing to time_t, /// because values above INT64_MAX would wrap to negative on cast. UInt64 raw = element.getUInt64(); + if (format_settings.throwOnDateTimeOverflow() && raw > 0xFFFFFFFF) + { + error = fmt::format("value {} is out of bounds of type DateTime", raw); + return false; + } value = static_cast(std::min(raw, UInt64(0xFFFFFFFF))); } } @@ -774,7 +784,7 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi /// exactly can cross the second boundary (`1703363853.9999999` arrives here as `1703363854.0`). String str_value = jsonElementToString(element, format_settings); ReadBufferFromMemory buf(str_value); - if (!tryReadDateTimeAsNumber(value, buf) || !buf.eof()) + if (!tryReadDateTimeAsNumber(value, buf, !format_settings.throwOnDateTimeOverflow()) || !buf.eof()) { error = fmt::format("cannot read DateTime value from JSON element: {}", str_value); return false; @@ -790,21 +800,22 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi return true; } - bool tryParse(time_t & value, std::string_view data, FormatSettings::DateTimeInputFormat date_time_input_format) const + bool tryParse(time_t & value, std::string_view data, FormatSettings::DateTimeInputFormat date_time_input_format, bool saturate_on_overflow) const { + const auto overflow = saturate_on_overflow ? DateTimeOverflow::Saturate : DateTimeOverflow::Report; ReadBufferFromMemory buf(data); switch (date_time_input_format) { case FormatSettings::DateTimeInputFormat::Basic: - if (tryReadDateTimeText(value, buf, time_zone) && buf.eof()) + if (tryReadDateTimeText(value, buf, time_zone, nullptr, nullptr, saturate_on_overflow) && buf.eof()) return true; break; case FormatSettings::DateTimeInputFormat::BestEffort: - if (tryParseDateTimeBestEffort(value, buf, time_zone, utc_time_zone) && buf.eof()) + if (tryParseDateTimeBestEffort(value, buf, time_zone, utc_time_zone, overflow) && buf.eof()) return true; break; case FormatSettings::DateTimeInputFormat::BestEffortUS: - if (tryParseDateTimeBestEffortUS(value, buf, time_zone, utc_time_zone) && buf.eof()) + if (tryParseDateTimeBestEffortUS(value, buf, time_zone, utc_time_zone, overflow) && buf.eof()) return true; break; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index e3158387728c0..79d6e6e8fe979 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -1084,7 +1084,7 @@ inline void convertFromTime(DataTypeTime::FieldType & x, time_t & /** Conversion of strings to numbers, dates, datetimes: through parsing. */ template -void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool precise_float_parsing) +void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool precise_float_parsing, bool) { if constexpr (is_floating_point) { @@ -1098,33 +1098,33 @@ void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTI } template <> -inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { DayNum tmp(0); - readDateText(tmp, rb, *time_zone); + readDateText(tmp, rb, *time_zone, saturate_on_overflow); x = tmp; } template <> -inline void parseImpl(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline void parseImpl(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { ExtendedDayNum tmp(0); - readDateText(tmp, rb, *time_zone); + readDateText(tmp, rb, *time_zone, saturate_on_overflow); x = tmp; } // NOTE: no need of extra overload of DateTime64, since readDateTimeText64 has different signature and that case is explicitly handled in the calling code. template <> -inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { time_t time = 0; - readDateTimeText(time, rb, *time_zone); + readDateTimeText(time, rb, *time_zone, saturate_on_overflow); convertFromTime(x, time); } template <> -inline void parseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline void parseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool) { time_t time = 0; readTimeText(time, rb, *time_zone); @@ -1132,7 +1132,7 @@ inline void parseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb } template <> -inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { UUID tmp; readUUIDText(tmp, rb); @@ -1140,7 +1140,7 @@ inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb } template <> -inline void parseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline void parseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { IPv4 tmp; readIPv4Text(tmp, rb); @@ -1148,7 +1148,7 @@ inline void parseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb } template <> -inline void parseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline void parseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { IPv6 tmp; readIPv6Text(tmp, rb); @@ -1156,7 +1156,7 @@ inline void parseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb } template -bool tryParseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool precise_float_parsing) +bool tryParseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool precise_float_parsing, bool) { if constexpr (is_floating_point) { @@ -1170,37 +1170,37 @@ bool tryParseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateL } template <> -inline bool tryParseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline bool tryParseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { DayNum tmp(0); - if (!tryReadDateText(tmp, rb, *time_zone)) + if (!tryReadDateText(tmp, rb, *time_zone, nullptr, saturate_on_overflow)) return false; x = tmp; return true; } template <> -inline bool tryParseImpl(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline bool tryParseImpl(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { ExtendedDayNum tmp(0); - if (!tryReadDateText(tmp, rb, *time_zone)) + if (!tryReadDateText(tmp, rb, *time_zone, nullptr, saturate_on_overflow)) return false; x = tmp; return true; } template <> -inline bool tryParseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline bool tryParseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { time_t time = 0; - if (!tryReadDateTimeText(time, rb, *time_zone)) + if (!tryReadDateTimeText(time, rb, *time_zone, nullptr, nullptr, saturate_on_overflow)) return false; convertFromTime(x, time); return true; } template <> -[[maybe_unused]]inline bool tryParseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +[[maybe_unused]]inline bool tryParseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool) { time_t time = 0; if (!tryReadTimeText(time, rb, *time_zone)) @@ -1210,7 +1210,7 @@ template <> } template <> -inline bool tryParseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline bool tryParseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { UUID tmp; if (!tryReadUUIDText(tmp, rb)) @@ -1221,7 +1221,7 @@ inline bool tryParseImpl(DataTypeUUID::FieldType & x, ReadBuffer & } template <> -inline bool tryParseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline bool tryParseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { IPv4 tmp; if (!tryReadIPv4Text(tmp, rb)) @@ -1232,7 +1232,7 @@ inline bool tryParseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & } template <> -inline bool tryParseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline bool tryParseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { IPv6 tmp; if (!tryReadIPv6Text(tmp, rb)) @@ -1365,6 +1365,11 @@ struct ConvertThroughParsing utc_time_zone = &DateLUT::instance("UTC"); } + /// Nothing to wrap around when parsing text, so `ignore` behaves like `saturate` + const bool saturate_on_overflow [[maybe_unused]] + = settings.date_time_overflow_behavior != FormatSettings::DateTimeOverflowBehavior::Throw; + const auto overflow [[maybe_unused]] = saturate_on_overflow ? DateTimeOverflow::Saturate : DateTimeOverflow::Report; + const IColumn * col_from = arguments[0].column.get(); const ColumnString * col_from_string = checkAndGetColumn(col_from); const ColumnFixedString * col_from_fixed_string = checkAndGetColumn(col_from); @@ -1459,7 +1464,7 @@ struct ConvertThroughParsing else { time_t res = 0; - parseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone); + parseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone, overflow); convertFromTime(vec_to[i], res); } } @@ -1486,7 +1491,7 @@ struct ConvertThroughParsing else { time_t res = 0; - parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); + parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone, overflow); convertFromTime(vec_to[i], res); } } @@ -1532,11 +1537,11 @@ struct ConvertThroughParsing } if constexpr (std::is_same_v) { - if (!tryParseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing)) + if (!tryParseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing, saturate_on_overflow)) throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "Cannot parse string to type {}", TypeName); } else - parseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing); + parseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing, saturate_on_overflow); } while (false); } } @@ -1571,7 +1576,7 @@ struct ConvertThroughParsing else { time_t res = 0; - parsed = tryParseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone); + parsed = tryParseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone, overflow); convertFromTime(vec_to[i],res); } } @@ -1598,7 +1603,7 @@ struct ConvertThroughParsing else { time_t res = 0; - parsed = tryParseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); + parsed = tryParseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone, overflow); convertFromTime(vec_to[i],res); } } @@ -1631,7 +1636,7 @@ struct ConvertThroughParsing } else { - parsed = tryParseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing); + parsed = tryParseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing, saturate_on_overflow); } } diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 7d0c89e43caad..8526315701bf7 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -47,6 +47,7 @@ namespace ErrorCodes extern const int TOO_DEEP_RECURSION; extern const int TOO_LARGE_STRING_SIZE; extern const int SYNTAX_ERROR; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; } /// Converts num_bytes hex-encoded bytes from src to dst in a single pass, folding validity into @@ -1695,8 +1696,9 @@ ReturnType readDateTimeTextFallback( second = (s[6] - '0') * 10 + (s[7] - '0'); } - if constexpr (throw_exception) + if (saturate_on_overflow) { + /// Use saturating version - makeDateTime saturates out-of-range years if (unlikely(year == 0)) datetime = 0; else @@ -1704,29 +1706,28 @@ ReturnType readDateTimeTextFallback( } else { - if (saturate_on_overflow) + /// Use non-saturating version - report out-of-range values instead of clamping them + auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second); + if (!datetime_maybe) { - /// Use saturating version - makeDateTime saturates out-of-range years - if (unlikely(year == 0)) - datetime = 0; + if constexpr (throw_exception) + throw Exception(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse DateTime"); else - datetime = makeDateTime(date_lut, year, month, day, hour, minute, second); - } - else - { - /// Use non-saturating version - return false for out-of-range values - auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second); - if (!datetime_maybe) return false; + } - if constexpr (!dt64_mode) + if constexpr (!dt64_mode) + { + if (*datetime_maybe < 0 || *datetime_maybe > static_cast(UINT32_MAX)) { - if (*datetime_maybe < 0 || *datetime_maybe > static_cast(UINT32_MAX)) + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type DateTime", *datetime_maybe); + else return false; } - - datetime = *datetime_maybe; } + + datetime = *datetime_maybe; } } else @@ -1758,6 +1759,8 @@ ReturnType readDateTimeTextFallback( return false; } + return checkParsedDateTimeRange(datetime, saturate_on_overflow); + } return ReturnType(true); @@ -2595,6 +2598,15 @@ namespace /// Scale `value` to whole seconds and clamp it to the `DateTime` range. The multiplication is bound-checked /// with truncating division rather than `common::mulOverflow`, a no-op stub for big-int types. +/// Whether `value` scaled to whole seconds fits the DateTime range, using the same truncating bound +bool datetimeSecondsInRange(Int128 value, UInt32 unread_scale) +{ + static constexpr Int128 max_seconds = 0xFFFFFFFF; + if (value < 0) + return false; + return value <= max_seconds / DecimalUtils::scaleMultiplier(unread_scale); +} + time_t datetimeSecondsFromNumber(Int128 value, UInt32 unread_scale) { static constexpr Int128 max_seconds = 0xFFFFFFFF; @@ -2619,7 +2631,7 @@ bool datetime64TicksFromNumber(DateTime64 & x, Int128 value, UInt32 unread_scale } template -ReturnType readDateTimeAsNumberImpl(time_t & x, ReadBuffer & buf) +ReturnType readDateTimeAsNumberImpl(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { static constexpr bool throw_exception = std::is_same_v; Decimal128 tmp; @@ -2639,12 +2651,20 @@ ReturnType readDateTimeAsNumberImpl(time_t & x, ReadBuffer & buf) else return ReturnType(false); } + if (!saturate_on_overflow && !datetimeSecondsInRange(tmp.value, unread_scale)) + { + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value is out of bounds of type DateTime"); + else + return ReturnType(false); + } + x = datetimeSecondsFromNumber(tmp.value, unread_scale); return ReturnType(true); } template -ReturnType readDateTimeAsRawValueImpl(time_t & x, ReadBuffer & buf) +ReturnType readDateTimeAsRawValueImpl(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { static constexpr bool throw_exception = std::is_same_v; /// Saturating 128-bit read: a plain `readIntText` does not check overflow, so an out-of-range value would @@ -2655,6 +2675,14 @@ ReturnType readDateTimeAsRawValueImpl(time_t & x, ReadBuffer & buf) else if (!readIntText128Saturating(tmp, buf)) return ReturnType(false); + if (!saturate_on_overflow && !datetimeSecondsInRange(tmp, 0)) + { + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value is out of bounds of type DateTime"); + else + return ReturnType(false); + } + x = datetimeSecondsFromNumber(tmp, 0); return ReturnType(true); } @@ -2685,6 +2713,7 @@ ReturnType readDateTime64AsNumberImpl(DateTime64 & x, UInt32 scale, ReadBuffer & else return ReturnType(false); } + return ReturnType(true); } @@ -2705,15 +2734,16 @@ ReturnType readDateTime64AsRawValueImpl(DateTime64 & x, ReadBuffer & buf) else return ReturnType(false); } + return ReturnType(true); } } -void readDateTimeAsNumber(time_t & x, ReadBuffer & buf) { readDateTimeAsNumberImpl(x, buf); } -bool tryReadDateTimeAsNumber(time_t & x, ReadBuffer & buf) { return readDateTimeAsNumberImpl(x, buf); } -void readDateTimeAsRawValue(time_t & x, ReadBuffer & buf) { readDateTimeAsRawValueImpl(x, buf); } -bool tryReadDateTimeAsRawValue(time_t & x, ReadBuffer & buf) { return readDateTimeAsRawValueImpl(x, buf); } +void readDateTimeAsNumber(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { readDateTimeAsNumberImpl(x, buf, saturate_on_overflow); } +bool tryReadDateTimeAsNumber(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { return readDateTimeAsNumberImpl(x, buf, saturate_on_overflow); } +void readDateTimeAsRawValue(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { readDateTimeAsRawValueImpl(x, buf, saturate_on_overflow); } +bool tryReadDateTimeAsRawValue(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { return readDateTimeAsRawValueImpl(x, buf, saturate_on_overflow); } void readDateTime64AsNumber(DateTime64 & x, UInt32 scale, ReadBuffer & buf) { readDateTime64AsNumberImpl(x, scale, buf); } bool tryReadDateTime64AsNumber(DateTime64 & x, UInt32 scale, ReadBuffer & buf) { return readDateTime64AsNumberImpl(x, scale, buf); } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 2d216684a75b0..173ebc3f9d6f7 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -57,6 +57,7 @@ namespace ErrorCodes extern const int TOO_LARGE_STRING_SIZE; extern const int TOO_LARGE_ARRAY_SIZE; extern const int SIZE_OF_FIXED_STRING_DOESNT_MATCH; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; } /// Helper functions for formatted input. @@ -645,39 +646,40 @@ inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLU LocalDate local_date; if constexpr (throw_exception) - { readDateTextImpl(local_date, buf, allowed_delimiters); + else if (!readDateTextImpl(local_date, buf, allowed_delimiters)) + return false; + + if (saturate_on_overflow) + { + /// Use saturating versions - makeDayNum saturates out-of-range years, convertToDayNum saturates to 0 or 0xFFFF ExtendedDayNum ret = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); convertToDayNum(date, ret); + return ReturnType(true); } - else + + auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); + if (!ret) { - if (!readDateTextImpl(local_date, buf, allowed_delimiters)) + if constexpr (throw_exception) + throw Exception(ErrorCodes::CANNOT_PARSE_DATE, "Cannot parse date"); + else return false; + } - if (saturate_on_overflow) - { - /// Use saturating versions - makeDayNum saturates out-of-range years, convertToDayNum saturates to 0 or 0xFFFF - ExtendedDayNum ret = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); - convertToDayNum(date, ret); - } + if (!tryToConvertToDayNum(date, *ret)) + { + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type Date", ret->toUnderType()); else - { - /// Use non-saturating versions - return false for out-of-range values - auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); - if (!ret) - return false; - - if (!tryToConvertToDayNum(date, *ret)) - return false; - } - - return true; + return false; } + + return ReturnType(true); } template -inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr) +inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr, bool saturate_on_overflow = true) { static constexpr bool throw_exception = std::is_same_v; @@ -688,6 +690,22 @@ inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, cons else if (!readDateTextImpl(local_date, buf, allowed_delimiters)) return false; + if (!saturate_on_overflow) + { + /// Every four-digit year fits into Date32, so only a calendar-invalid date can fail here + auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); + if (!ret) + { + if constexpr (throw_exception) + throw Exception(ErrorCodes::CANNOT_PARSE_DATE, "Cannot parse date"); + else + return false; + } + + date = *ret; + return ReturnType(true); + } + /// A calendar-invalid date (e.g. month 13) yields 1900-01-01 (-getDayNumOffsetEpoch(), -25567) for Date32 and 1970-01-01 for Date. date = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day(), -static_cast(getDayNumOffsetEpoch())); return ReturnType(true); @@ -699,14 +717,14 @@ inline void readDateText(LocalDate & date, ReadBuffer & buf) readDateTextImpl(date, buf); } -inline void readDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void readDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance(), bool saturate_on_overflow = true) { - readDateTextImpl(date, buf, date_lut); + readDateTextImpl(date, buf, date_lut, nullptr, saturate_on_overflow); } -inline void readDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void readDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance(), bool saturate_on_overflow = true) { - readDateTextImpl(date, buf, date_lut); + readDateTextImpl(date, buf, date_lut, nullptr, saturate_on_overflow); } inline bool tryReadDateText(LocalDate & date, ReadBuffer & buf, const char * allowed_delimiters = nullptr) @@ -719,9 +737,9 @@ inline bool tryReadDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & return readDateTextImpl(date, buf, time_zone, allowed_delimiters, saturate_on_overflow); } -inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr) +inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr, bool saturate_on_overflow = true) { - return readDateTextImpl(date, buf, time_zone, allowed_delimiters); + return readDateTextImpl(date, buf, time_zone, allowed_delimiters, saturate_on_overflow); } UUID parseUUID(std::span src); @@ -856,6 +874,23 @@ inline T parseFromStringWithoutAssertEOF(std::string_view str) template ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr, bool saturate_on_overflow = true); +/// A digit-only timestamp is read as a plain integer, so its range has to be checked separately +template +inline ReturnType checkParsedDateTimeRange(time_t datetime [[maybe_unused]], bool saturate_on_overflow [[maybe_unused]]) +{ + if constexpr (!dt64_mode) + { + if (!saturate_on_overflow && (datetime < 0 || datetime > static_cast(UINT32_MAX))) + { + if constexpr (std::is_same_v) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type DateTime", datetime); + else + return false; + } + } + return ReturnType(true); +} + template ReturnType readTimeTextFallback(time_t & time, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr); @@ -928,8 +963,9 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons second = (s[17] - '0') * 10 + (s[18] - '0'); } - if constexpr (throw_exception) + if (saturate_on_overflow) { + /// Use saturating version - makeDateTime saturates out-of-range years if (unlikely(year == 0)) datetime = 0; else @@ -937,30 +973,29 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons } else { - if (saturate_on_overflow) + /// Use non-saturating version - report out-of-range values instead of clamping them + auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second); + if (!datetime_maybe) { - /// Use saturating version - makeDateTime saturates out-of-range years - if (unlikely(year == 0)) - datetime = 0; + if constexpr (throw_exception) + throw Exception(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse datetime"); else - datetime = makeDateTime(date_lut, year, month, day, hour, minute, second); - } - else - { - /// Use non-saturating version - return false for out-of-range values - auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second); - if (!datetime_maybe) return false; + } - /// For usual DateTime check if value is within supported range - if constexpr (!dt64_mode) + /// For usual DateTime check if value is within supported range + if constexpr (!dt64_mode) + { + if (*datetime_maybe < 0 || *datetime_maybe > static_cast(UINT32_MAX)) { - if (*datetime_maybe < 0 || *datetime_maybe > static_cast(UINT32_MAX)) + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type DateTime", *datetime_maybe); + else return false; } - - datetime = *datetime_maybe; } + + datetime = *datetime_maybe; } if (dt_long) @@ -971,7 +1006,12 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons return ReturnType(true); } /// Why not readIntTextUnsafe? Because for needs of AdFox, parsing of unix timestamp with leading zeros is supported: 000...NNNN. - return readIntTextImpl(datetime, buf); + if constexpr (throw_exception) + readIntTextImpl(datetime, buf); + else if (!readIntTextImpl(datetime, buf)) + return false; + + return checkParsedDateTimeRange(datetime, saturate_on_overflow); } return readDateTimeTextFallback(datetime, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters, saturate_on_overflow); } @@ -1396,9 +1436,9 @@ inline ReturnType readTimeTextImpl(Time64 & time64, UInt32 scale, ReadBuffer & b return ReturnType(is_ok); } -inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) +inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), bool saturate_on_overflow = true) { - readDateTimeTextImpl(datetime, buf, time_zone); + readDateTimeTextImpl(datetime, buf, time_zone, nullptr, nullptr, saturate_on_overflow); } inline void readTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) @@ -1442,10 +1482,10 @@ inline bool tryReadTime64Text(Time64 & time64, UInt32 scale, ReadBuffer & buf, c /// `toDateTime64` and the `Values` format. Parsing stops at the first character that is not part of the /// number (e.g. the `,` or `}` following the value in JSON). The `AsRawValue` variants implement the legacy /// behavior, where the number is the raw underlying value. -void readDateTimeAsNumber(time_t & x, ReadBuffer & buf); -bool tryReadDateTimeAsNumber(time_t & x, ReadBuffer & buf); -void readDateTimeAsRawValue(time_t & x, ReadBuffer & buf); -bool tryReadDateTimeAsRawValue(time_t & x, ReadBuffer & buf); +void readDateTimeAsNumber(time_t & x, ReadBuffer & buf, bool saturate_on_overflow = true); +bool tryReadDateTimeAsNumber(time_t & x, ReadBuffer & buf, bool saturate_on_overflow = true); +void readDateTimeAsRawValue(time_t & x, ReadBuffer & buf, bool saturate_on_overflow = true); +bool tryReadDateTimeAsRawValue(time_t & x, ReadBuffer & buf, bool saturate_on_overflow = true); void readDateTime64AsNumber(DateTime64 & x, UInt32 scale, ReadBuffer & buf); bool tryReadDateTime64AsNumber(DateTime64 & x, UInt32 scale, ReadBuffer & buf); @@ -1667,8 +1707,8 @@ inline void readText(T & x, ReadBuffer & buf) { readFloatTextPrecise(x, buf); } inline void readText(String & x, ReadBuffer & buf) { readEscapedString(x, buf); } -inline void readText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { readDateText(x, buf, time_zone); } -inline bool tryReadText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { return tryReadDateText(x, buf, time_zone); } +inline void readText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), bool saturate_on_overflow = true) { readDateText(x, buf, time_zone, saturate_on_overflow); } +inline bool tryReadText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), bool saturate_on_overflow = true) { return tryReadDateText(x, buf, time_zone, nullptr, saturate_on_overflow); } inline void readText(LocalDate & x, ReadBuffer & buf) { readDateText(x, buf); } inline bool tryReadText(LocalDate & x, ReadBuffer & buf) { return tryReadDateText(x, buf); } @@ -1794,7 +1834,7 @@ inline ReturnType readCSVSimple(T & x, ReadBuffer & buf) // standalone overload for dates: to avoid instantiating DateLUTs while parsing other types template -inline ReturnType readCSVSimple(T & x, ReadBuffer & buf, const DateLUTImpl & time_zone) +inline ReturnType readCSVSimple(T & x, ReadBuffer & buf, const DateLUTImpl & time_zone, bool saturate_on_overflow = true) { static constexpr bool throw_exception = std::is_same_v; @@ -1811,8 +1851,8 @@ inline ReturnType readCSVSimple(T & x, ReadBuffer & buf, const DateLUTImpl & tim ++buf.position(); if constexpr (throw_exception) - readText(x, buf, time_zone); - else if (!tryReadText(x, buf, time_zone)) + readText(x, buf, time_zone, saturate_on_overflow); + else if (!tryReadText(x, buf, time_zone, saturate_on_overflow)) return ReturnType(false); if (maybe_quote == '\'' || maybe_quote == '\"') @@ -1853,8 +1893,8 @@ inline bool tryReadCSV(LocalDate & x, ReadBuffer & buf) { return readCSVSimple(x, buf); } -inline void readCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone) { readCSVSimple(x, buf, time_zone); } -inline bool tryReadCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone) { return readCSVSimple(x, buf, time_zone); } +inline void readCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone, bool saturate_on_overflow = true) { readCSVSimple(x, buf, time_zone, saturate_on_overflow); } +inline bool tryReadCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone, bool saturate_on_overflow = true) { return readCSVSimple(x, buf, time_zone, saturate_on_overflow); } inline void readCSV(LocalDateTime & x, ReadBuffer & buf) { readCSVSimple(x, buf); } inline bool tryReadCSV(LocalDateTime & x, ReadBuffer & buf) { return readCSVSimple(x, buf); } diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 6a41e80d165d0..1fb167e7267d6 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int CANNOT_PARSE_DATETIME; +extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; } @@ -99,7 +100,8 @@ ReturnType parseDateTimeBestEffortImpl( const DateLUTImpl & utc_time_zone, DateTimeSubsecondPart * fractional, const char * allowed_date_delimiters = nullptr, - bool * has_explicit_zero_year = nullptr) + bool * has_explicit_zero_year = nullptr, + DateTimeOverflow overflow = DateTimeOverflow::Saturate) { auto on_error = [&]( int error_code [[maybe_unused]], @@ -323,7 +325,7 @@ ReturnType parseDateTimeBestEffortImpl( /// Fractional part is not allowed. return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part"); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 16 && !year && !has_time) { @@ -339,7 +341,7 @@ ReturnType parseDateTimeBestEffortImpl( /// Fractional part is not allowed. return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part"); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 19 && !year && !has_time) { @@ -355,7 +357,7 @@ ReturnType parseDateTimeBestEffortImpl( /// Fractional part is not allowed. return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part"); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 10 && !year && !has_time) { @@ -374,7 +376,7 @@ ReturnType parseDateTimeBestEffortImpl( readDigits(digits, sizeof(digits), in))); readDecimalNumber(fractional->value, fractional->digits, digits); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 9 && !year && !has_time) { @@ -393,7 +395,7 @@ ReturnType parseDateTimeBestEffortImpl( readDigits(digits, sizeof(digits), in))); readDecimalNumber(fractional->value, fractional->digits, digits); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 14 && !year && !has_time) { @@ -935,6 +937,10 @@ ReturnType parseDateTimeBestEffortImpl( if (has_explicit_zero_year) *has_explicit_zero_year = zero_year_was_read; + /// Year 0000 is outside DateTime and the substitution below would hide that, which `throw` forbids + if (!is_64 && zero_year_was_read && overflow == DateTimeOverflow::Report) + return on_error(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Year 0000 is out of bounds of type DateTime"); + if constexpr (strict) return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: year is required"); @@ -996,7 +1002,8 @@ ReturnType parseDateTimeBestEffortImpl( } }; - if constexpr (!strict || std::is_same_v) + /// `strict` always range-checks, otherwise only when the caller asked not to saturate + if (overflow == DateTimeOverflow::Saturate && !(strict && std::is_same_v)) { if (has_time_zone_offset) { @@ -1008,52 +1015,33 @@ ReturnType parseDateTimeBestEffortImpl( res = local_time_zone.makeDateTime(year, month, day_of_month, hour, minute, second); } - if constexpr (std::is_same_v) - return true; + return ReturnType(true); } - else - { - if (has_time_zone_offset) - { - auto res_maybe = utc_time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second); - if (!res_maybe) - return false; - /// For usual DateTime check if value is within supported range - if constexpr (!is_64) - { - if (*res_maybe < 0 || *res_maybe > UINT32_MAX) - return false; - } - res = *res_maybe; - adjust_time_zone(); + const DateLUTImpl & time_zone = has_time_zone_offset ? utc_time_zone : local_time_zone; + auto res_maybe = time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second); + if (!res_maybe) + return on_error( + ErrorCodes::CANNOT_PARSE_DATETIME, + "Cannot read DateTime: unexpected date: {}-{}-{}", + year, + static_cast(month), + static_cast(day_of_month)); - /// After timezone adjustment, the value may have shifted outside the valid range. - /// For example, "2106-02-07 06:28:15-01:00" is within range before adjustment, - /// but after converting to UTC it exceeds UINT32_MAX. - if constexpr (!is_64) - { - if (res < 0 || static_cast(res) > UINT32_MAX) - return false; - } - } - else - { - auto res_maybe = local_time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second); - if (!res_maybe) - return false; + res = *res_maybe; - /// For usual DateTime check if value is within supported range - if constexpr (!is_64) - { - if (*res_maybe < 0 || *res_maybe > UINT32_MAX) - return false; - } - res = *res_maybe; - } + if (has_time_zone_offset) + adjust_time_zone(); - return true; + /// Only the adjusted value has to be in range: "2106-02-07 07:28:15+01:00" is past the maximum before the + /// offset is applied and is exactly the maximum after it, and the same holds at the lower bound. + if constexpr (!is_64) + { + if (res < 0 || res > UINT32_MAX) + return on_error(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type DateTime", res); } + + return ReturnType(true); } template @@ -1094,7 +1082,12 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); + parseDateTimeBestEffort(res, in, local_time_zone, utc_time_zone, DateTimeOverflow::Saturate); +} + +void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow) +{ + parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, nullptr, overflow); } void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, bool & has_explicit_zero_year) @@ -1103,19 +1096,19 @@ void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, &has_explicit_zero_year); } -void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow) { - parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); + parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, nullptr, overflow); } -bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow) { - return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); + return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, nullptr, overflow); } -bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow) { - return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); + return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, nullptr, overflow); } void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) diff --git a/src/IO/parseDateTimeBestEffort.h b/src/IO/parseDateTimeBestEffort.h index c42243b1efc05..a09d44de14823 100644 --- a/src/IO/parseDateTimeBestEffort.h +++ b/src/IO/parseDateTimeBestEffort.h @@ -55,8 +55,14 @@ class ReadBuffer; * Mon/Tue/Wed/Thu/Fri/Sat/Sun - simply ignored. */ +/// Whether an out-of-range result saturates to the bounds of the target type or is reported as an error +enum class DateTimeOverflow : uint8_t { Saturate, Report }; + void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); +/// Not defaulted, because the 4-argument form above would then be ambiguous with this one +void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow); + /// The same, but additionally reports whether the input contained an explicitly written year of `0000`. /// Such a year cannot be represented: internally a year field of `0` means "the year is not specified", /// so it is silently replaced with the current (or previous) year, and the returned value is then not the @@ -66,9 +72,9 @@ void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, bool & has_explicit_zero_year); void parseTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); -bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); -void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); -bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); +bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow = DateTimeOverflow::Saturate); +void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow = DateTimeOverflow::Saturate); +bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow = DateTimeOverflow::Saturate); bool tryParseTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); void parseTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); bool tryParseTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); diff --git a/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference new file mode 100644 index 0000000000000..a19952f23e73d --- /dev/null +++ b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference @@ -0,0 +1,42 @@ +saturate +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +2149-06-06 2149-06-06 2149-06-06 +2149-06-06 2149-06-06 +ignore +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +throw +throw, in range +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +2299-12-31 1900-01-01 2299-12-31 23:59:59.999 +throw, OrNull and OrZero still fall back +\N 1970-01-01 \N 1970-01-01 00:00:00 +throw, input formats +2149-06-06 +2106-02-07 06:28:15 +throw, tentative parsers must not accept a clamped value +2150-12-31 \N 2150-12-31 +2149-06-06 +throw, a digit-only timestamp is text too +2106-02-07 06:28:15 2023-11-14 22:13:20 \N 1970-01-01 00:00:00 +throw, Date32 rejects what it cannot represent instead of substituting a default +\N 2299-12-31 1900-01-01 +2000-13-01 +throw, an unquoted numeric token is checked too +2023-11-14 22:13:20 +2106-02-07 06:28:15 +2023-12-23 20:37:33 +throw, the last second keeps its fractional ticks +9999-12-31 23:59:59.500 +9999-12-31 23:59:59.999 +9999-12-31 23:59:59.500 +throw, the range is checked after the timezone offset is applied +2106-02-07 06:28:15 1970-01-01 00:00:00 +2106-02-07 06:28:15 2106-02-07 06:28:15 +throw, typed JSON columns are checked like declared ones +2149-06-06 +2299-12-31 23:59:59.999 +throw, JSONExtract keeps returning a default or NULL +1970-01-01 \N 2149-06-06 +throw, an explicitly written year 0000 is not silently replaced +\N 1970-01-01 00:00:00 +3 diff --git a/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql new file mode 100644 index 0000000000000..95e7f651a5034 --- /dev/null +++ b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql @@ -0,0 +1,105 @@ +-- date_time_overflow_behavior was ignored when the value came from text instead of a typed column +SET session_timezone = 'UTC'; + +SELECT 'saturate'; +SET date_time_overflow_behavior = 'saturate'; +SELECT toDate('9999-12-31'), toDate('1969-12-31'), toDateTime('2106-02-07 06:28:16'), toDateTime('1969-12-31 23:59:59'); +SELECT CAST('9999-12-31' AS Date), CAST(materialize('9999-12-31') AS Date), CAST('9999-12-31'::FixedString(10) AS Date); +SELECT toDateOrNull('9999-12-31'), toDateOrZero('9999-12-31'); + +SELECT 'ignore'; +SET date_time_overflow_behavior = 'ignore'; +SELECT toDate('9999-12-31'), toDate('1969-12-31'), toDateTime('2106-02-07 06:28:16'), toDateTime('1969-12-31 23:59:59'); + +SELECT 'throw'; +SET date_time_overflow_behavior = 'throw'; +SELECT toDate('2149-06-07'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDate('1969-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('2106-02-07 06:28:16'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('1969-12-31 23:59:59'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('2149-06-07' AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('2149-06-07') AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('2149-06-07'::FixedString(10) AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('2106-02-07 06:28:16') AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } + +SELECT 'throw, in range'; +SELECT toDate('2149-06-06'), toDate('1970-01-01'), toDateTime('2106-02-07 06:28:15'), toDateTime('1970-01-01 00:00:00'); +-- Date32 and DateTime64 are not covered yet: a high-scale DateTime64 text parse still raises DECIMAL_OVERFLOW +-- in every mode, because the tick range is only checked in DecimalUtils +SELECT toDate32('2299-12-31'), toDate32('1900-01-01'), toDateTime64('2299-12-31 23:59:59.999', 3); + +SELECT 'throw, OrNull and OrZero still fall back'; +SELECT toDateOrNull('2149-06-07'), toDateOrZero('2149-06-07'), toDateTimeOrNull('2106-02-07 06:28:16'), toDateTimeOrZero('1969-12-31 23:59:59'); + +SELECT 'throw, input formats'; +SELECT * FROM format(CSV, 'v Date', '2150-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v Date', '1960-01-01'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(TSV, 'v Date', '2150-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v Date', '{"v":"2150-12-31"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v DateTime', '2106-02-07 06:28:16'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(TSV, 'v DateTime', '1960-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"2106-02-07 06:28:16"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v Date', '2149-06-06'); +SELECT * FROM format(TSV, 'v DateTime', '2106-02-07 06:28:15'); + +SELECT 'throw, tentative parsers must not accept a clamped value'; +SELECT v, variantElement(v, 'Date') AS d, variantElement(v, 'String') AS s +FROM format(CSV, 'v Variant(Date, String)', '2150-12-31') SETTINGS allow_experimental_variant_type = 1; +SELECT v FROM format(CSV, 'v Variant(Date, String)', '2149-06-06') SETTINGS allow_experimental_variant_type = 1; + +SELECT 'throw, a digit-only timestamp is text too'; +SELECT toDateTime('4294967296'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('4294967296' AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('4294967296') AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"4294967296"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(Values, 'v DateTime', '(\'4294967296\')'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('4294967295'), toDateTime('1700000000'), toDateTimeOrNull('4294967296'), toDateTimeOrZero('4294967296'); + +SELECT 'throw, Date32 rejects what it cannot represent instead of substituting a default'; +SELECT toDate32('2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT toDate32('99999999'); -- { serverError CANNOT_PARSE_DATE } +SELECT CAST(materialize('2000-13-01') AS Date32); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(CSV, 'v Date32', '2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(TSV, 'v Date32', '2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(JSONEachRow, 'v Date32', '{"v":"2000-13-01"}'); -- { serverError CANNOT_PARSE_DATE } +SELECT toDate32OrNull('2000-13-01'), toDate32('2299-12-31'), toDate32('1900-01-01'); +SELECT v FROM format(CSV, 'v Variant(Date32, String)', '2000-13-01') SETTINGS allow_experimental_variant_type = 1; + +SELECT 'throw, an unquoted numeric token is checked too'; +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":4294967296}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":-1}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(Values, 'v DateTime', '(4294967296)'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":1700000000}'); +SELECT * FROM format(Values, 'v DateTime', '(4294967295)'); +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":1703363853.5}'); + +SELECT 'throw, the last second keeps its fractional ticks'; +SELECT * FROM format(JSONEachRow, 'v DateTime64(3)', '{"v":253402300799.5}'); +SELECT * FROM format(JSONEachRow, 'v DateTime64(3)', '{"v":253402300799.999}'); +SELECT * FROM format(Values, 'v DateTime64(3)', '(253402300799.5)'); + +SELECT 'throw, the range is checked after the timezone offset is applied'; +SELECT parseDateTimeBestEffort('2106-02-07 07:28:15+01:00', 'UTC'), parseDateTimeBestEffort('1969-12-31 23:00:00-01:00', 'UTC'); +SELECT parseDateTimeBestEffortOrNull('2106-02-07 07:28:15+01:00', 'UTC'), toDateTime('2106-02-07 07:28:15+01:00', 'UTC'); +SELECT parseDateTimeBestEffort('2106-02-07 08:28:15+01:00', 'UTC'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } + +SELECT 'throw, typed JSON columns are checked like declared ones'; +SELECT j.d FROM format(JSONEachRow, 'j JSON(d Date)', '{"j":{"d":"2150-12-31"}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":"2106-02-07 06:28:16"}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":"2106-02-07 06:28:16"}}') SETTINGS date_time_input_format = 'basic'; -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":4294967296}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d Date)', '{"j":{"d":"2149-06-06"}}'); +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime64(3))', '{"j":{"d":"2299-12-31 23:59:59.999"}}'); + +SELECT 'throw, JSONExtract keeps returning a default or NULL'; +SELECT JSONExtract('{"d":"2150-12-31"}', 'd', 'Date'), JSONExtract('{"d":"2150-12-31"}', 'd', 'Nullable(Date)'), JSONExtract('{"d":"2149-06-06"}', 'd', 'Date'); + +SELECT 'throw, an explicitly written year 0000 is not silently replaced'; +SELECT parseDateTimeBestEffort('0000-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffort('00000101'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffortUS('01/01/0000'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('0000-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"0000-01-01 00:00:00"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffortOrNull('0000-01-01 00:00:00'), parseDateTimeBestEffortOrZero('0000-01-01 00:00:00'); +-- An absent year is a documented best-effort feature, not an overflow +SELECT toMonth(parseDateTimeBestEffort('Mar 3 01:33:48')); From 2cbc5a63b0edc26b453bb46048a58c61e038fc50 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 13:32:16 +0000 Subject: [PATCH 061/296] Backport #120230 to 26.8: Fix a Redis full scan dropping rows, and an out-of-range access on a malformed MGET reply --- src/Storages/StorageRedis.cpp | 22 ++- .../test_storage_redis/fake_redis.py | 128 ++++++++++++++ tests/integration/test_storage_redis/test.py | 163 +++++++++++++++++- 3 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_storage_redis/fake_redis.py diff --git a/src/Storages/StorageRedis.cpp b/src/Storages/StorageRedis.cpp index b29f3c83332b9..04906d7878876 100644 --- a/src/Storages/StorageRedis.cpp +++ b/src/Storages/StorageRedis.cpp @@ -123,8 +123,13 @@ class RedisDataSource final : public ISource MutableColumns columns = sample_block.cloneEmptyColumns(); RedisArray values = storage.multiGet(scan_keys); - for (size_t i = 0; i < scan_keys.size() && !values.get(i).isNull(); i++) + for (size_t i = 0; i < scan_keys.size(); ++i) { + /// MGET answers by position, and a scanned key can hold another Redis type or expire + /// before the MGET runs, so a nil marks one absent value, not the end of the batch. + if (values.get(i).isNull()) + continue; + fillColumns(scan_keys.get(i).value(), values.get(i).value(), primary_key_pos, sample_block, columns @@ -461,7 +466,7 @@ Chunk StorageRedis::getBySerializedKeys(const RedisArray & keys, PaddedPODArray< "StorageRedis::getBySerializedKeys: null_map size {} does not match keys size {}", null_map->size(), keys.size()); - for (size_t i = 0; i < values.size(); ++i) + for (size_t i = 0; i < keys.size(); ++i) { if (null_map && !(*null_map)[i]) { @@ -517,7 +522,18 @@ RedisArray StorageRedis::multiGet(const RedisArray & keys) const for (size_t i = 0; i < keys.size(); ++i) cmd_mget.add(keys.get(i)); - return connection->client->execute(cmd_mget); + RedisArray values = connection->client->execute(cmd_mget); + + /// Callers pair the reply with the request by position, into arrays sized from `keys`. + if (values.isNull() || values.size() != keys.size()) + throw Exception( + ErrorCodes::INTERNAL_REDIS_ERROR, + "Redis table {} returned {} values for MGET of {} keys", + getStorageID().getFullNameNotQuoted(), + values.isNull() ? 0 : values.size(), + keys.size()); + + return values; } void StorageRedis::multiSet(const RedisArray & data) const diff --git a/tests/integration/test_storage_redis/fake_redis.py b/tests/integration/test_storage_redis/fake_redis.py new file mode 100644 index 0000000000000..2407443329828 --- /dev/null +++ b/tests/integration/test_storage_redis/fake_redis.py @@ -0,0 +1,128 @@ +""" +A minimal RESP server that answers MGET with a reply of the test's choosing. + +`StorageRedis` pairs the elements of an MGET reply with the requested keys by position, so both +the element count and the position of a nil decide what the engine does. Usage: + + fake_redis.py [ ] + +Every MGET is answered with `max(0, len(keys) + delta)` elements: the requested keys echoed back +as their own values, followed by RESP nils. `` and `` are comma-separated +key names; when they are given, SCAN answers with those keys and MGET answers nil for the names +in ``, so a full scan meets a fixed valid/nil sequence rather than Redis's own key +order. Anything else is answered `+OK`. +""" + +import socket +import sys +import threading + + +def read_exactly(stream, size): + data = stream.read(size) + if len(data) != size: + raise EOFError + return data + + +def read_line(stream): + line = stream.readline() + if not line: + raise EOFError + return line.rstrip(b"\r\n") + + +def read_command(stream): + """Read one RESP array. + + Argument payloads are read by declared length, never up to a newline: the keys on the wire + are ClickHouse's serializeBinary output and may contain \\r, \\n or non-UTF-8 bytes. + """ + header = read_line(stream) + if not header.startswith(b"*"): + return None + args = [] + for _ in range(int(header[1:])): + arg_header = read_line(stream) + if not arg_header.startswith(b"$"): + raise EOFError + args.append(read_exactly(stream, int(arg_header[1:]))) + read_exactly(stream, 2) # trailing CRLF + return args + + +def serialize_string(name): + """A String shorter than 128 bytes as ClickHouse serializes it: one length byte, then bytes. + + The engine deserializes the keys SCAN reports and the values MGET returns, so both have to + arrive in that form. See serialize_binary_for_string in test.py. + """ + return bytes([len(name)]) + name.encode() + + +def mget_reply(keys, delta, nil_keys): + count = max(0, len(keys) + delta) + out = [b"*%d\r\n" % count] + for i in range(count): + if i < len(keys) and keys[i] not in nil_keys: + out.append(b"$%d\r\n%s\r\n" % (len(keys[i]), keys[i])) + else: + out.append(b"$-1\r\n") + return b"".join(out) + + +def scan_reply(scan_keys): + """Cursor 0 with every key, so the engine reads the whole keyspace in one batch.""" + out = [b"*2\r\n$1\r\n0\r\n", b"*%d\r\n" % len(scan_keys)] + for key in scan_keys: + out.append(b"$%d\r\n%s\r\n" % (len(key), key)) + return b"".join(out) + + +def handle(conn, delta, scan_keys, nil_keys): + try: + with conn.makefile("rb") as stream: + while True: + args = read_command(stream) + if not args: + break + command = args[0].upper() + if command == b"MGET": + conn.sendall(mget_reply(args[1:], delta, nil_keys)) + elif command == b"SCAN" and scan_keys: + conn.sendall(scan_reply(scan_keys)) + else: + conn.sendall(b"+OK\r\n") + except (EOFError, OSError, ValueError): + pass + finally: + conn.close() + + +def parse_names(argument): + return [serialize_string(name) for name in argument.split(",") if name] + + +def main(): + port = int(sys.argv[1]) + delta = int(sys.argv[2]) + scan_keys = parse_names(sys.argv[3]) if len(sys.argv) > 3 else [] + nil_keys = parse_names(sys.argv[4]) if len(sys.argv) > 4 else [] + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("0.0.0.0", port)) + # The engine holds a connection pool and a direct join reads on several pipeline threads, so + # connections have to be served concurrently. + server.listen(64) + print(f"listening on {port}, MGET reply length = requested + {delta}", flush=True) + + while True: + conn, _ = server.accept() + threading.Thread( + target=handle, args=(conn, delta, scan_keys, nil_keys), daemon=True + ).start() + + +if __name__ == "__main__": + main() diff --git a/tests/integration/test_storage_redis/test.py b/tests/integration/test_storage_redis/test.py index af2892125fce3..8bf1f96526a71 100644 --- a/tests/integration/test_storage_redis/test.py +++ b/tests/integration/test_storage_redis/test.py @@ -1,5 +1,6 @@ ## sudo -H pip install redis import json +import os import struct import sys @@ -8,7 +9,9 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV +from helpers.test_tools import TSV, wait_condition + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) @@ -529,3 +532,161 @@ def check_query(query, read_type, keys_count, rows_read): plan = node.query("EXPLAIN actions=1, optimize=0 SELECT * FROM test_get_keys") assert 'ReadType: FullScan' in plan + + +# Ports of the three fake Redis endpoints, inside the ClickHouse container. +FAKE_REDIS_PORT_OK = 16379 +FAKE_REDIS_PORT_LONG = 16380 +FAKE_REDIS_PORT_SHORT = 16381 +FAKE_REDIS_PORT_SCAN = 16382 + + +def start_fake_redis(port, delta, keys=""): + node.exec_in_container( + [ + "bash", + "-c", + f"python3 /fake_redis.py {port} {delta} {keys}" + f" > /var/log/clickhouse-server/fake_redis_{port}.log 2>&1", + ], + detach=True, + user="root", + ) + wait_condition( + lambda: node.exec_in_container( + ["bash", "-c", f"exec 3<>/dev/tcp/127.0.0.1/{port} && echo OK"], + nothrow=True, + ), + lambda r: "OK" in r, + max_attempts=40, + delay=0.5, + ) + + +def test_malformed_mget_reply(started_cluster): + """An MGET reply whose element count differs from the request must be rejected. + + The result loop of StorageRedis was bounded by the reply length while the null map it + indexes is sized from the request, so an over-long reply read and wrote past the end of the + null map. A short reply produced fewer rows than keys, which breaks the row-per-key contract + that IKeyValueEntity::getByKeys promises to a direct join. + """ + tables = ("redis_fake_ok", "redis_fake_long", "redis_fake_short", "t_fake_left") + for table in tables: + drop_table(table) + + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_redis.py"), "/fake_redis.py" + ) + start_fake_redis(FAKE_REDIS_PORT_OK, 0) + start_fake_redis(FAKE_REDIS_PORT_LONG, 1) + start_fake_redis(FAKE_REDIS_PORT_SHORT, -1) + + node.query( + f""" + CREATE TABLE redis_fake_ok (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_OK}') PRIMARY KEY (key); + + CREATE TABLE redis_fake_long (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_LONG}') PRIMARY KEY (key); + + CREATE TABLE redis_fake_short (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_SHORT}') PRIMARY KEY (key); + + CREATE TABLE t_fake_left (k String) ENGINE = TinyLog; + INSERT INTO t_fake_left VALUES ('a'), ('b'); + """ + ) + + def direct_join(table): + return node.query( + f"SELECT key, value FROM (SELECT k AS key FROM t_fake_left) AS t " + f"INNER JOIN {table} USING (key) ORDER BY key " + f"SETTINGS join_algorithm = 'direct' FORMAT TSV" + ) + + # An endpoint that answers with one element per key still works, so a mock that never + # listens or frames RESP wrongly reddens here instead of green-washing the arms below. + assert TSV.toMat(direct_join("redis_fake_ok")) == [["a", "a"], ["b", "b"]] + + with pytest.raises(QueryRuntimeException) as long_join: + direct_join("redis_fake_long") + assert "INTERNAL_REDIS_ERROR" in str(long_join.value) + assert "for MGET of" in str(long_join.value) + + with pytest.raises(QueryRuntimeException) as short_join: + direct_join("redis_fake_short") + assert "INTERNAL_REDIS_ERROR" in str(short_join.value) + + # The other caller of the reply reads it with no null map at all. + with pytest.raises(QueryRuntimeException) as long_in: + node.query("SELECT * FROM redis_fake_long WHERE key IN ('a', 'b')") + assert "INTERNAL_REDIS_ERROR" in str(long_in.value) + + # A zero-element reply is a null array in Poco, not an empty one, so it is the isNull() + # term of the guard that rejects it. One key against the delta = -1 endpoint produces it. + with pytest.raises(QueryRuntimeException) as zero_in: + node.query("SELECT * FROM redis_fake_short WHERE key IN ('a')") + assert "INTERNAL_REDIS_ERROR" in str(zero_in.value) + assert "returned 0 values" in str(zero_in.value) + + for table in tables: + drop_table(table) + + +def test_full_scan_skips_missing_values(started_cluster): + """A full scan must skip the keys MGET answers with nil, not stop at the first one. + + MGET answers by position, and a key SCAN listed can hold a non-string type or expire + before the MGET runs, so a nil marks one absent value and not the end of the batch. + """ + address = get_address_for_ch() + table = "test_full_scan_missing" + fake_table = "redis_fake_scan" + + client = get_redis_connection(db_id=4) + client.flushdb() + drop_table(table) + drop_table(fake_table) + + # Redis alone decides in what order SCAN reports keys, so the mock pins the one thing this + # test is about: a nil arriving before a key that still has a value. + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_redis.py"), "/fake_redis.py" + ) + start_fake_redis(FAKE_REDIS_PORT_SCAN, 0, "a,b,c b") + + node.query( + f""" + CREATE TABLE {fake_table} (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_SCAN}') PRIMARY KEY (key); + """ + ) + rows = node.query(f"SELECT key, value FROM {fake_table} ORDER BY key FORMAT TSV") + assert TSV.toMat(rows) == [["a", "a"], ["c", "c"]] + + # The same thing on a real Redis, which answers nil for any key that holds another type. + node.query( + f""" + CREATE TABLE {table}(k String, v String) + Engine=Redis('{address}', 4, 'clickhouse') PRIMARY KEY (k); + + INSERT INTO {table} SELECT toString(number), toString(number) FROM numbers(16); + """ + ) + + # Control: every row is readable before the keys below exist, so a fixture that writes + # nothing reddens here instead of leaving the assertion after it vacuous. + assert int(node.query(f"SELECT uniqExact(k) FROM {table}")) == 16 + + for i in range(16): + client.rpush(f"list_{i}", "x") + + # SCAN can report a key twice while Redis rehashes and no read path dedupes the rows, so the + # oracle here is the key set: a lost key is the defect, a repeated one is not. + keys = node.query(f"SELECT DISTINCT k FROM {table} ORDER BY toUInt32(k) FORMAT TSV") + assert TSV.toMat(keys) == [[str(i)] for i in range(16)] + + client.flushdb() + drop_table(table) + drop_table(fake_table) From 5ea8ee9630fdbd7d5e4da40731805e735491fdfa Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 14:46:41 +0000 Subject: [PATCH 062/296] Backport #120713 to 26.8: Add a setting to disable reading through the executable table function and Executable/ExecutablePool tables --- .../experimental_settings_ignore.txt | 1 + src/Core/Settings.cpp | 5 ++ src/Core/SettingsChangesHistory.cpp | 1 + src/Storages/StorageExecutable.cpp | 8 ++ .../05218_allow_executable_tables.reference | 1 + .../05218_allow_executable_tables.sql | 35 +++++++++ ...e_tables_ddl_does_not_run_script.reference | 7 ++ ...ecutable_tables_ddl_does_not_run_script.sh | 73 +++++++++++++++++++ 8 files changed, 131 insertions(+) create mode 100644 tests/queries/0_stateless/05218_allow_executable_tables.reference create mode 100644 tests/queries/0_stateless/05218_allow_executable_tables.sql create mode 100644 tests/queries/0_stateless/05227_allow_executable_tables_ddl_does_not_run_script.reference create mode 100755 tests/queries/0_stateless/05227_allow_executable_tables_ddl_does_not_run_script.sh diff --git a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt index 4c4a0ec48a103..0b63ce3eb10b4 100644 --- a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt +++ b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt @@ -14,6 +14,7 @@ allow_deprecated_database_ordinary allow_deprecated_snowflake_conversion_functions allow_distributed_ddl allow_drop_detached +allow_executable_tables allow_execute_multiif_columnar allow_experimental_ai_functions allow_experimental_alter_materialized_view_structure diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 304d1177a1796..d405f501fe15a 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -4564,6 +4564,11 @@ Possible values: \ DECLARE(Bool, allow_execute_multiif_columnar, true, R"( Allow execute multiIf function columnar +)", 0) \ + DECLARE(Bool, allow_executable_tables, true, R"( +Allow reading through the `executable` table function and from `Executable` and `ExecutablePool` tables. + +Disabling this refuses reads only: creating, attaching, dropping and describing such tables still works. `ExecutablePool` processes that have already started are left running rather than terminated, but each read is still refused until the setting is enabled again. )", 0) \ DECLARE(Bool, formatdatetime_f_prints_single_zero, false, R"( Formatter '%f' in function 'formatDateTime' prints a single zero instead of six zeros if the formatted value has no fractional seconds. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a13257bba3538..89e1cfb534568 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -44,6 +44,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() addSettingsChanges(settings_changes_history, "26.8", { {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, + {"allow_executable_tables", true, true, "New setting to disable reading through the `executable` table function and from `Executable` and `ExecutablePool` tables."}, {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index 08922daedfd68..cd6eaf90454b2 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -37,6 +37,7 @@ namespace DB { namespace Setting { + extern const SettingsBool allow_executable_tables; extern const SettingsBool allow_experimental_analyzer; extern const SettingsSeconds max_execution_time; } @@ -58,6 +59,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int UNSUPPORTED_METHOD; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int SUPPORT_IS_DISABLED; } namespace @@ -163,6 +165,12 @@ void StorageExecutable::readImpl( size_t max_block_size, size_t /*threads*/) { + if (!context->getSettingsRef()[Setting::allow_executable_tables]) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "The `executable` table function and the `Executable` and `ExecutablePool` table " + "engines are disabled. Set `allow_executable_tables` setting to enable them"); + auto & script_name = settings->script_name; auto user_scripts_path = context->getUserScriptsPath(); diff --git a/tests/queries/0_stateless/05218_allow_executable_tables.reference b/tests/queries/0_stateless/05218_allow_executable_tables.reference new file mode 100644 index 0000000000000..604b153530121 --- /dev/null +++ b/tests/queries/0_stateless/05218_allow_executable_tables.reference @@ -0,0 +1 @@ +x UInt32 diff --git a/tests/queries/0_stateless/05218_allow_executable_tables.sql b/tests/queries/0_stateless/05218_allow_executable_tables.sql new file mode 100644 index 0000000000000..f56bfcf22caae --- /dev/null +++ b/tests/queries/0_stateless/05218_allow_executable_tables.sql @@ -0,0 +1,35 @@ +-- `allow_executable_tables` gates reading through the `executable` table function and from +-- `Executable` and `ExecutablePool` tables. + +SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError UNSUPPORTED_METHOD } + +SET allow_executable_tables = 0; +SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError SUPPORT_IS_DISABLED } +DESCRIBE executable('nonexist.sh', 'TSV', 'x UInt32'); + +-- A definition is still accepted and can be managed; only reading it is refused. +CREATE TABLE t_exec_gate (x UInt32) ENGINE = Executable('nonexist.sh', 'TSV'); +CREATE VIEW v_exec_gate AS SELECT * FROM t_exec_gate; +CREATE MATERIALIZED VIEW mv_exec_gate ENGINE = MergeTree ORDER BY x AS SELECT * FROM t_exec_gate; +CREATE TABLE t_exec_pool_gate (x UInt32) ENGINE = ExecutablePool('nonexist.sh', 'TSV'); +DETACH TABLE t_exec_gate; +ATTACH TABLE t_exec_gate; + +SELECT * FROM t_exec_gate; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM v_exec_gate; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM t_exec_pool_gate; -- { serverError SUPPORT_IS_DISABLED } + +-- The gate is evaluated on the node that runs the read, so a distributed wrapper does not lift it. +SELECT * FROM remote('127.0.0.1', executable('nonexist.sh', 'TSV', 'x UInt32')); -- { serverError SUPPORT_IS_DISABLED } + +-- A statement that reads as part of its own execution is refused too. +CREATE TABLE t_exec_gate_copy ENGINE = MergeTree ORDER BY x AS SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError SUPPORT_IS_DISABLED } +CREATE MATERIALIZED VIEW mv_exec_gate_populate ENGINE = MergeTree ORDER BY x POPULATE AS SELECT * FROM t_exec_gate; -- { serverError SUPPORT_IS_DISABLED } + +SET allow_executable_tables = 1; +SELECT * FROM t_exec_gate; -- { serverError UNSUPPORTED_METHOD } + +SET allow_executable_tables = 0; +DROP VIEW mv_exec_gate; +DROP VIEW v_exec_gate; +DROP TABLE t_exec_gate; diff --git a/tests/queries/0_stateless/05227_allow_executable_tables_ddl_does_not_run_script.reference b/tests/queries/0_stateless/05227_allow_executable_tables_ddl_does_not_run_script.reference new file mode 100644 index 0000000000000..b1b9467c5b0b7 --- /dev/null +++ b/tests/queries/0_stateless/05227_allow_executable_tables_ddl_does_not_run_script.reference @@ -0,0 +1,7 @@ +SUPPORT_IS_DISABLED +script did not run +1 +script ran +7 +7 +pool worker starts: 1 diff --git a/tests/queries/0_stateless/05227_allow_executable_tables_ddl_does_not_run_script.sh b/tests/queries/0_stateless/05227_allow_executable_tables_ddl_does_not_run_script.sh new file mode 100755 index 0000000000000..5f87c83191ce8 --- /dev/null +++ b/tests/queries/0_stateless/05227_allow_executable_tables_ddl_does_not_run_script.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Defining an `Executable` table must not run its script, and a closed `allow_executable_tables` +# must refuse a read before spawning the process. The script appends to a marker file, so any +# execution is observable. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SCRIPTS_DIR=$(mktemp -d "${CLICKHOUSE_TMP}/exec_gate_scripts_XXXXXX") +trap 'rm -rf "${SCRIPTS_DIR}"' EXIT + +MARKER="${SCRIPTS_DIR}/executed" +POOL_MARKER="${SCRIPTS_DIR}/pool_started" + +cat > "${SCRIPTS_DIR}/marker.sh" << SCRIPT +#!/usr/bin/env bash +echo ran >> '${MARKER}' +printf '1\n' +SCRIPT +chmod +x "${SCRIPTS_DIR}/marker.sh" + +cat > "${SCRIPTS_DIR}/marker_pool.sh" << SCRIPT +#!/usr/bin/env bash +echo ran >> '${POOL_MARKER}' +while IFS= read -r n +do + printf '%s\n' "\${n}" + IFS= read -r x + printf '%s\n' "\${x}" +done +SCRIPT +chmod +x "${SCRIPTS_DIR}/marker_pool.sh" + +CONFIG_FILE="${SCRIPTS_DIR}/local_config.xml" +cat > "${CONFIG_FILE}" << CONFIG + + ${SCRIPTS_DIR}/ + +CONFIG + +# Gate closed: the definition is accepted and the refused read spawns nothing. +$CLICKHOUSE_LOCAL --config-file="${CONFIG_FILE}" --query " +SET allow_executable_tables = 0; +CREATE TABLE t_gate (x UInt32) ENGINE = Executable('marker.sh', 'TSV'); +SELECT * FROM t_gate; +" 2>&1 | grep -o -m1 'SUPPORT_IS_DISABLED' + +# Gate open: defining and wrapping still run nothing. +$CLICKHOUSE_LOCAL --config-file="${CONFIG_FILE}" --query " +CREATE TABLE t_gate (x UInt32) ENGINE = Executable('marker.sh', 'TSV'); +CREATE VIEW v_gate AS SELECT * FROM t_gate; +" +if [ -e "${MARKER}" ]; then echo "script ran"; else echo "script did not run"; fi + +# Positive control: proves the marker is observable, so the check above is not vacuous. +$CLICKHOUSE_LOCAL --config-file="${CONFIG_FILE}" --query " +CREATE TABLE t_gate (x UInt32) ENGINE = Executable('marker.sh', 'TSV'); +SELECT * FROM t_gate; +" +if [ -e "${MARKER}" ]; then echo "script ran"; else echo "script did not run"; fi + +# A pooled worker started by a permitted read survives the gate closing: the refused read neither +# reuses nor respawns it, and reopening the gate serves from the same worker. +$CLICKHOUSE_LOCAL --config-file="${CONFIG_FILE}" --ignore-error --query " +CREATE TABLE t_pool (x UInt32) ENGINE = ExecutablePool('marker_pool.sh', 'TSV', (SELECT 7)) SETTINGS send_chunk_header = 1, pool_size = 1; +SELECT * FROM t_pool; +SET allow_executable_tables = 0; +SELECT * FROM t_pool; +SET allow_executable_tables = 1; +SELECT * FROM t_pool; +" < /dev/null 2>/dev/null +echo "pool worker starts: $(wc -l < "${POOL_MARKER}")" From 0c0d3e0519130ca1b1aa521f5537a58bfa98adcb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 15:56:57 +0000 Subject: [PATCH 063/296] Backport #121474 to 26.8: Hide the password of a cluster-name `remote()` call in the query log --- src/Parsers/FunctionSecretArgumentsFinder.cpp | 3 +- ...te_cluster_name_password_masking.reference | 13 ++++ ...7_remote_cluster_name_password_masking.sql | 59 +++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05237_remote_cluster_name_password_masking.reference create mode 100644 tests/queries/0_stateless/05237_remote_cluster_name_password_masking.sql diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 6215a2933f436..0275cd8165089 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -680,7 +680,8 @@ void FunctionSecretArgumentsFinder::findRemoteFunctionSecretArguments() { /// remote(named_collection, ..., password = 'password', ...) findSecretNamedArgument("password", 1); - return; + /// An identifier is also a cluster name when no such collection exists, and that form keeps the + /// password in a positional slot, so the walk below has to run for it too. } /// We're going to replace 'password' with '[HIDDEN'] for the following signatures: diff --git a/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.reference b/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.reference new file mode 100644 index 0000000000000..3b8835252a031 --- /dev/null +++ b/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.reference @@ -0,0 +1,13 @@ +1 +1 +1 +1 1 1 +SELECT 1 FROM remote(\'\', \'db\', \'t\', \'usr\', \'[HIDDEN]\') +SELECT 1 FROM remote(\'\', \'db\', \'t\', \'usr\', \'[HIDDEN]\', rand()) +SELECT 1 FROM remote(\'\', db.t, \'usr\', \'[HIDDEN]\') +SELECT 1 FROM remote(\'\', numbers(10), \'usr\', \'[HIDDEN]\') +SELECT 1 FROM remote(cluster_that_does_not_exist_05237, system, one, \'usr\', \'[HIDDEN]\') +SELECT 1 FROM remote(test_shard_localhost, system, one, \'usr\', \'[HIDDEN]\') +SELECT 1 FROM remote(test_shard_localhost, system.one, \'usr\', \'[HIDDEN]\') +SELECT 1 FROM remoteSecure(cluster_that_does_not_exist_05237, system, one, \'usr\', \'[HIDDEN]\') +1 1 1 1 1 diff --git a/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.sql b/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.sql new file mode 100644 index 0000000000000..c3deec41db3c5 --- /dev/null +++ b/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.sql @@ -0,0 +1,59 @@ +-- An identifier as the first argument of `remote` names either a named collection or a cluster +-- from the server configuration. In the cluster-name form the user and the password are ordinary +-- positional arguments, so the secret arguments finder has to run its positional walk for it too. + +-- These run: test_shard_localhost is a configured cluster, so the positional credentials are +-- parsed and then unused. +SELECT 1 FROM remote(test_shard_localhost, system, one, 'usr', 'SECRETPW1'); +SELECT 1 FROM remote(test_shard_localhost, system.one, 'usr', 'SECRETPW2'); + +-- An unknown cluster is reported only after the whole argument list has been parsed and logged. +SELECT 1 FROM remote(cluster_that_does_not_exist_05237, system, one, 'usr', 'SECRETPW3'); -- { serverError CLUSTER_DOESNT_EXIST } +SELECT 1 FROM remoteSecure(cluster_that_does_not_exist_05237, system, one, 'usr', 'SECRETPW4'); -- { serverError CLUSTER_DOESNT_EXIST } + +-- The addresses form, for comparison: `remote('')` is rejected in parseAddress, also after the +-- whole argument list has been parsed. +SELECT 1 FROM remote('', 'db', 't', 'usr', 'SECRETPW5'); -- { serverError BAD_ARGUMENTS } +SELECT 1 FROM remote('', db.t, 'usr', 'SECRETPW6'); -- { serverError BAD_ARGUMENTS } +SELECT 1 FROM remote('', numbers(10), 'usr', 'SECRETPW7'); -- { serverError BAD_ARGUMENTS } +SELECT 1 FROM remote('', 'db', 't', 'usr', 'SECRETPW8', rand()); -- { serverError BAD_ARGUMENTS } + +-- Not credentials, and they have to stay readable: a user name with no password after it, a +-- sharding key spelled as an equality whose left operand happens to be called `password`, and a +-- SETTINGS clause. +SELECT 1 FROM remote(test_shard_localhost, system, one, 'usr'); +SELECT 1 FROM remote('', 'db', 't', 'usr', password = 'NOT_A_CREDENTIAL'); -- { serverError BAD_ARGUMENTS } +SELECT 1 FROM remote('', 'db', 't', 'usr', rand(), SETTINGS skip_unavailable_shards = 1); -- { serverError BAD_ARGUMENTS } + +-- The analyzer keeps its own finder over the query tree, and `EXPLAIN QUERY TREE` renders through +-- it, so the same statement is masked on that path too. +SELECT countIf(explain LIKE '%[HIDDEN]%') = 1, countIf(explain LIKE '%SECRETPW9%') = 0, countIf(explain LIKE '%usr%') = 1 +FROM (EXPLAIN QUERY TREE SELECT 1 FROM remote(test_shard_localhost, system, one, 'usr', 'SECRETPW9')); + +SYSTEM FLUSH LOGS query_log; + +-- `system.query_log` holds the same masked text as `system.text_log` and the server log. The two +-- queries below exclude themselves through `system.query_log` and `EXPLAIN`, and they must not +-- filter on the secret: once it is masked, a secret-keyed filter matches nothing and passes +-- vacuously. +SELECT DISTINCT query FROM system.query_log +WHERE current_database = currentDatabase() + AND event_date >= yesterday() + AND query LIKE '%FROM remote%' + AND query NOT LIKE '%system.query_log%' + AND query NOT LIKE '%EXPLAIN%' + AND query LIKE '%[HIDDEN]%' +ORDER BY query; + +SELECT + uniqExact(query) = 11, + countIf(query LIKE '%SECRETPW%') = 0, + uniqExactIf(query, query LIKE '%[HIDDEN]%') = 8, + uniqExactIf(query, query LIKE '%NOT_A_CREDENTIAL%') = 1, + uniqExactIf(query, query LIKE '%skip_unavailable_shards%') = 1 +FROM system.query_log +WHERE current_database = currentDatabase() + AND event_date >= yesterday() + AND query LIKE '%FROM remote%' + AND query NOT LIKE '%system.query_log%' + AND query NOT LIKE '%EXPLAIN%'; From 414f313387dec19bdf8afb87df428a0c3f4d9ae2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 15:59:26 +0000 Subject: [PATCH 064/296] Backport #121134 to 26.8: Skip `Alias` tables in a database-wide `TRUNCATE` --- src/Storages/StorageAlias.h | 4 ++ ...s_skipped_by_truncate_all_tables.reference | 18 +++++++ ...3_alias_skipped_by_truncate_all_tables.sql | 51 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 tests/queries/0_stateless/05233_alias_skipped_by_truncate_all_tables.reference create mode 100644 tests/queries/0_stateless/05233_alias_skipped_by_truncate_all_tables.sql diff --git a/src/Storages/StorageAlias.h b/src/Storages/StorageAlias.h index fa0262c17b82a..3fa188702e069 100644 --- a/src/Storages/StorageAlias.h +++ b/src/Storages/StorageAlias.h @@ -35,6 +35,10 @@ class StorageAlias final : public IStorage, WithContext bool readsFromOtherTables() const override { return true; } + /// An `Alias` has no data of its own, so a bulk `TRUNCATE ALL TABLES` must skip it. + /// Only the bulk paths consult this; an explicit `TRUNCATE TABLE ` still truncates the target. + bool supportsTruncate() const override { return false; } + /// Get the target storage this alias points to StoragePtr getTargetTable(std::optional access_check = std::nullopt) const; StoragePtr tryGetTargetTable() const { return DatabaseCatalog::instance().tryGetTable(StorageID(target_database, target_table), getContext()); } diff --git a/tests/queries/0_stateless/05233_alias_skipped_by_truncate_all_tables.reference b/tests/queries/0_stateless/05233_alias_skipped_by_truncate_all_tables.reference new file mode 100644 index 0000000000000..922dbfa665083 --- /dev/null +++ b/tests/queries/0_stateless/05233_alias_skipped_by_truncate_all_tables.reference @@ -0,0 +1,18 @@ +-- fixture +al_1 Alias +al_2 Alias +al_3 Alias +al_4 Alias +al_5 Alias +al_6 Alias +al_7 Alias +al_8 Alias +j Join +-- truncate all tables: the target is emptied by its own entry +0 +-- an alias is not a data holder: truncating only the aliases truncates nothing +300 +-- an alias to a missing table no longer fails the whole statement +0 +-- the statement names one database, so a target in another one keeps its rows +300 diff --git a/tests/queries/0_stateless/05233_alias_skipped_by_truncate_all_tables.sql b/tests/queries/0_stateless/05233_alias_skipped_by_truncate_all_tables.sql new file mode 100644 index 0000000000000..e1fed47f288bd --- /dev/null +++ b/tests/queries/0_stateless/05233_alias_skipped_by_truncate_all_tables.sql @@ -0,0 +1,51 @@ +-- Tags: no-replicated-database +-- no-replicated-database: TRUNCATE ALL TABLES is not supported for Replicated databases. + +-- A database-wide TRUNCATE enqueues one task per table name, all sharing one query id. An `Alias` is +-- a name without data, so the alias entry and its target entry converge on one storage and both take +-- that storage's lock, which `RWLockImpl` rejects for a query id that already holds it. +-- +-- The target must stay non-MergeTree for this to be covered: two exclusive acquisitions abort, two +-- shared ones are ref-counted. `Join` is deliberate: the test runner rewrites +-- `Log`/`TinyLog`/`StripeLog`/`Memory` to `MergeTree` in some lanes, which would make this vacuous. + +CREATE TABLE j (k UInt64, v UInt64) ENGINE = Join(ANY, LEFT, k); +INSERT INTO j SELECT number, number FROM numbers(300); + +CREATE TABLE al_1 ENGINE = Alias(currentDatabase(), 'j'); +CREATE TABLE al_2 ENGINE = Alias(currentDatabase(), 'j'); +CREATE TABLE al_3 ENGINE = Alias(currentDatabase(), 'j'); +CREATE TABLE al_4 ENGINE = Alias(currentDatabase(), 'j'); +CREATE TABLE al_5 ENGINE = Alias(currentDatabase(), 'j'); +CREATE TABLE al_6 ENGINE = Alias(currentDatabase(), 'j'); +CREATE TABLE al_7 ENGINE = Alias(currentDatabase(), 'j'); +CREATE TABLE al_8 ENGINE = Alias(currentDatabase(), 'j'); + +-- Without this an arm could report the expected value while probing a plain table instead of an alias. +SELECT '-- fixture'; +SELECT name, engine FROM system.tables WHERE database = currentDatabase() ORDER BY name; + +SELECT '-- truncate all tables: the target is emptied by its own entry'; +TRUNCATE ALL TABLES FROM {CLICKHOUSE_DATABASE:Identifier}; +SELECT count() FROM j; + +SELECT '-- an alias is not a data holder: truncating only the aliases truncates nothing'; +INSERT INTO j SELECT number, number FROM numbers(300); +TRUNCATE TABLES FROM {CLICKHOUSE_DATABASE:Identifier} LIKE 'al%'; +SELECT count() FROM j; + +SELECT '-- an alias to a missing table no longer fails the whole statement'; +CREATE TABLE al_missing ENGINE = Alias(currentDatabase(), 'no_such_table'); +TRUNCATE ALL TABLES FROM {CLICKHOUSE_DATABASE:Identifier}; +SELECT count() FROM j; + +-- A `MergeTree` target on purpose: this arm's loss is silent, with no lock convergence and no error, +-- so it fails on its own rather than through the abort the arms above provoke. +SELECT '-- the statement names one database, so a target in another one keeps its rows'; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t (k UInt64) ENGINE = MergeTree ORDER BY k; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.t SELECT number FROM numbers(300); +CREATE TABLE al_cross ENGINE = Alias({CLICKHOUSE_DATABASE_1:String}, 't'); +TRUNCATE ALL TABLES FROM {CLICKHOUSE_DATABASE:Identifier}; +SELECT count() FROM {CLICKHOUSE_DATABASE_1:Identifier}.t; +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; From 2c89004b4d3d76173f07e5612f8f7abad86c9b27 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 16:37:26 +0000 Subject: [PATCH 065/296] Backport #118457 to 26.8: Fix Paimon incremental read failing with REPLICA_IS_ALREADY_ACTIVE after a Keeper session loss --- src/Common/FailPoint.cpp | 1 + .../DataLakes/Paimon/PaimonStreamState.cpp | 15 ++- .../test_paimon_incremental_read/test.py | 103 ++++++++++++++++++ 3 files changed, 117 insertions(+), 2 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 5df9d5d8f3b17..1cc541d930982 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -161,6 +161,7 @@ static struct InitFiu PAUSEABLE_ONCE(kafka2_remove_zk_before_get_children) \ PAUSEABLE_ONCE(kafka2_remove_zk_before_final_multi) \ PAUSEABLE_ONCE(keeper_map_delete_pause_before_multi) \ + PAUSEABLE_ONCE(paimon_incremental_read_pause_before_is_active_remove) \ PAUSEABLE(dummy_pausable_failpoint) \ PAUSEABLE(paimon_incremental_read_pause_after_watermark_commit) \ ONCE(execute_query_calling_empty_set_result_func_on_exception) \ diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonStreamState.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonStreamState.cpp index bda89875692ec..88ab26fb305f3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonStreamState.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonStreamState.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB { @@ -20,6 +21,11 @@ extern const int LOGICAL_ERROR; extern const int REPLICA_IS_ALREADY_ACTIVE; } +namespace FailPoints +{ +extern const char paimon_incremental_read_pause_before_is_active_remove[]; +} + PaimonStreamState::PaimonStreamState( zkutil::ZooKeeperPtr keeper_, const String & keeper_path_, @@ -162,14 +168,19 @@ bool PaimonStreamState::activate() { /// Stale node from our previous session — safe to reclaim. /// Use versioned delete (CAS) to guard against TOCTOU races. + FailPointInjection::pauseFailPoint( + FailPoints::paimon_incremental_read_pause_before_is_active_remove); auto remove_code = keeper->tryRemove(is_active_path, stat.version); - if (remove_code != Coordination::Error::ZOK) + /// ZNONODE is this removal's postcondition, not a failure: Keeper reaps the + /// ephemeral itself once the session that created it expires. + if (remove_code != Coordination::Error::ZOK && remove_code != Coordination::Error::ZNONODE) { LOG_WARNING(log, "Failed to remove stale is_active node at {} (code: {}). " "Will retry on next attempt.", is_active_path.string(), remove_code); return false; } - LOG_INFO(log, "Removed stale is_active node from previous session at {}", is_active_path.string()); + LOG_INFO(log, "Cleared stale is_active node from previous session at {} ({})", + is_active_path.string(), remove_code); } else { diff --git a/tests/integration/test_paimon_incremental_read/test.py b/tests/integration/test_paimon_incremental_read/test.py index 4e256a4a1de61..2ceb6c2987300 100644 --- a/tests/integration/test_paimon_incremental_read/test.py +++ b/tests/integration/test_paimon_incremental_read/test.py @@ -21,6 +21,7 @@ CH_MV_PAIMON_TABLE = "paimon_mv_source" CH_MV_MERGETREE_TABLE = "paimon_mv_dest" CH_MV_NAME = "paimon_refresh_mv" +CH_TABLE_NAME_ACTIVATE_RECLAIM = "paimon_inc_read_activate_reclaim" cluster = ClickHouseCluster(__file__) node = cluster.add_instance( @@ -425,3 +426,105 @@ def test_paimon_to_mergetree_via_refresh_mv(started_cluster): node.query(f"DROP VIEW IF EXISTS {CH_MV_NAME} SYNC;") node.query(f"DROP TABLE IF EXISTS {CH_MV_MERGETREE_TABLE} SYNC;") node.query(f"DROP TABLE IF EXISTS {CH_MV_PAIMON_TABLE} SYNC;") + + +def test_paimon_incremental_read_activate_tolerates_reaped_is_active(started_cluster): + """A read after a Keeper session loss reclaims the `is_active` marker its own + previous session left behind, and must survive Keeper reaping that ephemeral + concurrently: the marker is visible to the reclaim path's `tryGet` and already + gone by its versioned `tryRemove`, which then reports ZNONODE. The marker here + is fabricated because it reproduces exactly that observable state, whereas + waiting for the real reap to land inside a two-round-trip window would make the + test nondeterministic.""" + writer_container_id = cluster.get_instance_docker_id("paimon-incremental-writer") + + warehouse_name = "warehouse_activate_reclaim" + warehouse_uri = f"file://{USER_FILES_PATH}/{warehouse_name}/" + warehouse_dir = f"{USER_FILES_PATH}/{warehouse_name}" + table_path = f"{USER_FILES_PATH}/{warehouse_name}/test.db/test_table" + # Unique per run: committed_snapshot persists in Keeper, so a rerun against + # the same cluster must not inherit an earlier run's watermark. + keeper_path = f"/clickhouse/paimon_activate_reclaim_{uuid.uuid4().hex}" + is_active_path = f"{keeper_path}/replicas/r1/is_active" + failpoint = "paimon_incremental_read_pause_before_is_active_remove" + + _clean_warehouse(writer_container_id, warehouse_dir) + + # Warm-up commit (snapshot 1), consumed to establish the baseline. + _run_writer(writer_container_id, warehouse_uri=warehouse_uri, start_id=0, rows_per_commit=1, commit_times=1) + _create_clickhouse_table_for_paimon_incremental_read( + CH_TABLE_NAME_ACTIVATE_RECLAIM, table_path, keeper_path=keeper_path + ) + count_query = f"SELECT count() FROM {CH_TABLE_NAME_ACTIVATE_RECLAIM}" + _wait_until_query_result(count_query, "1\n", database="default") + _wait_until_query_result(count_query, "0\n", database="default") + + zk = cluster.get_kazoo_client("zoo1") + reader = None + reader_result = {} + try: + # Read the server's own marker instead of hardcoding its payload format. + identifier = zk.get(is_active_path)[0] + + # Snapshot 2: the batch the post-reconnect read must deliver. + _run_writer(writer_container_id, warehouse_uri=warehouse_uri, start_id=1, rows_per_commit=10, commit_times=1) + + session_query = ( + "SELECT client_id FROM system.zookeeper_connection WHERE name = 'default'" + ) + old_session = node.query(session_query) + # Finalizes the shared session, so the Keeper handle latched in + # PaimonStreamState stays expired and the next read takes the + # needsNewKeeper() branch that calls activate(). + node.query("SYSTEM RECONNECT ZOOKEEPER") + assert node.query(session_query) != old_session, ( + f"the Keeper session was not replaced (still {old_session!r})" + ) + + deadline = time.monotonic() + 60 + while zk.exists(is_active_path) is not None: + assert time.monotonic() < deadline, "the old session's is_active was never reaped" + time.sleep(0.5) + + # Persistent, not ephemeral: activate() never inspects stat.ephemeralOwner, + # so persistence is invisible to the code under test while keeping a second + # Keeper session out of the test's failure modes. + zk.create(is_active_path, identifier) + + node.query(f"SYSTEM ENABLE FAILPOINT {failpoint}") + reader = threading.Thread( + target=lambda: reader_result.update( + zip(("out", "err"), node.query_and_get_answer_with_error(count_query)) + ) + ) + reader.start() + + # Returns only once a thread has parked at the failpoint. Since the + # failpoint sits inside the identifier-matched branch, that proves the read + # entered the reclaim path with a marker it recognises as its own. + node.query(f"SYSTEM WAIT FAILPOINT {failpoint} PAUSE", timeout=60) + assert reader.is_alive(), ( + f"the reader returned before parking at the failpoint: {reader_result!r}" + ) + + # The reap lands inside the read's tryGet/tryRemove window. + zk.delete(is_active_path) + node.query(f"SYSTEM NOTIFY FAILPOINT {failpoint}") + + reader.join(timeout=120) + assert not reader.is_alive(), "the reader thread never finished" + assert not reader_result.get("err"), f"the read failed: {reader_result!r}" + # Not just "did not throw": the pending snapshot must still be delivered. + assert reader_result.get("out") == "10\n", ( + f"the pending snapshot was not delivered: {reader_result!r}" + ) + assert zk.get(f"{keeper_path}/committed_snapshot")[0] == b"2" + _wait_until_query_result(count_query, "0\n", database="default") + finally: + # The failpoint is process-global: an early failure must leave neither it + # armed nor the reader thread parked on it. + node.query(f"SYSTEM DISABLE FAILPOINT {failpoint}") + zk.stop() + if reader is not None: + reader.join(timeout=60) + node.query(f"DROP TABLE IF EXISTS {CH_TABLE_NAME_ACTIVATE_RECLAIM} SYNC;") From 32d9c0dcf6f6655396e77a7a4caa97911b3dfc27 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2026 17:15:36 +0000 Subject: [PATCH 066/296] Backport #121402 to 26.8: Fix OOB write in the stem function --- src/Functions/stem.cpp | 8 +++++-- .../queries/0_stateless/01890_stem.reference | 11 ++++++++++ tests/queries/0_stateless/01890_stem.sql | 21 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/Functions/stem.cpp b/src/Functions/stem.cpp index e732014562fba..c8784dfdebfed 100644 --- a/src/Functions/stem.cpp +++ b/src/Functions/stem.cpp @@ -99,9 +99,10 @@ class Stemmer /// Rows where null_map[i] != 0 are skipped and emitted as empty strings. /// For FixedString, getDataAt returns the value with null-byte padding; trimRight removes it. /// For String, trailing zero bytes are valid data and must not be trimmed. - /// Snowball stemming never lengthens a word, so upper_bound bytes is a safe pre-allocation. MutableColumnPtr stemColumn(const IColumn & col, size_t input_rows_count, const NullMap * null_map = nullptr) { + /// upper_bound is only an initial estimate: some stemmers lengthen a word (e.g. Turkish maps + /// the ASCII 'i' to the 2-byte 'ı'), so the loop grows res_data when the output overflows it. size_t upper_bound = 0; const bool is_fixed_string = checkAndGetColumn(&col) != nullptr; if (const auto * col_str = checkAndGetColumn(&col)) @@ -131,7 +132,10 @@ class Stemmer if (is_fixed_string) trimRight(word, '\0'); std::string_view stemmed = stem(word); - chassert(data_size + stemmed.size() <= res_data.size()); + + /// Stemming can lengthen the word, grow the output buffer. + if (data_size + stemmed.size() > res_data.size()) + res_data.resize(data_size + stemmed.size()); memcpy(res_data.data() + data_size, stemmed.data(), stemmed.size()); data_size += stemmed.size(); diff --git a/tests/queries/0_stateless/01890_stem.reference b/tests/queries/0_stateless/01890_stem.reference index 7b5f7af344898..6b0de99cce390 100644 --- a/tests/queries/0_stateless/01890_stem.reference +++ b/tests/queries/0_stateless/01890_stem.reference @@ -103,6 +103,17 @@ run bless bless disguis +- Lengthening stemmers. +-- Turkish stems a 3-byte word to a 5-byte word, so the output is longer than the input. +uagı +-- Over a multi-block scan the output must stay correct even when it overflows the input-sized estimate. +0 +-- Turkish also lengthens via a multi-byte substitution (3 bytes to 4 bytes). +aaç +-- Estonian lengthens by appending ASCII letters (4 bytes to 5 bytes), a different mechanism. +keesi +-- Estonian over a multi-block scan must also stay correct past the input-sized estimate. +0 - Negative tests. -- Whitespace in a String input raises BAD_ARGUMENTS. -- Whitespace in an Array element raises BAD_ARGUMENTS. diff --git a/tests/queries/0_stateless/01890_stem.sql b/tests/queries/0_stateless/01890_stem.sql index cc5dd09fac708..c678bc88fb960 100644 --- a/tests/queries/0_stateless/01890_stem.sql +++ b/tests/queries/0_stateless/01890_stem.sql @@ -144,6 +144,27 @@ INSERT INTO stem_test_lc VALUES ('blessing'), ('disguise'), ('blessing'); SELECT stem(word, 'en') FROM stem_test_lc ORDER BY word; DROP TABLE stem_test_lc; +SELECT '- Lengthening stemmers.'; + +SELECT '-- Turkish stems a 3-byte word to a 5-byte word, so the output is longer than the input.'; +SELECT stem('uag', 'tr'); + +SELECT '-- Over a multi-block scan the output must stay correct even when it overflows the input-sized estimate.'; +SELECT countIf(s != 'uagı') +FROM (SELECT stem(materialize('uag'), 'tr') AS s FROM numbers(6800)) +SETTINGS max_block_size = 1700; + +SELECT '-- Turkish also lengthens via a multi-byte substitution (3 bytes to 4 bytes).'; +SELECT stem('aac', 'tr'); + +SELECT '-- Estonian lengthens by appending ASCII letters (4 bytes to 5 bytes), a different mechanism.'; +SELECT stem('keeb', 'et'); + +SELECT '-- Estonian over a multi-block scan must also stay correct past the input-sized estimate.'; +SELECT countIf(s != 'keesi') +FROM (SELECT stem(materialize('keeb'), 'et') AS s FROM numbers(6800)) +SETTINGS max_block_size = 1700; + SELECT '- Negative tests.'; SELECT '-- Whitespace in a String input raises BAD_ARGUMENTS.'; From b4bef00062dbf355c3ffe753010b371b1fc148ef Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 22 Sep 2026 23:05:02 +0200 Subject: [PATCH 067/296] Keep external roles when copying a Context Since https://github.com/ClickHouse/ClickHouse/pull/110867 a remote node of a secret interserver query enables the initiator's current roles only as external roles and clears the user's default roles. The `ContextData` copy constructor did not copy `external_roles`, so any copied context that recalculated access (for example, a view with an access-related setting in its `SETTINGS` clause) lost all roles and a user whose grants come from roles got `ACCESS_DENIED` on the remote node. This is the relevant part of https://github.com/ClickHouse/ClickHouse/pull/110144 (master): copy `external_roles` and make `setExternalRolesWithLock` replace the list instead of appending, so that `setUser` on a copied context (e.g. `EXECUTE AS`) does not inherit the pushed roles. --- src/Interpreters/Context.cpp | 20 ++++---- ...ushed_roles_survive_context_copy.reference | 6 +++ ...erver_pushed_roles_survive_context_copy.sh | 51 +++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05238_interserver_pushed_roles_survive_context_copy.reference create mode 100755 tests/queries/0_stateless/05238_interserver_pushed_roles_survive_context_copy.sh diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 756c5ee47bcdd..5649d6f637b97 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1369,6 +1369,7 @@ ContextData::ContextData(const ContextData &o) : input_blocks_reader(o.input_blocks_reader), user_id(o.user_id), current_roles(o.current_roles), + external_roles(o.external_roles), settings_constraints_and_current_profiles(o.settings_constraints_and_current_profiles), access(o.access), need_recalculate_access(o.need_recalculate_access), @@ -2231,15 +2232,16 @@ void Context::setCurrentRolesWithLock(const std::vector & new_current_role void Context::setExternalRolesWithLock(const std::vector & new_external_roles, const std::lock_guard &) { - // External roles are roles received from other node, current roles is a collection of roles that were assigned locally - if (!new_external_roles.empty()) - { - if (external_roles) - external_roles->insert(external_roles->end(), new_external_roles.begin(), new_external_roles.end()); - else - external_roles = std::make_shared>(new_external_roles); - need_recalculate_access = true; - } + // External roles are roles received from another node; current roles is a collection of roles that were assigned locally. + // Replace them unconditionally (rather than append) so that switching the principal via `setUser` clears any external + // roles carried over from a previous principal on the same or a copied context. `ContextData`'s copy constructor + // preserves `external_roles`, so without this reset a context authenticated with pushed roles would keep them after + // `setUser(target_user)` (e.g. `EXECUTE AS target_user`), silently widening the target's privileges. + if (new_external_roles.empty()) + external_roles = nullptr; + else + external_roles = std::make_shared>(new_external_roles); + need_recalculate_access = true; } void Context::setCurrentRolesImpl(const std::vector & new_current_roles, bool throw_if_not_granted, bool skip_if_not_granted, const std::shared_ptr & user) diff --git a/tests/queries/0_stateless/05238_interserver_pushed_roles_survive_context_copy.reference b/tests/queries/0_stateless/05238_interserver_pushed_roles_survive_context_copy.reference new file mode 100644 index 0000000000000..a0fd5153ac79f --- /dev/null +++ b/tests/queries/0_stateless/05238_interserver_pushed_roles_survive_context_copy.reference @@ -0,0 +1,6 @@ +-- remote read of a table +90 +-- remote read of a view with SETTINGS +90 +-- remote read of a view with SETTINGS, explicit SET ROLE +90 diff --git a/tests/queries/0_stateless/05238_interserver_pushed_roles_survive_context_copy.sh b/tests/queries/0_stateless/05238_interserver_pushed_roles_survive_context_copy.sh new file mode 100755 index 0000000000000..fd1f2c002b055 --- /dev/null +++ b/tests/queries/0_stateless/05238_interserver_pushed_roles_survive_context_copy.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# On a secret interserver query the remote node enables the initiator's current roles as external roles and +# drops the user's default roles. The external roles must survive `Context::createCopy`: a copied context that +# recalculates access (here, a view with an access-related setting in its `SETTINGS` clause) must not lose them, +# otherwise a user whose grants come only from roles gets `ACCESS_DENIED` on the remote node. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER="user_${CLICKHOUSE_DATABASE}" +ROLE="role_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -m -q " +DROP TABLE IF EXISTS t; +CREATE TABLE t (x UInt32) ENGINE = MergeTree ORDER BY x; +INSERT INTO t SELECT number FROM numbers(10); +CREATE VIEW v AS SELECT x FROM t SETTINGS allow_introspection_functions = 1; +CREATE TABLE t_dist AS t ENGINE = Distributed(test_cluster_interserver_secret, ${CLICKHOUSE_DATABASE}, t); +CREATE TABLE v_dist AS t ENGINE = Distributed(test_cluster_interserver_secret, ${CLICKHOUSE_DATABASE}, v); + +DROP ROLE IF EXISTS ${ROLE}; +CREATE ROLE ${ROLE}; +GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${ROLE}; +DROP USER IF EXISTS ${USER}; +CREATE USER ${USER} IDENTIFIED WITH no_password SETTINGS readonly = 0; +GRANT ${ROLE} TO ${USER}; +ALTER USER ${USER} DEFAULT ROLE ${ROLE}; +" + +echo "-- remote read of a table" +$CLICKHOUSE_CLIENT --user "${USER}" -q "SELECT sum(x) FROM t_dist SETTINGS prefer_localhost_replica = 0" + +echo "-- remote read of a view with SETTINGS" +$CLICKHOUSE_CLIENT --user "${USER}" -q "SELECT sum(x) FROM v_dist SETTINGS prefer_localhost_replica = 0" + +echo "-- remote read of a view with SETTINGS, explicit SET ROLE" +$CLICKHOUSE_CLIENT --user "${USER}" -m -q " +SET ROLE ${ROLE}; +SELECT sum(x) FROM v_dist SETTINGS prefer_localhost_replica = 0; +" + +$CLICKHOUSE_CLIENT -m -q " +DROP TABLE IF EXISTS v_dist; +DROP TABLE IF EXISTS t_dist; +DROP VIEW IF EXISTS v; +DROP TABLE IF EXISTS t; +DROP USER IF EXISTS ${USER}; +DROP ROLE IF EXISTS ${ROLE}; +" From c3b63d002dbd5dd2ae6ff82a728db1d63798a5f7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 01:47:27 +0000 Subject: [PATCH 068/296] Backport #121424 to 26.8: Disable the AI function per-query quotas by default --- src/Core/Settings.cpp | 14 +-- src/Core/SettingsChangesHistory.cpp | 102 ++++++++++++++++++ tests/integration/test_ai_functions/test.py | 4 +- .../0_stateless/03300_ai_functions.reference | 6 +- ...functions_compatibility_defaults.reference | 2 + ...28_ai_functions_compatibility_defaults.sql | 15 ++- 6 files changed, 127 insertions(+), 16 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 304d1177a1796..b762ebe4ed938 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8736,21 +8736,21 @@ Initial delay in milliseconds before the first retry of a failed AI function API DECLARE(Bool, ai_function_throw_on_error, true, R"( If true (default), an AI function call that fails permanently after exhausting all retries aborts the query with an exception. If false, the failed row receives the default value for the column type (empty string for String) and processing continues. )", BETA) \ - DECLARE(UInt64, ai_function_max_input_tokens_per_query, 1000000, R"( -Maximum total input (prompt) tokens across all AI function API calls in a single query. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of input tokens per in-flight request, since a call's input tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. + DECLARE(UInt64, ai_function_max_input_tokens_per_query, 0, R"( +Maximum total input (prompt) tokens across all AI function API calls in a single query. 0 (default) disables the limit. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of input tokens per in-flight request, since a call's input tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. This limit is only enforced for providers that report a `usage` object in their response (OpenAI, Anthropic, vLLM). Providers that omit token usage (notably HuggingFace TEI) cause the counter to stay at 0 — use `ai_function_max_api_calls_per_query` instead to bound such calls. )", BETA) \ - DECLARE(UInt64, ai_function_max_output_tokens_per_query, 500000, R"( -Maximum total output (completion) tokens across all AI function API calls in a single query. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of output tokens per in-flight request, since a call's output tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. + DECLARE(UInt64, ai_function_max_output_tokens_per_query, 0, R"( +Maximum total output (completion) tokens across all AI function API calls in a single query. 0 (default) disables the limit. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of output tokens per in-flight request, since a call's output tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. This limit is only enforced for providers that report a `usage` object in their response (OpenAI, Anthropic, vLLM). It does not apply to the embedding functions (`aiEmbed`, `aiSimilarity`), which never produce output tokens. )", BETA) \ - DECLARE(UInt64, ai_function_max_api_calls_per_query, 1000, R"( -Maximum number of HTTP requests that AI functions may dispatch per query. Enforced independently by each server and query fragment: within one execution context it is an exact cap shared by every AI function, block, and thread there, but a distributed query (across shards or parallel-replica fragments) may dispatch up to this many requests per shard/fragment. It must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. + DECLARE(UInt64, ai_function_max_api_calls_per_query, 0, R"( +Maximum number of HTTP requests that AI functions may dispatch per query. 0 (default) disables the limit. Enforced independently by each server and query fragment: within one execution context it is an exact cap shared by every AI function, block, and thread there, but a distributed query (across shards or parallel-replica fragments) may dispatch up to this many requests per shard/fragment. It must be set in the top-level query - a sub-query `SETTINGS` override is ignored. )", BETA) \ DECLARE(Bool, ai_function_throw_on_quota_exceeded, true, R"( -If true (default), exceeding an AI function quota limit (`ai_function_max_input_tokens_per_query`, `ai_function_max_output_tokens_per_query`, or `ai_function_max_api_calls_per_query`) aborts the query with an exception. If false, remaining rows receive the default value for the column type (empty string for String). Like the quota limits, this must be set in the top-level query - a sub-query `SETTINGS` override is ignored. +If true (default), exceeding an AI function quota limit (`ai_function_max_input_tokens_per_query`, `ai_function_max_output_tokens_per_query`, or `ai_function_max_api_calls_per_query`) aborts the query with an exception. All three limits are disabled by default, so this has no effect until one of them is set. If false, remaining rows receive the default value for the column type (empty string for String). Like the quota limits, this must be set in the top-level query - a sub-query `SETTINGS` override is ignored. )", BETA) \ DECLARE(NonZeroUInt64, ai_function_embedding_max_batch_size, 100, R"( Maximum number of texts to include in a single HTTP request made by the embedding functions (`aiEmbed`, `aiSimilarity`). Texts are grouped into batches of this size to reduce API call overhead. For example, 500 unique texts with a batch size of 100 result in 5 HTTP requests. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a13257bba3538..97a310296201d 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -41,6 +41,108 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "26.10", + { + {"allow_executable_tables", true, true, "New setting to disable reading through the `executable` table function and from `Executable` and `ExecutablePool` tables."}, + {"ai_function_max_input_tokens_per_query", 1000000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, + {"ai_function_max_output_tokens_per_query", 500000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, + {"ai_function_max_api_calls_per_query", 1000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, + {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema and log a warning instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.10 restores the previous strict behavior."}, + {"legacy_join_size_limits_trigger_spilling", true, false, "`max_rows_in_join` / `max_bytes_in_join` are now hard caps for every hash join, including the ones that spill to disk, where they used to act as the spill trigger. Spilling is driven by `max_bytes_before_external_join` / `max_bytes_ratio_before_external_join` alone."}, + {"prefer_optimize_projection", false, false, "New setting: choose a usable projection regardless of its estimated cost, like `force_optimize_projection`, but without failing the query when no projection is used."}, + {"query_plan_optimize_join_order_conflict_detector", "", "", "New setting selecting the conflict detector that decides join reordering validity in the DPsub join order algorithm: `a` for the (correct but incomplete) CD-A, `c` for the (correct and complete) CD-C, empty for none."}, + {"use_text_index_postings_cache", false, true, "Enabled the text index posting lists cache globally. Previously each query used a small private cache, which caused posting lists and phrase search results to be recomputed within a single query on large tables."}, + {"output_format_arrow_unsupported_types", "binary", "binary", "New setting superseding `output_format_arrow_unsupported_types_as_binary`, adding a `text` mode. Its default matches the previous behavior, so `compatibility` must not change it."}, + }); + addSettingsChanges(settings_changes_history, "26.9", + { + {"max_bytes_before_external_distinct", 0, 0, "New setting to enable spilling of `DISTINCT` to disk when memory usage exceeds the given threshold in bytes. If 0, only `max_bytes_ratio_before_external_distinct` applies."}, + {"max_bytes_ratio_before_external_distinct", 0., 0.5, "New setting to enable spilling of `DISTINCT` to disk when memory usage exceeds the given ratio of available memory. If 0, only `max_bytes_before_external_distinct` applies."}, + {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7 or 26.8: the previous value is deliberately equal to the new one, because those versions already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, + {"allow_delta_lake_create_table", false, false, "New setting: allow creating a new DeltaLake table using delta-kernel-rs or registering an existing one into a catalog."}, + {"delta_lake_accurate_write_cast", false, true, "New setting: cast written values to the Delta write-schema type with an accurate cast that throws on a value that does not fit the target type instead of silently truncating; `compatibility` below 26.9 uses the plain, non-throwing cast."}, + {"allow_experimental_nullable_tuple_type", false, true, "`Nullable(Tuple)` is now GA"}, + {"enable_nullable_tuple_type", false, true, "`Nullable(Tuple)` is now GA"}, + {"allow_nullable_tuple_in_extracted_subcolumns", false, true, "`Nullable(Tuple)` is now GA: a `Tuple` subcolumn extracted from a `Tuple`, `Variant`, `Dynamic` or `JSON` column is `Nullable(Tuple)` and is NULL in the rows where the subcolumn is missing. The setting is read once at server startup, so `compatibility` restores the previous behavior only from the startup profile (for example, users.xml), not from a session-level `SET`."}, + {"workload_admission_timeout_ms", 0, 0, "New setting bounding how long a query waits to be admitted by workload scheduling (acquiring its query slot and memory reservation) before failing; 0 (default) preserves the previous unbounded wait."}, + {"s3_disable_checksum", false, false, "Obsolete setting: checksum calculation no longer re-reads the source"}, + {"session_query_ids_history_size", 0, 1000, "New setting limiting the size of the session-local query id history exposed through the new `system.session_query_ids` system table. The previous value `0` (recording disabled) reproduces the pre-26.9 behavior."}, + {"query_plan_optimize_join_order_conflict_detector", "", "", "New setting selecting the conflict detector that decides join reordering validity in the DPsub join order algorithm: `a` for the (correct but incomplete) CD-A, `c` for the (correct and complete) CD-C, empty for none."}, + {"reader_executor_window_size", 4194304, 8388608, "Raised the default read window of the experimental `ReaderExecutor` from 4 MiB to 8 MiB. Under memory pressure the window is reduced from this base, floored at 128 KiB."}, + {"webassembly_udf_input_split_memory_ratio", 0.0, 0.5, "New setting controlling the fraction of a WebAssembly UDF instance's linear memory that one call's serialized input may occupy, which also enables the dynamic splitting of that input by its serialized size; `compatibility` below 26.9 sets it to 0 and restores the previous behavior, where `webassembly_udf_max_input_block_size = 0` meant one call per pipeline block."}, + {"cascades_aggregation_pushdown", false, true, "New setting to consider pushing partial aggregation below a join (eager aggregation) in the Cascades optimizer."}, + {"optimize_read_in_reverse_order_final", false, true, "New setting to enable the read-in-order optimization when reading in reverse order of the sorting key with the `FINAL` modifier from `ReplacingMergeTree` tables."}, + {"load_marks_asynchronously", false, true, "Load marks of all streams in parallel by default. On remote disks, synchronous loading of marks of columns with many substreams (such as `JSON`) took one network round trip per stream."}, + {"ast_fuzzer_oracle", false, false, "New setting to enable correctness oracle checks in the server-side AST fuzzer."}, + {"create_token_default_ttl_seconds", 1800, 1800, "New setting giving a lifetime to a token created by `CREATE TOKEN` without an explicit `VALID UNTIL` or `VALID FOR` clause. The statement is new, so there is no earlier behavior to restore and the previous value is the default itself: a `compatibility` with an older version must not turn tokens into never-expiring ones."}, + {"enable_hash_join_row_store", false, true, "New setting to enable transforming the payload of a hash join into a row-major layout."}, + {"min_rows_ratio_for_hash_join_row_store", 5.0, 5.0, "New setting to control the minimum estimated ratio of join output rows to build-side rows to enable transforming hash join payload to row-major. 0 means the transformation is always allowed."}, + {"enable_sharding_aggregator", false, false, "Obsolete setting, the sharded aggregator has been removed in favor of the adaptive aggregator (`enable_adaptive_aggregator`)."}, + {"allow_preliminary_distinct_abandoning", false, true, "New setting that lets the preliminary `DISTINCT` give up deduplicating mostly-unique input, because the final `DISTINCT` deduplicates its output again."}, + {"query_plan_fuse_filter_into_array_join", false, true, "New optimization to fuse a filter on ARRAY JOINed columns into the ARRAY JOIN step, enabled by default."}, + {"iceberg_file_entries_queue_size", 100, 100, "New setting for the previously hardcoded capacity of the queue between the Iceberg data manifest decode tasks and the query."}, + {"iceberg_manifest_decode_concurrency", 2, 4, "New setting bounding how many Iceberg manifest files are decoded concurrently, for delete and data manifests alike. It replaces `iceberg_delete_manifest_decode_concurrency` (kept as an alias). `2` approximates the pre-26.9 data path, which decoded one manifest at a time with the next one's fetch already in flight; under `compatibility` at or below 26.8 the delete decode therefore also runs at 2 rather than its released default of 4, preserving the older data-path memory envelope at the cost of some delete-decode overlap."}, + {"query_plan_lower_array_join_function", false, false, "New optimization to lower an arrayJoin function into a real ARRAY JOIN step; disabled by default."}, + {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, + {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, + {"allow_experimental_analyzer", true, true, "The setting is obsolete: the analyzer is mandatory and the old query analysis is no longer supported. Disabling it is refused instead of being ignored, and `compatibility` with a version below 24.3 no longer reverts it."}, + {"allow_url_wildcard_from_index_pages", false, false, "Added an alias for setting `allow_experimental_url_wildcard_from_index_pages`."}, + {"allow_kafka_offsets_storage_in_keeper", false, false, "Added an alias for setting `allow_experimental_kafka_offsets_storage_in_keeper`."}, + {"allow_correlated_subqueries", true, true, "Added an alias for setting `allow_experimental_correlated_subqueries`."}, + {"allow_geo_types_in_iceberg", false, false, "Added an alias for setting `allow_experimental_geo_types_in_iceberg`."}, + {"enable_materialized_postgresql_table", false, false, "Added an alias for setting `allow_experimental_materialized_postgresql_table`."}, + {"enable_funnel_functions", false, false, "Added an alias for setting `allow_experimental_funnel_functions`."}, + {"enable_unique_key", false, false, "Added an alias for setting `allow_experimental_unique_key`."}, + {"allow_join_right_table_sorting", false, false, "Added an alias for setting `allow_experimental_join_right_table_sorting`."}, + {"enable_json_lazy_type_hints", false, false, "Added an alias for setting `allow_experimental_json_lazy_type_hints`."}, + {"enable_join_runtime_filters", true, true, "The JOIN runtime filters became a Production tier feature."}, + {"join_runtime_filter_exact_values_limit", 10000, 10000, "The JOIN runtime filters became a Production tier feature."}, + {"join_runtime_bloom_filter_bytes", 512_KiB, 512_KiB, "The JOIN runtime filters became a Production tier feature."}, + {"join_runtime_bloom_filter_hash_functions", 3, 3, "The JOIN runtime filters became a Production tier feature."}, + {"join_runtime_filter_pass_ratio_threshold_for_disabling", 0.7, 0.7, "The JOIN runtime filters became a Production tier feature."}, + {"join_runtime_filter_blocks_to_skip_before_reenabling", 30, 30, "The JOIN runtime filters became a Production tier feature."}, + {"join_runtime_bloom_filter_max_ratio_of_set_bits", 0.7, 0.7, "The JOIN runtime filters became a Production tier feature."}, + {"join_runtime_filter_min_probe_rows", 1000, 1000, "The JOIN runtime filters became a Production tier feature."}, + {"enable_join_runtime_filters_index_analysis", false, false, "The JOIN runtime filters became a Production tier feature."}, + {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, + {"query_plan_aggregation_bucket_top_k", false, true, "New setting to toggle the plan optimization that materializes only each two-level bucket's best n groups when a final aggregation feeds ORDER BY over its outputs with LIMIT n and the per-bucket selection is provably exact."}, + {"enable_trino_dialect", false, false, "New setting to enable the `trino` value of the `dialect` setting, which translates Trino SQL syntax and maps Trino function names to ClickHouse equivalents."}, + {"enable_join_key_only_hash_tables", false, true, "New setting to store the join keys alone, without a reference to a right row, in the hash tables of joins whose result can never contain a value taken from a right row (`LEFT ANTI`, and `LEFT SEMI` when no right column is selected)."}, + {"distributed_plan_read_in_order", false, false, "New setting to allow the read-in-order optimization for `ORDER BY` in a distributed query plan, so a sorted read of the table's sorting key can skip the sort and stop early. Off by default: only shapes where no exchange survives between the read and the sort are safe today."}, + {"distributed_cache_client_id", "", "", "New setting (CI tests only) to override the distributed cache client id per query."}, + {"query_plan_propagate_predicate_across_join", false, true, "New setting that lifts filter conjuncts across equi-join keys so primary-key pruning fires on both sides."}, + {"read_through_distributed_cache", false, false, "The setting moved to the server configuration and is ignored as a profile setting: reading from the distributed cache is now switched by the server setting `enable_read_through_distributed_cache`, which is applied without a restart (so that merges, mutations and `Buffer` flushes follow it too, instead of being pinned to the value the server started with). Use `force_read_through_distributed_cache` to deviate from the server setting per query."}, + {"write_through_distributed_cache", false, false, "The setting moved to the server configuration and is ignored as a profile setting: writing to the distributed cache is now switched by the server setting `enable_write_through_distributed_cache`, which is applied without a restart (so that merges, mutations and `Buffer` flushes follow it too, instead of being pinned to the value the server started with). Use `force_write_through_distributed_cache` to deviate from the server setting per query."}, + {"force_read_through_distributed_cache", "auto", "auto", "New setting overriding the server setting `enable_read_through_distributed_cache` for a single query."}, + {"force_write_through_distributed_cache", "auto", "auto", "New setting overriding the server setting `enable_write_through_distributed_cache` for a single query."}, + {"distributed_cache_min_inflight_bytes_to_discard_connection_on_seek", 0, 4 * 1024 * 1024, "New setting to drop and reopen a distributed cache connection on a seek when too many in-flight bytes would otherwise be discarded. Defaults to 4 MiB; 0 restores the previous behavior (always reuse the connection via the read range id)."}, + {"distributed_plan_workers_provisioning_timeout_ms", 10000, 10000, "New setting bounding how long a query waits for leased stateless workers to become reachable before execution."}, + {"distributed_plan_fallback_to_local_execution", false, true, "New setting to fall back to local execution when a plan cannot be distributed (only takes effect under `make_distributed_plan`)."}, + {"query_plan_optimize_lazy_materialization_for_object_storage", false, true, "New setting to use lazy materialization for `ORDER BY ... LIMIT n` queries reading Parquet files from object storage (including Iceberg tables)."}, + {"iceberg_compaction_commit_batch_size", 100, 100, "New setting"}, + {"iceberg_compaction_max_rows_in_data_file", std::numeric_limits::max(), std::numeric_limits::max(), "New setting for the max rows of an iceberg data file produced by compaction, separate from the insert-time limit."}, + {"iceberg_compaction_max_bytes_in_data_file", std::numeric_limits::max(), std::numeric_limits::max(), "New setting for the max bytes of an iceberg data file produced by compaction, separate from the insert-time limit."}, + {"enable_json_lazy_type_hints", false, false, "Lazy JSON type hints are now Beta. An alias for setting 'allow_experimental_json_lazy_type_hints'."}, + {"s3_upload_checksum_algorithm", "", "", "New setting to choose the checksum algorithm for S3 uploads."}, + {"network_compression_method", "LZ4", "ZSTD", "Switched the default compression method for client/server and server/server communication from `LZ4` to `ZSTD` to reduce network traffic."}, + {"network_zstd_compression_level", 1, 3, "Aligned the default network `ZSTD` compression level with the new default on-disk `ZSTD(3)` compression."}, + {"use_statistics_for_min_max_aggregation", false, true, "New setting to answer `min`, `max` and `count` aggregations without `GROUP BY` and filters from per-part column statistics for parts that have them materialized, reading only the remaining parts. previous_value=false so `compatibility` with versions before 26.9 keeps the optimization disabled and restores the pre-existing plan."}, + {"parallel_replicas_allow_merge_tables", false, false, "New setting to allow reading from a `Merge` table with plan-based parallel replicas, by expanding the `Merge` read into a union of the reads from the underlying `MergeTree` tables. It only has an effect together with `parallel_replicas_plan_based`."}, + {"iceberg_compaction_max_bytes_in_data_file", std::numeric_limits::max(), 512 * 1024 * 1024, "New setting for the max bytes of an iceberg data file produced by compaction, separate from the insert-time limit. The default is aligned with the documented default of the Iceberg table property `write.target-file-size-bytes` (512 MiB), see https://iceberg.apache.org/docs/1.5.2/configuration/. Previously compaction merged all eligible files of a partition into a single output file."}, + {"iceberg_insert_max_bytes_in_data_file", 1024 * 1024 * 1024, 512 * 1024 * 1024, "Aligned with the documented default of the Iceberg table property `write.target-file-size-bytes` (512 MiB), see https://iceberg.apache.org/docs/1.5.2/configuration/."}, + {"iceberg_data_file_size_lower_threshold_compaction", 10 * 1024 * 1024, 384 * 1024 * 1024, "Aligned with how the Iceberg `rewrite_data_files` procedure derives `min-file-size-bytes`: 0.75 of the target file size (512 MiB). Compaction now selects files below 384 MiB instead of below 10 MiB."}, + {"iceberg_data_file_size_upper_threshold_compaction", 10ULL * 1024 * 1024 * 1024, 512ULL * 1024 * 1024 * 9 / 5, "Aligned with how the Iceberg `rewrite_data_files` procedure derives `max-file-size-bytes`: 1.8 of the target file size (512 MiB)."}, + {"iceberg_manifest_min_count_to_compact", 30, 100, "Aligned with the documented default of the Iceberg table property `commit.manifest.min-count-to-merge` (100), see https://iceberg.apache.org/docs/1.5.2/configuration/."}, + {"parallel_replicas_allow_merge_tables", false, false, "New setting to allow reading from a `Merge` table with plan-based parallel replicas, by expanding the `Merge` read into a union of the reads from the underlying `MergeTree` tables. It only has an effect together with `parallel_replicas_plan_based`."}, + {"optimize_mutations_with_partition_pruning", false, true, "New setting to automatically prune partitions for mutations based on WHERE clause"}, + {"statistics_max_set_size_for_exact_selectivity_estimation", 10000, 10000, "The bound on the cost of estimating the selectivity of `IN` with a large set is kept under `compatibility` with an earlier version: the previous value is deliberately equal to the new one, so that the uncapped estimation, which could add hundreds of milliseconds to the planning of a single query, is not restored."}, + {"type_json_skip_null_typed_paths", false, false, "New setting to treat NULL values in typed JSON paths as absent"}, + {"use_iceberg_manifest_list_partition_pruning", false, true, "New setting to skip Iceberg manifest files whose manifest-list partition summaries cannot match the query filter, without reading them."}, + {"enable_time_series_table", false, false, "The `TimeSeries` table engine and the `promql` dialect were moved to the private preview tier. Added an alias for setting `allow_experimental_time_series_table`."}, + {"enable_time_series_aggregate_functions", false, false, "The `timeSeries*` aggregate functions were moved to the private preview tier. Added an alias for setting `allow_experimental_time_series_aggregate_functions`."}, + {"output_format_arrow_record_batch_size", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target row count. The default `0` preserves one record batch per block."}, + {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, + }); addSettingsChanges(settings_changes_history, "26.8", { {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 5c20520e2fb76..1c7e063b7f5d1 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -2153,8 +2153,8 @@ def test_api_call_quota_ignores_subquery_settings(started_cluster): ) outer_wins = int(get_profile_events(qid)["api_calls"]) - # The quota is set only in the subquery; the outer query leaves it at the default (far - # above 64). The subquery cap is ignored, so all 64 rows run rather than stopping at 5 - + # The quota is set only in the subquery; the outer query leaves it at the default (0 - + # no limit). The subquery cap is ignored, so all 64 rows run rather than stopping at 5 - # a quota set only in a subquery has no effect. qid = unique_query_id("quota_levels_subquery_only") instance.query( diff --git a/tests/queries/0_stateless/03300_ai_functions.reference b/tests/queries/0_stateless/03300_ai_functions.reference index 2e1f69e2d613c..7fd7da0270370 100644 --- a/tests/queries/0_stateless/03300_ai_functions.reference +++ b/tests/queries/0_stateless/03300_ai_functions.reference @@ -44,9 +44,9 @@ String -- Setting defaults ai_function_embedding_default_credentials ai_function_embedding_max_batch_size 100 -ai_function_max_api_calls_per_query 1000 -ai_function_max_input_tokens_per_query 1000000 -ai_function_max_output_tokens_per_query 500000 +ai_function_max_api_calls_per_query 0 +ai_function_max_input_tokens_per_query 0 +ai_function_max_output_tokens_per_query 0 ai_function_max_retries 1 ai_function_request_timeout_sec 60 ai_function_retry_initial_delay_ms 1000 diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference index a6800ceefe7f8..b4345fecb8359 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference @@ -1,4 +1,6 @@ -- Current defaults +false 0 1 +-- compatibility = 26.9 restores the API call quota false 1000 1 -- compatibility = 26.6 restores the legacy defaults true 0 0 diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql index cf37018c792d0..1881d8b85a6f5 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql @@ -3,10 +3,12 @@ -- no-replicated-database: named collections are server-global, not database-scoped -- ============================================================================= --- Three AI function defaults were flipped: `ai_function_allow_insecure_endpoint` from 1 to 0 --- and `ai_function_max_api_calls_per_query` from 0 (unlimited) to 1000 in 26.8, and --- `ai_function_max_retries` from 0 to 1 in 26.9. `compatibility = 26.6` predates all three --- and restores them, which pins the previous_value/new_value pairs in `SettingsChangesHistory`. +-- Four AI function default flips: `ai_function_allow_insecure_endpoint` from 1 to 0 and +-- `ai_function_max_api_calls_per_query` from 0 (unlimited) to 1000 in 26.8, then +-- `ai_function_max_retries` from 0 to 1 in 26.9, and `ai_function_max_api_calls_per_query` +-- back to 0 (unlimited) in 26.10. `compatibility = 26.6` predates all of them and +-- `compatibility = 26.9` reverts only the last one, which pins the previous_value/new_value +-- pairs in `SettingsChangesHistory`. -- -- The endpoint check runs in `resolveAIParams`, before the zero-row early return -- in `executeImpl`, so an empty source table exercises it without any real HTTP @@ -24,6 +26,11 @@ SELECT '-- Current defaults'; SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); SELECT aiGenerate(x, map('credentials', 'ai_compat_remote_http')) FROM tab; -- { serverError BAD_ARGUMENTS } +SELECT '-- compatibility = 26.9 restores the API call quota'; +SET compatibility = '26.9'; +SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); +SELECT aiGenerate(x, map('credentials', 'ai_compat_remote_http')) FROM tab; -- { serverError BAD_ARGUMENTS } + SELECT '-- compatibility = 26.6 restores the legacy defaults'; SET compatibility = '26.6'; SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); From fc66ebbb23dd880e7954149bc263208a076344bc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 01:52:41 +0000 Subject: [PATCH 069/296] Backport #121544 to 26.8: Cast Delta Lake partition columns to the declared type --- .../StorageObjectStorageSource.cpp | 6 +++ ...ake_legacy_partition_column_type.reference | 10 ++++ ...delta_lake_legacy_partition_column_type.sh | 52 +++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 tests/queries/0_stateless/05237_delta_lake_legacy_partition_column_type.reference create mode 100755 tests/queries/0_stateless/05237_delta_lake_legacy_partition_column_type.sh diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 42f4fcaa6d458..97dfc9cb58684 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -793,6 +794,11 @@ Chunk StorageObjectStorageSource::generate() const auto column_pos = read_from_format_info.source_header.getPositionByName(name_and_type.name); auto partition_column = name_and_type.type->createColumnConst(chunk.getNumRows(), value)->convertToFullColumnIfConst(); + /// The `_delta_log` type differs from the declared one when the columns were + /// specified rather than inferred, and the block follows the declared schema. + const auto & declared_type = read_from_format_info.source_header.getByPosition(column_pos).type; + if (!name_and_type.type->equals(*declared_type)) + partition_column = castColumn({partition_column, name_and_type.type, name_and_type.name}, declared_type); /// This column is filled with default value now, remove it. chunk.erase(column_pos); /// Add correct values. diff --git a/tests/queries/0_stateless/05237_delta_lake_legacy_partition_column_type.reference b/tests/queries/0_stateless/05237_delta_lake_legacy_partition_column_type.reference new file mode 100644 index 0000000000000..f9023ab5da2cf --- /dev/null +++ b/tests/queries/0_stateless/05237_delta_lake_legacy_partition_column_type.reference @@ -0,0 +1,10 @@ +-- allow_delta_kernel_rs = 0 +DateTime 1 2026-09-21 09:00:00 +1 2026-09-21 09:00:00 +1 2026-09-21 09:00:00 +Nullable(DateTime) 1 2026-09-21 09:00:00 +-- allow_delta_kernel_rs = 1 +DateTime 1 2026-09-21 09:00:00 +1 2026-09-21 09:00:00 +1 2026-09-21 09:00:00 +Nullable(DateTime) 1 2026-09-21 09:00:00 diff --git a/tests/queries/0_stateless/05237_delta_lake_legacy_partition_column_type.sh b/tests/queries/0_stateless/05237_delta_lake_legacy_partition_column_type.sh new file mode 100755 index 0000000000000..a57c95c560125 --- /dev/null +++ b/tests/queries/0_stateless/05237_delta_lake_legacy_partition_column_type.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: delta-kernel pulls in extra dependencies. +# Tag no-msan: delta-kernel-rs (Rust) is not built under MSan, so DeltaLakeLocal is absent. + +# The legacy (non delta-kernel) DeltaLake reader inserts partition columns into the chunk with the +# type parsed from `_delta_log`, which is not the declared column type when the table function (or +# the insertion table, through `use_structure_from_insertion_table_in_table_functions`) asks for a +# different one. Here `_delta_log` says `Nullable(DateTime64(6))` and the query asks for `DateTime`. +# Without a conversion, a query with no filter reads the column through the wrong type and returns +# garbage, and a filter on it throws a `LOGICAL_ERROR` out of the expression it is compared with. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +ROOT="${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}_delta_legacy_partition" +trap 'rm -rf "${ROOT}" 2>/dev/null' EXIT +rm -rf "${ROOT}" +mkdir -p "${ROOT}/_delta_log" + +# Data file holding only the non-partition column, as a partitioned Delta table stores it. +${CLICKHOUSE_CLIENT} --query " + INSERT INTO FUNCTION file('${ROOT}/process_time=2026-09-21 09%3A00%3A00/data.parquet', Parquet, 'id Int32') VALUES (1); +" < /dev/null + +# A v0 transaction log declaring `process_time` as a nullable `timestamp` partition column. +SCHEMA='{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":false,\"metadata\":{}},{\"name\":\"process_time\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}' +cat > "${ROOT}/_delta_log/00000000000000000000.json" < SELECT * FROM deltaLake(...)` +# passes down when the destination declares `process_time` as `DateTime`. +TABLE_FUNCTION="deltaLakeLocal('${ROOT}', 'Parquet', 'id Int32, process_time DateTime')" +NULLABLE_TABLE_FUNCTION="deltaLakeLocal('${ROOT}', 'Parquet', 'id Int32, process_time Nullable(DateTime)')" + +for kernel in 0 1 +do + echo "-- allow_delta_kernel_rs = ${kernel}" + # `session_timezone` is pinned because the two readers disagree on the zone a partition + # timestamp is parsed in, which is a separate question from the type this test is about. + ${CLICKHOUSE_CLIENT} --allow_delta_kernel_rs="${kernel}" --session_timezone UTC --query " + SELECT toTypeName(process_time), * FROM ${TABLE_FUNCTION}; + SELECT * FROM ${TABLE_FUNCTION} WHERE process_time = '2026-09-21 09:00:00'; + SELECT * FROM ${TABLE_FUNCTION} WHERE toDateTime(process_time) = toDateTime('2026-09-21 09:00:00'); + SELECT * FROM ${TABLE_FUNCTION} WHERE process_time = '2026-09-21 10:00:00'; + SELECT toTypeName(process_time), * FROM ${NULLABLE_TABLE_FUNCTION}; + " < /dev/null +done From 70ebdd9e8e1901efaa2455e8d77e6d0bb61f778b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 10:18:08 +0000 Subject: [PATCH 070/296] Backport #121623 to 26.8: Add setting analyzer_compatibility_allow_cte_redefinition --- .../performance-and-monitoring/analyzer.mdx | 25 +++-- .../Resolve/IdentifierResolveScope.cpp | 8 +- src/Analyzer/Resolve/IdentifierResolveScope.h | 4 +- src/Analyzer/Resolve/QueryAnalyzer.cpp | 88 ++++++++++++++--- src/Analyzer/Resolve/QueryAnalyzer.h | 3 + src/Core/Settings.cpp | 8 ++ src/Core/SettingsChangesHistory.cpp | 1 + ...atibility_allow_cte_redefinition.reference | 34 +++++++ ...r_compatibility_allow_cte_redefinition.sql | 99 +++++++++++++++++++ 9 files changed, 246 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/05238_analyzer_compatibility_allow_cte_redefinition.reference create mode 100644 tests/queries/0_stateless/05238_analyzer_compatibility_allow_cte_redefinition.sql diff --git a/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx b/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx index 730c453a6957d..2bf316f458c8e 100644 --- a/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx +++ b/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx @@ -277,22 +277,31 @@ SELECT category, sum(value) FROM t WHERE service = 'svc1' GROUP BY category; Error: `CTE with name ... already exists (MULTIPLE_EXPRESSIONS_FOR_ALIAS)`. Exception code: 179 -Cause: The old analyzer permitted defining multiple Common Table Expressions (WITH ...) with the same name shadowing the earlier one. The analyzer forbids this ambiguity. +Cause: The old analyzer permitted defining multiple Common Table Expressions (WITH ...) with the same name, a later definition shadowing the earlier one. The analyzer rejects this ambiguity by default. -Solution: Rename duplicate CTEs to be unique. +Solution: Rename duplicate CTEs to be unique. As a migration aid, enable `analyzer_compatibility_allow_cte_redefinition = 1` (available since ClickHouse `26.10`) to restore the legacy behavior: a reference binds to the latest definition of the name that is not being resolved at that moment, so a redefinition can read the previous definition and the query body reads the last one. + +Limitations: a CTE declared as `MATERIALIZED` and a CTE in a `WITH RECURSIVE` clause cannot be redefined even with the setting enabled. One shape differs from the old analyzer: a CTE declared between two definitions of a name also binds to the last definition, where the old analyzer bound it to the definition visible at its declaration point. ```sql /* ORIGINAL QUERY */ -WITH - data AS (SELECT 1 AS id), - data AS (SELECT 2 AS id) -- Redefined +WITH + data AS (SELECT 1 AS id), + data AS (SELECT id + 1 AS id FROM data) -- Redefined, reads the previous definition SELECT * FROM data; /* FIXED QUERY */ -WITH - raw_data AS (SELECT 1 AS id), - processed_data AS (SELECT 2 AS id) +WITH + raw_data AS (SELECT 1 AS id), + processed_data AS (SELECT id + 1 AS id FROM raw_data) SELECT * FROM processed_data; + +/* LEGACY BEHAVIOR AS A MIGRATION AID */ +WITH + data AS (SELECT 1 AS id), + data AS (SELECT id + 1 AS id FROM data) +SELECT * FROM data +SETTINGS analyzer_compatibility_allow_cte_redefinition = 1; ``` ### Ambiguous column identifiers {#ambiguous-column-identifiers} diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.cpp b/src/Analyzer/Resolve/IdentifierResolveScope.cpp index a169c74744eb6..5fdb1eb7d4438 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.cpp +++ b/src/Analyzer/Resolve/IdentifierResolveScope.cpp @@ -321,7 +321,13 @@ void dump_list(WriteBuffer & buffer, const String & list_name, const std::ranges dump_mapping(buffer, "Alias name to expression node", aliases.alias_name_to_expression_node); dump_mapping(buffer, "Alias name to function node", aliases.alias_name_to_lambda_node); dump_mapping(buffer, "Alias name to table expression node", aliases.alias_name_to_table_expression_node); - dump_mapping(buffer, "CTE name to query node", cte_name_to_query_node); + if (!cte_name_to_query_node.empty()) + { + buffer << "CTE name to query node table size: " << cte_name_to_query_node.size() << '\n'; + for (const auto & [cte_name, cte_nodes] : cte_name_to_query_node) + for (const auto & cte_node : cte_nodes) + buffer << " { '" << cte_name << "' : " << cte_node->formatASTForErrorMessage() << " }\n"; + } dump_mapping(buffer, "WINDOW name to window node", window_name_to_window_node); dump_list(buffer, "Nodes with duplicated aliases size ", aliases.nodes_with_duplicated_aliases); diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.h b/src/Analyzer/Resolve/IdentifierResolveScope.h index 2864b0b311684..33609d26a3ae0 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.h +++ b/src/Analyzer/Resolve/IdentifierResolveScope.h @@ -154,8 +154,8 @@ struct IdentifierResolveScope std::list *> join_using_columns; - /// CTE name to query node - std::unordered_map cte_name_to_query_node; + /// CTE name to its definitions in declaration order (several only with `analyzer_compatibility_allow_cte_redefinition`) + std::unordered_map cte_name_to_query_node; /// Window name to window node std::unordered_map window_name_to_window_node; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 3c684a85673b4..71b19fafddb43 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -85,6 +85,7 @@ namespace Setting { extern const SettingsBool aggregate_functions_null_for_empty; extern const SettingsBool analyzer_compatibility_allow_non_aggregate_in_having; + extern const SettingsBool analyzer_compatibility_allow_cte_redefinition; extern const SettingsBool enable_streaming_queries; extern const SettingsBool analyzer_compatibility_join_using_top_level_identifier; extern const SettingsBool analyzer_compatibility_multiple_joins_qualify_column_names; @@ -149,6 +150,18 @@ namespace ErrorCodes namespace { +/// True for a `WITH` element declared `AS MATERIALIZED`, before or after its replacement by a `TableNode`. +bool isMaterializedCTEDefinition(const QueryTreeNodePtr & node) +{ + if (const auto * query_node = node->as()) + return query_node->isMaterialized(); + if (const auto * union_node = node->as()) + return union_node->isMaterialized(); + if (const auto * table_node = node->as()) + return table_node->isMaterializedCTE(); + return false; +} + /// Recursively clears aliases from `node` and all of its descendants, stopping at /// nested-scope boundaries (`QUERY`, `UNION`, `LAMBDA`). /// @@ -1306,7 +1319,9 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierFromCTE( ) { auto full_name = identifier_lookup.identifier.getFullName(); - auto cte_query_node_it = scope.cte_name_to_query_node.find(full_name); + auto cte_nodes_it = scope.cte_name_to_query_node.find(full_name); + if (cte_nodes_it == scope.cte_name_to_query_node.end()) + return {}; /// CTE may reference table expressions with the same name, e.g.: /// @@ -1320,10 +1335,23 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierFromCTE( /// /// To accomplish this behaviour it's not allowed to resolve identifiers to /// CTE that is being resolved. - if (cte_query_node_it == scope.cte_name_to_query_node.end() || ctes_in_resolve_process.contains(cte_query_node_it->second)) + /// + /// With `analyzer_compatibility_allow_cte_redefinition` a name can have several definitions; the latest one + /// not being resolved wins, so a redefinition reads the previous definition and the query body the last one. + auto & cte_nodes = cte_nodes_it->second; + /// Every site that marks a scope-map CTE node as being resolved updates both sets. With one definition keep the + /// structural check: the materialized-CTE expression site inserts a clone, which only identity would miss. + const bool several_definitions = cte_nodes.size() > 1; + auto cte_node_it = std::find_if(cte_nodes.rbegin(), cte_nodes.rend(), [this, several_definitions](const QueryTreeNodePtr & node) + { + if (several_definitions) + return !cte_definitions_in_resolve_process.contains(node.get()); + return !ctes_in_resolve_process.contains(node); + }); + if (cte_node_it == cte_nodes.rend()) return {}; - auto & cte_node = cte_query_node_it->second; + auto & cte_node = *cte_node_it; auto * query_node = cte_node->as(); auto * union_node = cte_node->as(); @@ -3299,6 +3327,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( /// In this example argument of function `in` is being resolve here. If CTE `test1` is not forbidden, /// `test1` is resolved to CTE (not to the table) in `initializeQueryJoinTreeNode` function. ctes_in_resolve_process.insert(original_cte_node); + cte_definitions_in_resolve_process.insert(original_cte_node.get()); if (subquery_node) resolveQuery(resolved_identifier_node, subquery_scope); @@ -3306,6 +3335,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( resolveUnion(resolved_identifier_node, subquery_scope); ctes_in_resolve_process.erase(original_cte_node); + cte_definitions_in_resolve_process.erase(original_cte_node.get()); } else if (table_node != nullptr && table_node->isMaterializedCTE()) { @@ -5986,12 +6016,18 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, QueryTreeNodePtr original_cte_node = try_get_original_cte_node(join_tree_node); if (original_cte_node) + { ctes_in_resolve_process.insert(original_cte_node); + cte_definitions_in_resolve_process.insert(original_cte_node.get()); + } resolveExpressionNode(join_tree_node, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/, true /*ignore_alias=*/); if (original_cte_node) + { ctes_in_resolve_process.erase(original_cte_node); + cte_definitions_in_resolve_process.erase(original_cte_node.get()); + } break; } case QueryTreeNodeType::TABLE_FUNCTION: @@ -6014,19 +6050,22 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, /// Prevent recursive CTE references during subquery resolution. const auto & cte_name = materialized_cte_ptr->cte_name; - QueryTreeNodePtr cte_map_node; + QueryTreeNodes cte_map_nodes; for (auto * s = &scope; s; s = s->parent_scope) { auto it = s->cte_name_to_query_node.find(cte_name); if (it != s->cte_name_to_query_node.end()) { - cte_map_node = it->second; + cte_map_nodes = it->second; break; } } - if (cte_map_node) + for (const auto & cte_map_node : cte_map_nodes) + { ctes_in_resolve_process.insert(cte_map_node); + cte_definitions_in_resolve_process.insert(cte_map_node.get()); + } IdentifierResolveScope & subquery_scope = createIdentifierResolveScope(subquery, &scope); subquery_scope.subquery_depth = scope.subquery_depth + 1; @@ -6036,8 +6075,11 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, else resolveUnion(subquery, subquery_scope); - if (cte_map_node) + for (const auto & cte_map_node : cte_map_nodes) + { ctes_in_resolve_process.erase(cte_map_node); + cte_definitions_in_resolve_process.erase(cte_map_node.get()); + } bool is_correlated = subquery->as() ? subquery->as()->isCorrelated() @@ -6490,12 +6532,32 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier continue; const auto & cte_name = subquery_node ? subquery_node->getCTEName() : union_node->getCTEName(); - auto [_, inserted] = scope.cte_name_to_query_node.emplace(cte_name, node); - if (!inserted) - throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, - "CTE with name {} already exists. In scope {}", - cte_name, - scope.scope_node->formatASTForErrorMessage()); + auto & cte_nodes = scope.cte_name_to_query_node[cte_name]; + if (!cte_nodes.empty()) + { + if (query_node_typed.isRecursiveWith()) + throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, + "CTE with name {} already exists and cannot be redefined in a recursive WITH clause. In scope {}", + cte_name, + scope.scope_node->formatASTForErrorMessage()); + + /// A redefinition is rejected on its second registration, so only the first node can be materialized. + if (isMaterializedCTEDefinition(node) || isMaterializedCTEDefinition(cte_nodes.front())) + throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, + "CTE with name {} already exists and cannot be redefined because it is declared as MATERIALIZED. In scope {}", + cte_name, + scope.scope_node->formatASTForErrorMessage()); + + /// Checked last, so the hint is given only when enabling the setting would help. + if (!scope.context->getSettingsRef()[Setting::analyzer_compatibility_allow_cte_redefinition]) + throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, + "CTE with name {} already exists. Enable the setting analyzer_compatibility_allow_cte_redefinition " + "to let a later definition shadow the earlier one. In scope {}", + cte_name, + scope.scope_node->formatASTForErrorMessage()); + } + + cte_nodes.push_back(node); } /** WITH section can be safely removed, because WITH section only can provide aliases to query expressions diff --git a/src/Analyzer/Resolve/QueryAnalyzer.h b/src/Analyzer/Resolve/QueryAnalyzer.h index 129c38311cd72..b478854440f23 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.h +++ b/src/Analyzer/Resolve/QueryAnalyzer.h @@ -312,6 +312,9 @@ class QueryAnalyzer /// CTEs that are currently in resolve process QueryTreeNodePtrWithHashSet ctes_in_resolve_process; + /// Same as `ctes_in_resolve_process` but by identity: structural comparison cannot tell identical redefinitions apart. + std::unordered_set cte_definitions_in_resolve_process; + /// Window definitions that are currently in resolve process std::unordered_set windows_in_resolve_process; diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 304d1177a1796..372310666525a 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8322,6 +8322,14 @@ SELECT ll.Date FROM (SELECT * FROM t AS ll LEFT JOIN t1 ON ll.k = t1.k LEFT JOIN ``` Takes effect only when the analyzer is enabled (`enable_analyzer = 1`). +)", 0) \ + DECLARE(Bool, analyzer_compatibility_allow_cte_redefinition, false, R"( +Allow a Common Table Expression name to be defined more than once in a single `WITH` clause. A reference to such a name binds to the latest definition that is not being resolved at that moment: a redefinition can read the previous definition of the same name, and the query body reads the last one. This matches the query analysis that ClickHouse used before v24.3, where a later definition silently shadowed the earlier ones. One shape differs from that analysis: a CTE declared between two definitions of a name also binds to the last definition, where the old analysis bound it to the definition visible at its declaration point. By default a redefinition is rejected with `MULTIPLE_EXPRESSIONS_FOR_ALIAS`. A CTE declared as `MATERIALIZED` and a CTE in a `WITH RECURSIVE` clause cannot be redefined even when the setting is enabled. + +Possible values: + +- 0 - A CTE name can be defined only once in a `WITH` clause. +- 1 - A later definition of a CTE name shadows the earlier ones. )", 0) \ DECLARE(Bool, enable_identifier_resolve_cache, true, R"( Enable the identifier resolution cache in the query analyzer. The cache shares resolved alias nodes to prevent AST explosion when the same alias is referenced multiple times. Set to false to disable caching if incorrect results are suspected. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a13257bba3538..4e1f44a9c8340 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -43,6 +43,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// Note: please check if the key already exists to prevent duplicate entries. addSettingsChanges(settings_changes_history, "26.8", { + {"analyzer_compatibility_allow_cte_redefinition", false, false, "New compatibility setting. When enabled, the analyzer accepts a CTE name defined more than once in a single `WITH` clause and lets a later definition shadow the earlier ones, as the query analysis before v24.3 did."}, {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, diff --git a/tests/queries/0_stateless/05238_analyzer_compatibility_allow_cte_redefinition.reference b/tests/queries/0_stateless/05238_analyzer_compatibility_allow_cte_redefinition.reference new file mode 100644 index 0000000000000..72c437349e4f6 --- /dev/null +++ b/tests/queries/0_stateless/05238_analyzer_compatibility_allow_cte_redefinition.reference @@ -0,0 +1,34 @@ +-- 1 unreferenced duplicates +1 10 +2 20 +-- 2 redefinition reads the previous definition +40 +-- 3 the query body reads the last definition +2 +-- 4 chain of three definitions +20 +-- 5 two references to the redefined name +2 2 +-- 6 redefinition in a nested scope shadows the outer CTE +3 +-- 7 IN with a redefined CTE +2 +-- 8 the first definition reads the table of the same name +12 +-- 9 progressive redefinition +0 0 0 100 300 +1 10 20 120 360 +2 20 40 140 420 +-- 10 UNION CTE redefined +3 +-- 11 consumer declared between two definitions +2 2 +-- 12 view +1 80 high +2 30 low +-- 13 MATERIALIZED and RECURSIVE CTEs cannot be redefined +-- 14 identical redefinitions are distinct definitions +20 +6 +-- 15 all definitions in resolve process fall through to the outer CTE +102 diff --git a/tests/queries/0_stateless/05238_analyzer_compatibility_allow_cte_redefinition.sql b/tests/queries/0_stateless/05238_analyzer_compatibility_allow_cte_redefinition.sql new file mode 100644 index 0000000000000..b32f65522af39 --- /dev/null +++ b/tests/queries/0_stateless/05238_analyzer_compatibility_allow_cte_redefinition.sql @@ -0,0 +1,99 @@ +-- Compatibility setting `analyzer_compatibility_allow_cte_redefinition`: a CTE name may be defined more than once +-- in one WITH clause, a later definition shadowing the earlier ones as the query analysis before v24.3 did. +-- Expected values come from that analysis (26.8 with enable_analyzer = 0), except where noted. + +DROP TABLE IF EXISTS t_cte_redefinition; +DROP VIEW IF EXISTS v_cte_redefinition; + +-- Default: a redefinition is rejected, referenced or not. +WITH d AS (SELECT 1 AS id), d AS (SELECT 2 AS id) SELECT * FROM d; -- { serverError MULTIPLE_EXPRESSIONS_FOR_ALIAS } +WITH d AS (SELECT 1 AS id), d AS (SELECT 2 AS id) SELECT 1; -- { serverError MULTIPLE_EXPRESSIONS_FOR_ALIAS } + +SET analyzer_compatibility_allow_cte_redefinition = 1; + +SELECT '-- 1 unreferenced duplicates'; +WITH qb AS (SELECT 1 AS Id, 10 AS Shop), qb AS (SELECT 2 AS Id, 20 AS Shop) +SELECT a.Id, a.Shop FROM (SELECT 1 AS Id, 10 AS Shop UNION ALL SELECT 2, 20) AS a ORDER BY Id; + +SELECT '-- 2 redefinition reads the previous definition'; +WITH + qb AS (SELECT Id, Shop FROM (SELECT 1 AS Id, 10 AS Shop UNION ALL SELECT 2, 20 UNION ALL SELECT 1, 30) WHERE Id = 1), + qb AS (SELECT sum(Shop) AS x FROM qb) +SELECT a.x FROM qb AS a; + +SELECT '-- 3 the query body reads the last definition'; +WITH d AS (SELECT 1 AS id), d AS (SELECT 2 AS id) SELECT * FROM d; + +SELECT '-- 4 chain of three definitions'; +WITH a AS (SELECT 1 AS x), a AS (SELECT x + 1 AS x FROM a), a AS (SELECT x * 10 AS x FROM a) SELECT x FROM a; + +SELECT '-- 5 two references to the redefined name'; +WITH d AS (SELECT 1 AS id), d AS (SELECT id + 1 AS id FROM d) SELECT l.id, r.id FROM d AS l CROSS JOIN d AS r; + +SELECT '-- 6 redefinition in a nested scope shadows the outer CTE'; +WITH a AS (SELECT 1 AS x) SELECT * FROM (WITH a AS (SELECT 2 AS x), a AS (SELECT x + 1 AS x FROM a) SELECT * FROM a); + +SELECT '-- 7 IN with a redefined CTE'; +WITH s AS (SELECT 1 AS x), s AS (SELECT x + 1 AS x FROM s) SELECT number FROM numbers(5) WHERE number IN s; + +SELECT '-- 8 the first definition reads the table of the same name'; +CREATE TABLE t_cte_redefinition (x UInt8) ENGINE = Memory; +INSERT INTO t_cte_redefinition VALUES (5); +WITH + t_cte_redefinition AS (SELECT x + 1 AS x FROM t_cte_redefinition), + t_cte_redefinition AS (SELECT x * 2 AS x FROM t_cte_redefinition) +SELECT x FROM t_cte_redefinition; + +SELECT '-- 9 progressive redefinition'; +WITH + base AS (SELECT number AS id, number * 10 AS val FROM numbers(3)), + joined AS (SELECT id, val, val * 2 AS doubled FROM base), + joined AS (SELECT *, doubled + 100 AS shifted FROM joined), + joined AS (SELECT *, shifted * 3 AS final_val FROM joined) +SELECT * FROM joined ORDER BY id; + +SELECT '-- 10 UNION CTE redefined'; +WITH u AS (SELECT 1 AS x UNION ALL SELECT 2), u AS (SELECT sum(x) AS x FROM u) SELECT x FROM u; + +SELECT '-- 11 consumer declared between two definitions'; +-- The analysis before v24.3 bound `b` to the first `a` (result 1, 2). Here a reference that is not +-- inside a definition of `a` binds to the last definition of `a`. +WITH a AS (SELECT 1 AS v), b AS (SELECT v FROM a), a AS (SELECT 2 AS v), c AS (SELECT v FROM a) +SELECT b.v, c.v FROM b CROSS JOIN c; + +SELECT '-- 12 view'; +CREATE VIEW v_cte_redefinition AS +WITH + raw AS (SELECT 1 AS id, 80 AS pct UNION ALL SELECT 2, 30), + enriched AS (SELECT id, pct FROM raw), + enriched AS (SELECT *, if(pct > 50, 'high', 'low') AS tier FROM enriched) +SELECT * FROM enriched ORDER BY id; +SELECT * FROM v_cte_redefinition; +SELECT * FROM v_cte_redefinition SETTINGS analyzer_compatibility_allow_cte_redefinition = 0; -- { serverError MULTIPLE_EXPRESSIONS_FOR_ALIAS } + +SELECT '-- 13 MATERIALIZED and RECURSIVE CTEs cannot be redefined'; +SET enable_materialized_cte = 1; +WITH m AS MATERIALIZED (SELECT 1 AS x), m AS (SELECT x + 1 AS x FROM m) SELECT * FROM m; -- { serverError MULTIPLE_EXPRESSIONS_FOR_ALIAS } +WITH m AS (SELECT 1 AS x), m AS MATERIALIZED (SELECT x + 1 AS x FROM m) SELECT * FROM m; -- { serverError MULTIPLE_EXPRESSIONS_FOR_ALIAS } +WITH m AS MATERIALIZED (SELECT 1 AS x), m AS MATERIALIZED (SELECT 2 AS x) SELECT 1; -- { serverError MULTIPLE_EXPRESSIONS_FOR_ALIAS } +SET enable_materialized_cte = 0; +WITH m AS MATERIALIZED (SELECT 1 AS x), m AS (SELECT 2 AS x) SELECT 1; -- { serverError MULTIPLE_EXPRESSIONS_FOR_ALIAS } +WITH RECURSIVE r AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM r WHERE n < 3), r AS (SELECT 10 AS n) SELECT * FROM r; -- { serverError MULTIPLE_EXPRESSIONS_FOR_ALIAS } + +SELECT '-- 14 identical redefinitions are distinct definitions'; +WITH + t_cte_redefinition AS (SELECT x * 2 AS x FROM t_cte_redefinition), + t_cte_redefinition AS (SELECT x * 2 AS x FROM t_cte_redefinition) +SELECT x FROM t_cte_redefinition; +WITH + d AS (SELECT 1 AS x), + d AS (SELECT x + 1 AS x FROM d), + d AS (SELECT x + 1 AS x FROM d), + d AS (SELECT l.x + r.x AS x FROM d AS l CROSS JOIN d AS r) +SELECT x FROM d; + +SELECT '-- 15 all definitions in resolve process fall through to the outer CTE'; +WITH a AS (SELECT 100 AS x) SELECT x FROM (WITH a AS (SELECT x + 1 AS x FROM a), a AS (SELECT x + 1 AS x FROM a) SELECT x FROM a); + +DROP VIEW v_cte_redefinition; +DROP TABLE t_cte_redefinition; From 85fe9c4f83a1c464dae46e8c15ae884ef2b486a0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 11:14:09 +0000 Subject: [PATCH 071/296] Backport #111535 to 26.8: Fix logical error !raw_type->isNullable() in AND-compare chain optimization --- .../Passes/LogicalExpressionOptimizerPass.cpp | 12 +- ...pare_chain_nothing_collapsed_and.reference | 19 +++ ...nd_compare_chain_nothing_collapsed_and.sql | 131 ++++++++++++++++++ 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference create mode 100644 tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 793cfb6e64970..1f27cc986e5d1 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -899,9 +899,17 @@ static AddComparisonFilterResult addComparisonFilter( return AddComparisonFilterResult::ADDED; } + /// A comparison with a nullable result is ambiguous under NULL and must not be pruned or folded; + /// keep it as-is. Test the comparison node's result type, not the raw operand type, so nested and + /// carrier-hidden nullability (e.g. `LowCardinality(Nullable)`, `Dynamic`, `Variant`) is caught. + if (isNullableOrLowCardinalityNullable(new_filter.original_node->getResultType())) + { + filter_map[expression].opaque_filters.push_back(std::move(new_filter)); + return AddComparisonFilterResult::ADDED; + } + /// Step 1: convert the constant to the column's type for uniform comparison. const auto & raw_type = expression->getResultType(); - chassert(!raw_type->isNullable()); auto expr_type = removeLowCardinality(raw_type); new_filter.converted_value = tryConvertToColumnType(new_filter.constant_node, expr_type); @@ -1961,7 +1969,7 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont expression = lhs; } - /// Both sides are non-constant — keep as-is. + /// Both sides are non-constant (or the constant is NULL-valued) — keep as-is. if (!constant) { all_operands.emplace_back(argument_index, argument); diff --git a/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference new file mode 100644 index 0000000000000..b170576ac2c00 --- /dev/null +++ b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference @@ -0,0 +1,19 @@ +1 +1 +1 +[10] +[10] +[10] +[10] +1 +1 +1 +1 +1 +1 +1 +1 +[10] +[10] +1 +1 diff --git a/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql new file mode 100644 index 0000000000000..4bb4cb0727345 --- /dev/null +++ b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql @@ -0,0 +1,131 @@ +-- The `optimize_and_compare_chain` / `optimize_redundant_comparisons` passes prune and fold an +-- AND-chain of comparisons. Both skip a Nullable-typed AND, because a comparison over a Nullable +-- operand cannot participate in range pruning. That guard checks only the AND's own result type, +-- which is unsound: when one AND operand is `Nothing`-typed, the function resolver collapses the +-- whole AND's result type to bare `Nothing` (`Nothing::isNullable()` is false), so the guard passes +-- even though another operand is a comparison over a directly-Nullable expression. That operand then +-- reached the pruning path and hit `chassert(!raw_type->isNullable())`. + +SET enable_analyzer = 1; + +-- 1) No logical error. These queries are otherwise invalid (a `Nothing`-typed value cannot be +-- materialized), so they must fail with a normal handled exception, never a logical error. +-- optimize_and_compare_chain path (chassert reached via the seed loop of tryOptimizeAndCompareChain): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(1::Int32)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 1; -- { serverError ILLEGAL_COLUMN } +-- optimize_redundant_comparisons path (tryOptimizeAndCompareNotEqualsChain), independent of the chain setting: +SELECT tuple((materialize(toNullable(1::Int32)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(1::Int32)) != 1) AND (assumeNotNull(materialize(toNullable(NULL))) != 2)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } + +-- 2) The optimizer still runs on such a `Nothing`-collapsed AND: the directly-Nullable operand is +-- kept as-is (the new opaque-filter fallback) while a redundant non-null comparison in the SAME +-- AND is still folded. In `x > 3 AND x > 5`, `x > 3` is redundant and pruned, so exactly one +-- `greater` survives when enabled against both when disabled. Pin both counts: a relative +-- comparison also holds when the predicates disappear entirely. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain LIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 0) WHERE explain LIKE '%function_name: greater,%'; +-- The exact predicate that used to hit the assertion is `equals(...) -> Nullable(Nothing)`; assert it +-- survives (exactly one such node) rather than being silently dropped. The chain's other `equals` +-- returns `Nothing`, so match the `Nullable(Nothing)` result type specifically. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain LIKE '%function_name: equals, function_type: ordinary, result_type: Nullable(Nothing)%'; + +-- 3) Correctness guard: a genuine (non-collapsed) AND-compare chain over a Nullable column must still +-- be pruned/folded correctly, i.e. the optimization result matches the unoptimized one, and NULLs +-- are excluded by the comparisons. `optimize_and_compare_chain` / `optimize_redundant_comparisons` +-- must not change results. +DROP TABLE IF EXISTS t_and_chain_nullable; +CREATE TABLE t_and_chain_nullable (x Nullable(Int32)) ENGINE = Memory; +INSERT INTO t_and_chain_nullable VALUES (1), (5), (NULL), (10); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x > 3) AND (x > 5) ORDER BY x SETTINGS optimize_and_compare_chain = 1); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x > 3) AND (x > 5) ORDER BY x SETTINGS optimize_and_compare_chain = 0); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 1); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 0); +DROP TABLE t_and_chain_nullable; +-- The checks above compare results, which stay equal even if `optimize_and_compare_chain` stops +-- deriving anything. Pin the derivation itself with exact node counts: `a < b AND b < 5` gains the +-- transitive `a < 5`, so the enabled tree holds exactly 3 `less` nodes against 2 when disabled. +SELECT count() = 3 FROM (EXPLAIN QUERY TREE SELECT a, b FROM values('a Int32, b Int32', (1, 2), (4, 9)) WHERE (a < b) AND (b < 5) SETTINGS optimize_and_compare_chain = 1) WHERE explain ILIKE '%function_name: less,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT a, b FROM values('a Int32, b Int32', (1, 2), (4, 9)) WHERE (a < b) AND (b < 5) SETTINGS optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: less,%'; + +-- 4) In a `Nothing`-collapsed AND, an operand may be a comparison whose constant side is itself +-- NULL-valued (e.g. `expr = NULL`), found by the AST fuzzer mutating case 2's `= 2` to `= NULL`. +-- A NULL-valued constant carries no comparable value, so `tryOptimizeAndCompareNotEqualsChain` +-- must not treat it as the constant side (it used to hit `chassert(!literal->getValue().isNull())`). +-- The whole AND is still `Nothing`-typed, so these queries fail with a normal handled exception. +-- RHS-NULL constant: +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (assumeNotNull(materialize(toNullable(NULL))) != NULL)) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- LHS-NULL constant (the other assertion branch): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (NULL = assumeNotNull(materialize(toNullable(NULL))))) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- Independent of the pruning setting (the classification loop runs unconditionally): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } + +-- 5) Liveness of the keep-as-is mechanism for the NULL-valued-constant operand: the optimizer must +-- still run on this `Nothing`-collapsed AND (so a redundant sibling is pruned) while keeping the +-- NULL-valued `equals` operands rather than skipping classification entirely. In the tree with +-- pruning on, `x > 3` is redundant given `x > 5`, so exactly one `greater` node survives; with +-- pruning off both survive; both NULL-valued `equals` operands are kept in either case. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: equals,%'; + +-- 6) A comparison whose RESULT is nullable must be kept as-is too, even when the raw operand type does +-- not report `isNullable`: `LowCardinality(Nullable(T))` (nested nullability) and the NULL-capable +-- carriers `Dynamic` / `Variant` all yield a nullable comparison result. They used to slip past the +-- guard, fold the contradictory `x = 1 AND x = 2`, and change the handled exception depending on the +-- setting. The error must now be the same regardless of the pruning / chain settings. +SET allow_suspicious_low_cardinality_types = 1; +SET allow_experimental_dynamic_type = 1; +SET allow_experimental_variant_type = 1; +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Dynamic', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Dynamic', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Variant(Int32, String)', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Variant(Int32, String)', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- Liveness for the LC(Nullable) operand: a redundant NON-nullable sibling (`> 3` given `> 5`) is still +-- pruned to one `greater` node while the LC-nullable `equals` operand is kept, proving the optimizer +-- runs and keeps only that operand opaque rather than declining the whole collapsed AND. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: equals,%'; +-- A non-collapsed `LowCardinality(Nullable(T))` chain must still return correct results, unchanged by +-- the optimization (NULL excluded, `1`/`5` filtered out; only `10` remains). +SELECT groupArray(x) FROM (SELECT x FROM values('x LowCardinality(Nullable(Int32))', 1, 5, NULL, 10) WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 1); +SELECT groupArray(x) FROM (SELECT x FROM values('x LowCardinality(Nullable(Int32))', 1, 5, NULL, 10) WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 0); + +-- 7) The carrier the AST fuzzer keeps rediscovering on master (`Logical error: +-- '!raw_type->isNullable()'`, STID `2508-50fe`, e.g. +-- https://s3.amazonaws.com/clickhouse-test-reports/json.html?REF=master&sha=d469feea5f342065ffe8b2384d4ddf354dae3978&name_0=MasterCI&name_1=Stress%20test%20%28arm_asan_ubsan%29 ). +-- It reaches the same `addComparisonFilter` through a different route than the cases above: the +-- `Nothing`-typed operand comes from `ARRAY JOIN []` (whose element type is `Nothing`) instead of +-- `assumeNotNull(materialize(toNullable(NULL)))`, and the collapsed AND sits in a `JOIN ON` +-- section. `JOIN ON` is load-bearing - the same AND in a `WHERE` is rejected earlier, so only the +-- join expression lets a `Nothing`-typed AND reach the optimizer. +DROP TABLE IF EXISTS t_and_chain_array_join; +CREATE TABLE t_and_chain_array_join (c0 Int32) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO t_and_chain_array_join VALUES (1), (2); + +-- `t2.c0 = a0` is `Nothing`-typed and collapses the AND's own result type, while +-- `toNullable(t2.c0) > 0` stays `Nullable(UInt8)` and reaches the pruning path. `ARRAY JOIN []` +-- produces no rows, so the query is valid and returns nothing; assert the empty result on every +-- combination of the two settings, since either entry point alone reaches the assertion. +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; +-- A result-only check stays green if the queries stop running the optimizer at all, so pin the +-- pruning as well: given `> 5`, the sibling `> 3` is redundant and folded away, leaving the +-- Nullable-result `greater` plus one surviving constant comparison. Both settings are pinned on +-- every query below because `clickhouse-test` randomizes `optimize_and_compare_chain`. +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 3 FROM (EXPLAIN QUERY TREE SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +-- The exact fuzzer query: here the Nullable-result comparison is over a correlated scalar +-- subquery rather than a column. `optimize_and_compare_chain` does not gate this one - +-- `tryOptimizeAndCompareChain` skips a chain holding a correlated subquery, while +-- `tryOptimizeAndCompareNotEqualsChain` has no such guard - so it arrives only via +-- `optimize_redundant_comparisons`. A correlated subquery is not supported in a join expression, +-- so the query must report that handled exception instead of aborting. +SELECT 1 AS x FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join ON (t_and_chain_array_join.c0 = a0) AND (t_and_chain_array_join.c0 != a0) AND (0 > (SELECT t_and_chain_array_join.c0)) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError NOT_IMPLEMENTED } +SELECT 1 AS x FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join ON (t_and_chain_array_join.c0 = a0) AND (t_and_chain_array_join.c0 != a0) AND (0 > (SELECT t_and_chain_array_join.c0)) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0; -- { serverError NOT_IMPLEMENTED } +DROP TABLE t_and_chain_array_join; From d5e75c168af4cb73a26af21596a3292b39460406 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 12:13:34 +0000 Subject: [PATCH 072/296] Backport #121559 to 26.8: keeper: change request queue push timeout --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 0e61bf81933b2..039ff73ba69c7 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1785,14 +1785,17 @@ void ZooKeeper::pushRequest(RequestInfo && info) info.request->spans.maybeInitialize(KeeperSpan::ClientRequestsQueue, info.request->tracing_context.get()); - if (!requests_queue.tryPush(std::move(info), args.operation_timeout_ms)) + /// A failed push kills the session (the `catch` below calls `finalize`), so be patient here. + const UInt64 push_timeout_ms = 3 * static_cast(args.session_timeout_ms); + + if (!requests_queue.tryPush(std::move(info), push_timeout_ms)) { if (requests_queue.isFinished()) throw Exception::fromMessage(Error::ZSESSIONEXPIRED, "Session expired"); throw Exception(Error::ZOPERATIONTIMEOUT, - "Cannot push request to queue within operation timeout of {} ms", - args.operation_timeout_ms); + "Cannot push request to queue within {} ms", + push_timeout_ms); } } catch (...) From 3ebbecc896aac56072ed20b1e621ba9f8a241898 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 14:03:29 +0000 Subject: [PATCH 073/296] Backport #121065 to 26.8: Keep detecting an always-false equals chain when comparison pruning is off --- .../Passes/LogicalExpressionOptimizerPass.cpp | 40 +++++++---- .../Passes/LogicalExpressionOptimizerPass.h | 2 +- src/Core/Settings.cpp | 2 +- ...comparison_filter_optimization_1.reference | 8 +-- ...comparison_filter_optimization_2.reference | 6 +- ...omparison_filter_type_conversion.reference | 10 +-- ...lse_equals_chain_without_pruning.reference | 20 ++++++ ...ays_false_equals_chain_without_pruning.sql | 66 +++++++++++++++++++ 8 files changed, 128 insertions(+), 26 deletions(-) create mode 100644 tests/queries/0_stateless/05230_always_false_equals_chain_without_pruning.reference create mode 100644 tests/queries/0_stateless/05230_always_false_equals_chain_without_pruning.sql diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 1f27cc986e5d1..520da50c21243 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -417,6 +417,8 @@ struct ExpressionFilters /// Excluded from the analysis: non-lossless conversions (they also veto the fold-to-false /// collapse), NaN constants, and everything when pruning is disabled. std::vector opaque_filters; + /// Index in `opaque_filters` of the first `equals` on this expression; set only when pruning is disabled. + std::optional first_equals_position; }; using ComparisonFilterMap = QueryTreeNodePtrWithHashMap; @@ -881,8 +883,8 @@ static void rebuildComparisonNode(ComparisonFilterInfo & filter, const ContextPt } /// Insert a new comparison filter for `expression` into `filter_map`. -/// When `enable_pruning` is true, performs type conversion, boundary folding, and -/// comparison against existing filters for the same expression. +/// Performs type conversion; with `enable_pruning` also boundary folding and comparison against +/// every existing filter for the expression, otherwise only against the first `equals` seen. /// Returns ALWAYS_FALSE if a contradiction is found, ALWAYS_TRUE if the condition holds /// for the column type or is implied by existing filters, or ADDED otherwise. static AddComparisonFilterResult addComparisonFilter( @@ -892,13 +894,6 @@ static AddComparisonFilterResult addComparisonFilter( bool enable_pruning, const ContextPtr & context) { - /// Pruning disabled — just store the filter without analysis. - if (!enable_pruning) - { - filter_map[expression].opaque_filters.push_back(std::move(new_filter)); - return AddComparisonFilterResult::ADDED; - } - /// A comparison with a nullable result is ambiguous under NULL and must not be pruned or folded; /// keep it as-is. Test the comparison node's result type, not the raw operand type, so nested and /// carrier-hidden nullability (e.g. `LowCardinality(Nullable)`, `Dynamic`, `Variant`) is caught. @@ -915,8 +910,11 @@ static AddComparisonFilterResult addComparisonFilter( new_filter.converted_value = tryConvertToColumnType(new_filter.constant_node, expr_type); /// Step 2: for integer columns, try boundary folding / float-literal rewriting. - if (auto result = tryFoldBoundaryOrRewriteFloatForIntColumn(new_filter, expr_type)) - return *result; + if (enable_pruning) + { + if (auto result = tryFoldBoundaryOrRewriteFloatForIntColumn(new_filter, expr_type)) + return *result; + } auto & filters = filter_map[expression]; @@ -931,6 +929,24 @@ static AddComparisonFilterResult addComparisonFilter( return AddComparisonFilterResult::ADDED; } + if (!enable_pruning) + { + auto result = AddComparisonFilterResult::ADDED; + if (new_filter.function == ComparisonFunction::EQUALS) + { + if (filters.first_equals_position) + { + if (compareComparisonFilters(filters.opaque_filters[*filters.first_equals_position], new_filter) + == ValueComparisonResult::ALWAYS_FALSE) + result = AddComparisonFilterResult::ALWAYS_FALSE; + } + else + filters.first_equals_position = filters.opaque_filters.size(); + } + filters.opaque_filters.push_back(std::move(new_filter)); + return result; + } + /// Step 3: compare against the existing equals/range filters. auto & range_filters = filters.range_filters; for (size_t i = 0; i < range_filters.size(); ++i) @@ -1889,7 +1905,7 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont /** Optimize AND chains by analyzing comparison conditions on the same expression. * This method performs two things in a single pass: * - * (a) Comparison chain pruning (when `optimize_redundant_comparisons` is enabled): + * (a) Comparison chain pruning (when `optimize_redundant_comparisons` is enabled, except an always-false `equals` pair): * Given an AND expression where the same column appears in multiple comparisons * against constants (e.g. `a = 3 AND a < 5 AND a > 1`), we collect all conditions * on the same non-constant expression into a per-expression `ComparisonFilterMap`. diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h index 8ffb09f5e17b7..253a3614ac309 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -107,7 +107,7 @@ namespace DB * ------------------------------- * * 8. Prune redundant comparisons and detect conflicting comparison conditions on the same expression - * within AND chains. Controlled by setting `optimize_redundant_comparisons`. + * within AND chains. Controlled by `optimize_redundant_comparisons`, except an always-false `equals` pair. * Handles all six comparison operators (=, !=, <, <=, >, >=) and their combinations: * duplicate removal, contradiction detection, and range tightening. * ------------------------------- diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 8a97e1d50638c..d8d0f8dd345ce 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8453,7 +8453,7 @@ Allow extracting common expressions from disjunctions in WHERE, PREWHERE, ON, HA Populate constant comparison in AND chains to enhance filtering ability. Support operators `<`, `<=`, `>`, `>=`, `=` and mix of them. For example, `(a < b) AND (b < c) AND (c < 5)` would be `(a < b) AND (b < c) AND (c < 5) AND indexHint(b < 5) AND indexHint(a < 5)`. The derived comparisons are wrapped in `indexHint`: they participate in index analysis (primary key, partition key, skipping indexes) and prune the read set, but cost nothing per row and do not affect PREWHERE. A comparison derived through expressions of different tables stays executable (`(t1.a < t2.b) AND (t2.b < 5)` derives plain `t1.a < 5`): it is the only condition that can be pushed below the join, where it filters a join input the original chain cannot reach. Derived comparisons that contradict an existing condition are also added as plain conditions, so the `AND` folds to `false`. )", 0) \ DECLARE(Bool, optimize_redundant_comparisons, true, R"( -Detect conflicting and redundant comparison conditions on the same expression within AND chains. For example, `a < 1 AND a > 5` would be rewritten to `false`. +Detect conflicting and redundant comparison conditions on the same expression within AND chains. For example, `a < 1 AND a > 5` would be rewritten to `false`. A contradiction between two `equals` on the same expression (for example, `a = 1 AND a = 2`) is detected independently of this setting. )", 0) \ DECLARE(UInt64, optimize_and_compare_chain_max_hash_work, 5'000'000, R"( Work budget for the `optimize_and_compare_chain` optimization during query analysis, measured in the number of query-tree nodes hashed by `getTreeHash` (the dominant cost of this optimization). Once a query has hashed more than this many nodes while applying the optimization, it stops applying it for the rest of the query. This bounds analysis time for queries with very many or very large `AND`-chains of comparisons, where the optimization can otherwise dominate analysis while folding nothing. Stopping early is always safe: it only forgoes an optimization and never changes results. Set to `0` to disable the budget (unlimited). diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference index a0f46fa1a31f6..148993579eca6 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference @@ -4,10 +4,10 @@ eq_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 3))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, 3)\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 5))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_flip -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(3, __table1.i), equals(__table1.i, 5))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_cross_same 3 30 3 c y 2024-06-15 12:00:00 @@ -15,7 +15,7 @@ eq_eq_cross_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, toUInt8(3)))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, 3)\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_cross_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, toUInt8(5)))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_float_same 3 30 3 c y 2024-06-15 12:00:00 @@ -194,5 +194,5 @@ multi_expr SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(greater(__table1.i, 1), less(__table1.i, 5), greater(__table1.f, 2.), less(__table1.f, 6.))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(greater(__table1.i, 1), less(__table1.i, 5), greater(__table1.f, 2.), less(__table1.f, 6.))\nSETTINGS optimize_redundant_comparisons = 1 multi_expr_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 5), greater(__table1.f, 1.))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference index 08f40bf45839f..4e7ebdf27a6d3 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference @@ -15,7 +15,7 @@ lc_eq_eq SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.lc, \'y\'), equals(__table1.lc, \'y\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE _CAST(equals(__table1.lc, \'y\'), \'UInt8\')\nSETTINGS optimize_redundant_comparisons = 1 lc_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.lc, \'y\'), equals(__table1.lc, \'z\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 str_eq_eq 3 30 3 c y 2024-06-15 12:00:00 @@ -23,10 +23,10 @@ str_eq_eq SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.s, \'c\'), equals(__table1.s, \'c\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.s, \'c\')\nSETTINGS optimize_redundant_comparisons = 1 str_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.s, \'c\'), equals(__table1.s, \'a\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 transitive_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, __table1.u), equals(__table1.u, 5), equals(__table1.i, 5), equals(__table1.u, 3))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 transitive_no_redundant SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(less(__table1.i, 3), greater(__table1.u, 3), less(__table1.u, 10))\nSETTINGS optimize_redundant_comparisons = 0 diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference b/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference index bceb5ddeb7ed3..76b4c36132c46 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference @@ -4,7 +4,7 @@ int_str_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, \'3\'), equals(__table1.i, 3))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, \'3\')\nSETTINGS optimize_redundant_comparisons = 1 int_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, \'3\'), equals(__table1.i, \'5\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 int_str_range 3 30 3 c y 2024-06-15 12:00:00 @@ -17,7 +17,7 @@ float_int_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, 3), equals(__table1.f, 3.))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.f, 3)\nSETTINGS optimize_redundant_comparisons = 1 float_int_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, 3), equals(__table1.f, 4))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 float_int_range 1 10 1.5 a x 2024-01-01 00:00:00 @@ -42,7 +42,7 @@ float_str_and_int SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, \'3.0\'), less(__table1.f, 5))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.f, \'3.0\')\nSETTINGS optimize_redundant_comparisons = 1 float_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, \'3.0\'), equals(__table1.f, \'5.0\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 float_str_range 1 10 1.5 a x 2024-01-01 00:00:00 @@ -57,7 +57,7 @@ dt_str_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, \'2024-06-15 12:00:00\'), equals(__table1.dt, \'2024-06-15 12:00:00\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.dt, \'2024-06-15 12:00:00\')\nSETTINGS optimize_redundant_comparisons = 1 dt_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, \'2024-06-15 12:00:00\'), equals(__table1.dt, \'2025-01-01 00:00:00\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 dt_str_range 3 30 3 c y 2024-06-15 12:00:00 @@ -78,7 +78,7 @@ dt_int_eq_prune SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, 1718452800), greater(__table1.dt, 1704067200))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.dt, 1718452800)\nSETTINGS optimize_redundant_comparisons = 1 dt_int_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, 1718452800), equals(__table1.dt, 1735689600))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 dt_int_range 3 30 3 c y 2024-06-15 12:00:00 diff --git a/tests/queries/0_stateless/05230_always_false_equals_chain_without_pruning.reference b/tests/queries/0_stateless/05230_always_false_equals_chain_without_pruning.reference new file mode 100644 index 0000000000000..880c4353ea110 --- /dev/null +++ b/tests/queries/0_stateless/05230_always_false_equals_chain_without_pruning.reference @@ -0,0 +1,20 @@ +1 +1 +1 +1 +0 +0 +[1] +[1] +1 +1 +1 +1 +0 +0 +0 +0 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/05230_always_false_equals_chain_without_pruning.sql b/tests/queries/0_stateless/05230_always_false_equals_chain_without_pruning.sql new file mode 100644 index 0000000000000..4ad605f383b8c --- /dev/null +++ b/tests/queries/0_stateless/05230_always_false_equals_chain_without_pruning.sql @@ -0,0 +1,66 @@ +-- `optimize_redundant_comparisons` gates comparison-chain pruning, boundary folding and range +-- strengthening, all introduced together with the setting. Detecting that two `equals` on the same +-- expression carry different values, and folding the AND to `false`, is older: it ran unconditionally +-- before the setting existed. Leaving that fold behind the setting made `compatibility` below the +-- release that added it, and an explicit `optimize_redundant_comparisons = 0`, execute the full plan +-- of a query whose filter is always false. +-- Every query pins `optimize_and_compare_chain` (the test runner randomizes it, and it derives +-- transitive conjuncts that change node counts) and `enable_analyzer = 1` (the pass is analyzer-only). + +-- 1) The fold happens with pruning disabled. Counting `equals` nodes rather than matching a constant's +-- rendered value: 'Low' and 'Medium' also appear in the fixture's own arguments. +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT s FROM values('s String', ('Low'), ('Medium')) WHERE (s = 'Low') AND (s = 'Medium') SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: equals,%'; +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT s FROM values('s String', ('Low'), ('Medium')) WHERE (s = 'Low') AND (s = 'Medium') SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: equals,%'; +-- A run of identical `equals` before the conflicting one: the conflict is found against one stored +-- representative, whose slot must survive the growth of the filter list. +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT s FROM values('s String', ('Low'), ('Medium')) WHERE (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Low') AND (s = 'Medium') SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: equals,%'; + +-- 2) The same fold through the reported carrier: `compatibility` below the release that added the +-- setting resolves it to `false`. +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT s FROM values('s String', ('Low'), ('Medium')) WHERE (s = 'Low') AND (s = 'Medium') SETTINGS enable_analyzer = 1, compatibility = '25.12', optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: equals,%'; + +-- 3) The reported shape: a LEFT JOIN onto an aggregated subquery, filtered by an always-false pair of +-- `equals` on the same expression. The fold must keep the aggregation from running at all, not +-- merely return no rows -- `throwIf` in the subquery's own WHERE cannot be pruned away, so the +-- query raises FUNCTION_THROW_IF_VALUE_IS_NON_ZERO if the right side is read. +DROP TABLE IF EXISTS t_left; +DROP TABLE IF EXISTS t_right; +CREATE TABLE t_left (k String, sev String) ENGINE = MergeTree ORDER BY k; +CREATE TABLE t_right (k String, sev String, v UInt32) ENGINE = MergeTree ORDER BY k; +INSERT INTO t_left SELECT toString(number), 'Low' FROM numbers(200); +INSERT INTO t_right SELECT toString(number % 100), 'Low', number FROM numbers(500); +SELECT count() FROM (SELECT m.k FROM t_left AS m LEFT JOIN (SELECT k, argMax(sev, v) AS sev FROM t_right WHERE throwIf(v >= 0, 'right side executed') GROUP BY k) AS p ON m.k = p.k WHERE (ifNull(p.sev, m.sev) = 'Low') AND (ifNull(p.sev, m.sev) = 'Medium') SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0); +SELECT count() FROM (SELECT m.k FROM t_left AS m LEFT JOIN (SELECT k, argMax(sev, v) AS sev FROM t_right WHERE throwIf(v >= 0, 'right side executed') GROUP BY k) AS p ON m.k = p.k WHERE (ifNull(p.sev, m.sev) = 'Low') AND (ifNull(p.sev, m.sev) = 'Medium') SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 0); +DROP TABLE t_left; +DROP TABLE t_right; + +-- 4) Two `equals` conflict only when their values differ in the column's type, never merely because a +-- second `equals` arrives: `1` and `1.0` are the same Float64, so the row survives. +SELECT groupArray(x) FROM (SELECT x FROM values('x Float64', (1.0), (2.0)) WHERE (x = 1) AND (x = 1.0) ORDER BY x SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0); +SELECT groupArray(x) FROM (SELECT x FROM values('x Float64', (1.0), (2.0)) WHERE (x = 1) AND (x = 1.0) ORDER BY x SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 0); + +-- 5) A conflict does not collapse an AND that also holds a comparison whose constant cannot be +-- converted: executing it raises TYPE_MISMATCH, and dropping it would make the error depend on the +-- setting. The convertible counterpart below shows the conflict is otherwise found on both settings. +SELECT count() FROM values('i Int32', (1)) WHERE (i = 1) AND (i = 2) AND (i > 'str') SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0; -- { serverError TYPE_MISMATCH } +SELECT count() FROM values('i Int32', (1)) WHERE (i = 1) AND (i = 2) AND (i > 'str') SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 0; -- { serverError TYPE_MISMATCH } +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT i FROM values('i Int32', (1)) WHERE (i = 1) AND (i = 2) AND (i > 0) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: equals,%'; +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT i FROM values('i Int32', (1)) WHERE (i = 1) AND (i = 2) AND (i > 0) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: equals,%'; + +-- 6) Only the equals/equals conflict is ungated. A contradiction between two ranges is part of what +-- the setting introduced and stays gated: one `less` survives with pruning off, none with it on. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT a FROM values('a Int32', (3)) WHERE (a < 1) AND (a > 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: less,%'; +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT a FROM values('a Int32', (3)) WHERE (a < 1) AND (a > 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: less,%'; + +-- 7) `optimize_and_compare_chain` appends a contradiction it derives (`x = 3 AND x = y AND y = 5` +-- yields `x = 5`) as a plain conjunct, for this pass to fold. With pruning off that conjunct is now +-- folded too, so the derived contradiction collapses the AND; with the chain pass off nothing is +-- derived and nothing collapses. The result is empty in every combination either way. +SELECT count() FROM values('x Int32, y Int32', (3, 5), (3, 3)) WHERE (x = 3) AND (x = y) AND (y = 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0; +SELECT count() FROM values('x Int32, y Int32', (3, 5), (3, 3)) WHERE (x = 3) AND (x = y) AND (y = 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 1; +SELECT count() FROM values('x Int32, y Int32', (3, 5), (3, 3)) WHERE (x = 3) AND (x = y) AND (y = 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 0; +SELECT count() FROM values('x Int32, y Int32', (3, 5), (3, 3)) WHERE (x = 3) AND (x = y) AND (y = 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 1; +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT x FROM values('x Int32, y Int32', (3, 5), (3, 3)) WHERE (x = 3) AND (x = y) AND (y = 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: and,%'; +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT x FROM values('x Int32, y Int32', (3, 5), (3, 3)) WHERE (x = 3) AND (x = y) AND (y = 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 0, optimize_and_compare_chain = 1) WHERE explain ILIKE '%function_name: and,%'; +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT x FROM values('x Int32, y Int32', (3, 5), (3, 3)) WHERE (x = 3) AND (x = y) AND (y = 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: and,%'; +SELECT count() = 0 FROM (EXPLAIN QUERY TREE SELECT x FROM values('x Int32, y Int32', (3, 5), (3, 3)) WHERE (x = 3) AND (x = y) AND (y = 5) SETTINGS enable_analyzer = 1, optimize_redundant_comparisons = 1, optimize_and_compare_chain = 1) WHERE explain ILIKE '%function_name: and,%'; From 30f0fd25292a6b1c30a570a7cc7bbb660605e729 Mon Sep 17 00:00:00 2001 From: Elmi Ahmadov Date: Wed, 23 Sep 2026 14:42:48 +0000 Subject: [PATCH 074/296] Backport #114728 to 26.8: Text index: support '%' and '%' patterns for LIKE/ILIKE optimization --- .../mergetree-family/textindexes.mdx | 29 +- src/Core/Settings.cpp | 5 +- .../MergeTree/MergeTreeIndexConditionText.cpp | 142 +++++--- .../MergeTree/MergeTreeIndexConditionText.h | 1 + .../MergeTree/MergeTreeReaderTextIndex.cpp | 25 +- src/Storages/MergeTree/TextIndexAnalyzer.cpp | 23 +- src/Storages/MergeTree/TextIndexAnalyzer.h | 2 + tests/performance/text_index_like.xml | 18 +- ...6_text_index_function_like_affix.reference | 123 +++++++ .../02346_text_index_function_like_affix.sql | 338 ++++++++++++++++++ ...4050_text_index_starts_ends_with.reference | 28 +- ...04061_text_index_json_all_values.reference | 8 +- 12 files changed, 649 insertions(+), 93 deletions(-) create mode 100644 tests/queries/0_stateless/02346_text_index_function_like_affix.reference create mode 100644 tests/queries/0_stateless/02346_text_index_function_like_affix.sql diff --git a/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx b/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx index 2866642128ba2..29a4dbc5d465a 100644 --- a/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx +++ b/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx @@ -528,8 +528,8 @@ Search tokens that the postprocessor maps to an empty string are ignored, i.e. t | [hasAnyTokens(col, arr)](/reference/functions/regular-functions/string-search-functions#hasAnyTokens) | no (array elements are tokens as-is) | all | yes | | [hasAllTokens(col, arr)](/reference/functions/regular-functions/string-search-functions#hasAllTokens) | no (array elements are tokens as-is) | all | yes | | [hasPhrase](/reference/functions/regular-functions/string-search-functions#hasPhrase) | yes | `splitByNonAlpha`, `splitByString`, `splitByRegexp`³, `ngrams`, `asciiCJK`, `icu` | yes³ | -| [startsWith](/reference/functions/regular-functions/string-functions#startsWith) | yes | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK` | yes | -| [endsWith](/reference/functions/regular-functions/string-functions#endsWith) | yes | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK` | yes | +| [startsWith](/reference/functions/regular-functions/string-functions#startsWith) | yes⁴ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`, `array`⁴ | yes⁴ | +| [endsWith](/reference/functions/regular-functions/string-functions#endsWith) | yes⁴ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`, `array`⁴ | yes⁴ | | [like](/reference/functions/regular-functions/string-search-functions#like) | yes¹ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`¹ | yes¹ | | [match](/reference/functions/regular-functions/string-search-functions#match) | yes¹ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`¹ | yes¹ | | [ilike](/reference/functions/regular-functions/string-search-functions#like) | yes² (`lower`/`upper` only) | `splitByNonAlpha`, `array`² | no² | @@ -542,14 +542,19 @@ Search tokens that the postprocessor maps to an empty string are ignored, i.e. t | [hasAll](/reference/functions/regular-functions/array-functions#hasAll) | yes | `array` | yes | ¹ `LIKE` and `match` use direct read as a hint for the listed tokenizers, otherwise they fall back to brute-force scan. -`LIKE` additionally supports a *direct read (without hint)* (enabled via `use_text_index_like_evaluation_by_dictionary_scan`) for `splitByNonAlpha` and `array` tokenizers without preprocessor or postprocessor. +`LIKE` additionally supports evaluation by a dictionary scan (enabled via `use_text_index_like_evaluation_by_dictionary_scan`) for `splitByNonAlpha` and `array` tokenizers without preprocessor or postprocessor. +A `%value%` pattern is then a *direct read (without hint)*, while `value%` and `%value` patterns remain a hint, see [LIKE/ILIKE queries](#like-ilike-queries-perf). -² `ILIKE` is only supported via direct read (without hint) (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer). -There is no fallback to using the index as a hint: if the setting is disabled or the tokenizer is not in the supported set, the index is not used for `ILIKE`. +² `ILIKE` is only supported via evaluation by a dictionary scan (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer). +There is no fallback to using the index as a hint for patterns the dictionary scan does not support: if the setting is disabled or the tokenizer is not in the supported set, the index is not used for `ILIKE`. The preprocessor, if present, must be `lower` or `upper`; postprocessors are not supported. ³ `hasPhrase` on a `splitByRegexp` text index does **not** support a postprocessor: the combination is rejected with an exception, because the postprocessor row-level rewrite assumes whitespace-splitting `splitByNonAlpha`-style tokens. Without a postprocessor, `splitByRegexp` is fully supported by `hasPhrase`. +⁴ `startsWith` and `endsWith` search the complete tokens of the needle, and the token at the open end of the needle is incomplete because the value continues there: `startsWith(col, 'ClickHouse is')` searches the token `ClickHouse`, while `startsWith(col, 'ClickHouse')` has no complete token to search. +The latter is instead evaluated by a dictionary scan (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer, no preprocessor or postprocessor), which is also the path taken by `col LIKE 'ClickHouse%'` because the analyzer pass [optimize_rewrite_like_perfect_affix](/reference/settings/session-settings/optimize-rewrite#optimize_rewrite_like_perfect_affix) rewrites it into `startsWith`. +See [LIKE/ILIKE queries](#like-ilike-queries-perf). + **Experimental: Support phrase search argument (optional)**. Experimental parameter `support_phrase_search` (default: `0`) controls whether the index stores token positions. @@ -1393,10 +1398,20 @@ This ordering enables skipping even more data granules than the granules skipped ### LIKE/ILIKE queries {#like-ilike-queries-perf} -When a LIKE/ILIKE query pattern is `%%` and the text index tokenizer is `splitByNonAlpha` or `array`, ClickHouse leverages the inverted index to speed up LIKE/ILIKE queries significantly. To achieve that, ClickHouse scans the inverted index dictionary instead of a full-table scan to find the matching pattern. +When a LIKE/ILIKE query pattern is `%%`, `%` or `%` and the text index tokenizer is `splitByNonAlpha` or `array`, ClickHouse leverages the inverted index to speed up LIKE/ILIKE queries significantly. To achieve that, ClickHouse scans the inverted index dictionary instead of a full-table scan to find the matching pattern. + +How the result of the dictionary scan is used depends on where the needle is anchored: +- `%value%` matches a row if and only if it matches one of the row's tokens, so the index decides the query on its own: a [direct read (without hint)](#direct-read) that removes the original predicate. +- `value%` and `%value` anchor the needle at the whole value, whereas the dictionary scan can only anchor it at a token, so the scan returns a superset of the matching rows and is used as a [direct read as a hint](#direct-read). + +The same dictionary scan serves `startsWith(col, 'value')` and `endsWith(col, 'value')` when the needle has no complete token to search. +This is the path most `value%` and `%value` patterns actually take, because the analyzer pass [optimize_rewrite_like_perfect_affix](/reference/settings/session-settings/optimize-rewrite#optimize_rewrite_like_perfect_affix) (enabled by default) rewrites `col LIKE 'value%'` into `startsWith(col, 'value')` and `col LIKE '%value'` into `endsWith(col, 'value')`. +Needles that span several tokens keep using the complete tokens of the needle and do not need a dictionary scan. When the optimization is enabled, LIKE/ILIKE queries should be significantly faster than a full-table scan. However, when the pattern matches most dictionary tokens, the performance can be worse compared to a full-table scan. Luckily, there is a fallback mechanism to prevent that. +The speed-up of a `value%` or `%value` pattern comes from skipping granules, so it depends on how the matching rows are laid out. Needles that match rows in every granule prune nothing, and the query pays for the dictionary scan on top of the full scan it would have done anyway. Disable [use_text_index_like_evaluation_by_dictionary_scan](/reference/settings/session-settings/use-text#use_text_index_like_evaluation_by_dictionary_scan) for such workloads. + The optimization is controlled by a setting: - [use_text_index_like_evaluation_by_dictionary_scan](/reference/settings/session-settings/use-text#use_text_index_like_evaluation_by_dictionary_scan) @@ -1404,7 +1419,7 @@ The fallback mechanism is controlled by two settings: - [text_index_like_min_pattern_length](/reference/settings/session-settings/text-index#text_index_like_min_pattern_length) - [text_index_like_max_postings_to_read](/reference/settings/session-settings/text-index#text_index_like_max_postings_to_read) -This optimization supports only functions `like` and `ilike`. +This optimization supports only functions `like`, `ilike`, `startsWith`, and `endsWith`. ### Trivial count queries {#count-queries-perf} diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 8a97e1d50638c..fa6107178cb1b 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8899,9 +8899,12 @@ Maximal selectivity of the filter to use the hint built from the inverted text i )", 0) \ DECLARE(Bool, use_text_index_like_evaluation_by_dictionary_scan, true, R"( Enable evaluation of LIKE/ILIKE queries by scanning the inverted text index dictionary. + +The accelerated patterns are `%value%`, `value%` and `%value`, as well as the `startsWith` and `endsWith` calls that `optimize_rewrite_like_perfect_affix` rewrites into `value%` and `%value`. )", 0) \ DECLARE(UInt64, text_index_like_min_pattern_length, 4, R"( -Minimum length of the alphanumeric needle in a LIKE/ILIKE pattern required to use the text index LIKE evaluation by the dictionary scan. +Minimum length of the alphanumeric needle in a LIKE/ILIKE pattern, or of a `startsWith`/`endsWith` needle, +required to use the text index LIKE evaluation by the dictionary scan. Patterns shorter than this threshold match too many dictionary tokens and are skipped to avoid expensive scans. Requires `use_text_index_like_evaluation_by_dictionary_scan` to be enabled. diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index 548148d12bd7d..d8c5094ce71c2 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -382,7 +382,6 @@ namespace /// A query in `All` mode folds postings by intersection, so a partially folded /// posting list is a superset of the result and can be used for pruning right away. bool queryMayBeTrueInRange( - const TextSearchQuery & query, const TextIndexAnalyzer::QueryBuilder & query_builder, const std::optional & current_range, TextSearchMode search_mode) @@ -391,8 +390,8 @@ bool queryMayBeTrueInRange( if (query_builder.is_failed) return false; - /// Pattern bypass means analysis is incomplete, so conservatively return true. - if (query_builder.is_bypassed && !query.getPatterns().empty()) + /// An incomplete scan may have missed matching tokens, so nothing can be pruned. + if (query_builder.is_analysis_incomplete) return true; if (!current_range.has_value()) @@ -424,7 +423,7 @@ bool hasAnyTokensInRange(const TextSearchQuery & query, const TextIndexAnalyzer: if (query.getTokens().empty()) return false; - return queryMayBeTrueInRange(query, query_builder, current_range, TextSearchMode::Any); + return queryMayBeTrueInRange(query_builder, current_range, TextSearchMode::Any); } bool hasAnyPatternsInRange(const TextSearchQuery & query, const TextIndexAnalyzer::QueryBuilder & query_builder, const std::optional & current_range) @@ -432,7 +431,7 @@ bool hasAnyPatternsInRange(const TextSearchQuery & query, const TextIndexAnalyze if (query.getPatterns().empty()) return false; - return queryMayBeTrueInRange(query, query_builder, current_range, TextSearchMode::Any); + return queryMayBeTrueInRange(query_builder, current_range, TextSearchMode::Any); } bool hasAllTokensOrEmptyInRange(const TextSearchQuery & query, const TextIndexAnalyzer::QueryBuilder & query_builder, const std::optional & current_range) @@ -440,7 +439,7 @@ bool hasAllTokensOrEmptyInRange(const TextSearchQuery & query, const TextIndexAn if (query.getTokens().empty()) return true; - return queryMayBeTrueInRange(query, query_builder, current_range, TextSearchMode::All); + return queryMayBeTrueInRange(query_builder, current_range, TextSearchMode::All); } bool hasAllTokensInRange(const TextSearchQuery & query, const TextIndexAnalyzer::QueryBuilder & query_builder, const std::optional & current_range) @@ -448,7 +447,7 @@ bool hasAllTokensInRange(const TextSearchQuery & query, const TextIndexAnalyzer: if (query.getTokens().empty()) return false; - return queryMayBeTrueInRange(query, query_builder, current_range, TextSearchMode::All); + return queryMayBeTrueInRange(query_builder, current_range, TextSearchMode::All); } } @@ -786,12 +785,21 @@ VectorWithMemoryTracking MergeTreeIndexConditionText::stringLikeToTokens return VectorWithMemoryTracking(unique_tokens.begin(), unique_tokens.end()); } -std::vector MergeTreeIndexConditionText::stringLikeToPatterns(const Field & field, bool case_insensitive) const +namespace +{ + +/// '%needle%' is anchored the same way in a token as in the whole value; an affix is not. +bool isInfixPattern(const String & pattern) { - /// Only handles the pure '%value%' form: one leading '%', a non-empty alphanumeric token immediately following, - /// then one trailing '%' immediately after the token, and nothing else. - /// Returns a single-element vector on success, empty on anything more complex. - /// Only this form is eligible for direct read mode. + return pattern.starts_with('%') && pattern.ends_with('%'); +} + +} + +std::vector +MergeTreeIndexConditionText::stringLikeToPatterns(const Field & field, bool case_insensitive) const +{ + /// Handles '%value%', 'value%' and '%value' with an alphanumeric needle; rejects anything more complex. const String value = preprocessor->processConstant(field.safeGet()); if (value.empty()) @@ -801,35 +809,22 @@ std::vector MergeTreeIndexConditionText::stringLikeT const size_t length = value.size(); size_t pos = 0; - const auto is_token_char = [](unsigned char c) { return isASCII(c) && isAlphaNumericASCII(static_cast(c)); }; - - const size_t min_pattern_length = getContext()->getSettingsRef()[Setting::text_index_like_min_pattern_length]; - - /// Must start with at least one '%'. - if (data[pos] != '%') - return {}; - while (pos < length && data[pos] == '%') ++pos; + const bool has_leading_wildcard = pos > 0; + /// Alphanumeric content must follow immediately. - if (pos >= length || !is_token_char(static_cast(data[pos]))) + if (pos >= length || !isAlphaNumericASCII(data[pos])) return {}; const size_t start = pos; - while (pos < length && is_token_char(static_cast(data[pos]))) + while (pos < length && isAlphaNumericASCII(data[pos])) ++pos; const size_t end = pos; - - /// Reject short needles: it might match too many dictionary tokens. - if (end - start < min_pattern_length) - return {}; - - /// Trailing '%' must follow immediately after the content. - if (pos >= length || data[pos] != '%') - return {}; + const bool has_trailing_wildcard = pos < length && data[pos] == '%'; while (pos < length && data[pos] == '%') ++pos; @@ -838,10 +833,20 @@ std::vector MergeTreeIndexConditionText::stringLikeT if (pos < length) return {}; + /// A pattern without wildcards is a plain equality comparison, which is served by the exact tokens path. + if (!has_leading_wildcard && !has_trailing_wildcard) + return {}; + + /// Reject short needles: they might match too many dictionary tokens. + if (end - start < getContext()->getSettingsRef()[Setting::text_index_like_min_pattern_length]) + return {}; + String pattern; - pattern += '%'; + if (has_leading_wildcard) + pattern += '%'; pattern.append(data + start, end - start); - pattern += '%'; + if (has_trailing_wildcard) + pattern += '%'; std::vector patterns; if (case_insensitive) @@ -958,6 +963,11 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( const bool is_array_tokenizer = (tokenizer->getType() == ITokenizer::Type::Array); + const auto * index_column_dag_node = index_column_node.getDAGNode(); + const DataTypePtr index_column_type = index_column_dag_node ? index_column_dag_node->result_type : nullptr; + /// A UInt8 virtual column cannot carry the NULL a predicate returns for a NULL value, which NOT flips to true. + const bool affix_patterns_allowed = index_column_type && !isNullableOrLowCardinalityNullable(index_column_type); + /// like/ilike optimization is only supported for splitByNonAlpha and array tokenizers. static const std::unordered_set like_optimization_supported_tokenizers = { ITokenizer::Type::SplitByNonAlpha, @@ -1256,20 +1266,39 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( out.text_search_queries.emplace_back(std::make_shared(function_name, TextSearchMode::All, direct_read_mode, std::move(tokens))); return true; } - if (function_name == "startsWith" && tokenizer->supportsStringLike()) + if (function_name == "startsWith" || function_name == "endsWith") { if (!value_data_type.isStringOrFixedString()) return false; - auto tokens = substringToTokens(value_field, true, false); - out.function = RPNElement::FUNCTION_EQUALS; - out.text_search_queries.emplace_back(std::make_shared(function_name, TextSearchMode::All, direct_read_mode, std::move(tokens))); - return true; - } - if (function_name == "endsWith" && tokenizer->supportsStringLike()) - { - if (!value_data_type.isStringOrFixedString()) + + const bool is_prefix = (function_name == "startsWith"); + + /// A needle inside a single token yields no complete token below, so evaluate it as `LIKE 'needle%'`. + /// Safe for a literal needle: one that would need LIKE escaping is not alphanumeric and is rejected. + if (like_optimization_supported_tokenizers.contains(tokenizer->getType()) && !has_preprocessor && !has_postprocessor + && settings[Setting::use_text_index_like_evaluation_by_dictionary_scan]) + { + const auto & needle = value_field.safeGet(); + const auto affix_pattern = is_prefix ? needle + "%" : "%" + needle; + auto patterns = stringLikeToPatterns(affix_pattern, /*case_insensitive=*/ false); + if (patterns.size() == 1 && affix_patterns_allowed) + { + const auto is_exact = is_array_tokenizer && candidate_for_exact_mode; + const auto pattern_read_mode = is_exact ? TextIndexDirectReadMode::Exact : direct_read_mode; + + out.function = RPNElement::FUNCTION_LIKE; + out.text_search_queries.emplace_back( + std::make_shared( + function_name, TextSearchMode::Any, pattern_read_mode, + VectorWithMemoryTracking(), std::move(patterns))); + return true; + } + } + + if (!tokenizer->supportsStringLike()) return false; - auto tokens = substringToTokens(value_field, false, true); + + auto tokens = substringToTokens(value_field, is_prefix, !is_prefix); out.function = RPNElement::FUNCTION_EQUALS; out.text_search_queries.emplace_back(std::make_shared(function_name, TextSearchMode::All, direct_read_mode, std::move(tokens))); return true; @@ -1282,7 +1311,7 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( if (like_optimization_supported_tokenizers.contains(tokenizer->getType()) && !has_preprocessor && !has_postprocessor && settings[Setting::use_text_index_like_evaluation_by_dictionary_scan]) { - /// TODO(ahmadov): Only '%foo%' pattern is eligible for direct read mode. An empty vector means the pattern is too complex. + /// TODO(ahmadov): Only the '%foo%', 'foo%' and '%foo' patterns are eligible for a dictionary scan. /// Add support for multiple patterns later with hint mode: /// 1. Handle multiple patterns e.g. %foo bar% -> postings_pattern(%foo) && postings_pattern(bar%) && regex(%foo bar%) /// 2. Handle exact tokens and patterns e.g. %foo bar baz% -> postings_exact(bar) && postings_pattern(%foo) && postings_pattern(bar%) @@ -1290,15 +1319,21 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( /// 4. Fall-back to the brute-force search for other cases for now. /// Follow-up: /// 1. Handle more complex patterns e.g. %foo%bar% -> (postings_pattern(%foo%) && postings_pattern(%bar%)) || postings_pattern(%foo%bar%) - auto patterns = stringLikeToPatterns(value_field, false); - if (patterns.size() == 1) + /// 2. Seek the sorted dictionary to the matching range of 'foo%' instead of scanning every block. + /// 3. Bypass a non-selective hint: it prunes nothing and still costs a dictionary scan. + const auto & like_pattern = value_field.safeGet(); + auto patterns = stringLikeToPatterns(value_field, /*case_insensitive=*/ false); + const bool is_infix = isInfixPattern(like_pattern); + if (patterns.size() == 1 && (is_infix || affix_patterns_allowed)) { - const auto pattern_read_mode = candidate_for_exact_mode ? TextIndexDirectReadMode::Exact : getHintOrNoneMode(); + const auto is_exact = (is_infix || is_array_tokenizer) && candidate_for_exact_mode; + const auto pattern_read_mode = is_exact ? TextIndexDirectReadMode::Exact : direct_read_mode; out.function = RPNElement::FUNCTION_LIKE; out.text_search_queries.emplace_back( std::make_shared( - function_name, TextSearchMode::Any, pattern_read_mode, VectorWithMemoryTracking(), std::move(patterns))); + function_name, TextSearchMode::Any, pattern_read_mode, + VectorWithMemoryTracking(), std::move(patterns))); return true; } } @@ -1321,15 +1356,20 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( if (has_postprocessor) return false; - auto patterns = stringLikeToPatterns(value_field, true); - if (patterns.size() == 1) + const auto & like_pattern = value_field.safeGet(); + auto patterns = stringLikeToPatterns(value_field, /*case_insensitive=*/ true); + const bool is_infix = isInfixPattern(like_pattern); + if (patterns.size() == 1 && (is_infix || affix_patterns_allowed)) { - const auto pattern_read_mode = candidate_for_exact_mode ? TextIndexDirectReadMode::Exact : getHintOrNoneMode(); + /// `getDirectReadMode` does not list `ilike`, which is served by the dictionary scan only. + const auto is_exact = (is_infix || is_array_tokenizer) && candidate_for_exact_mode; + const auto pattern_read_mode = is_exact ? TextIndexDirectReadMode::Exact : getHintOrNoneMode(); out.function = RPNElement::FUNCTION_LIKE; out.text_search_queries.emplace_back( std::make_shared( - function_name, TextSearchMode::Any, pattern_read_mode, VectorWithMemoryTracking(), std::move(patterns))); + function_name, TextSearchMode::Any, pattern_read_mode, + VectorWithMemoryTracking(), std::move(patterns))); return true; } return false; diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.h b/src/Storages/MergeTree/MergeTreeIndexConditionText.h index 442e21d6a59d4..5ac4da009fc11 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.h +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.h @@ -185,6 +185,7 @@ class MergeTreeIndexConditionText final : public IMergeTreeIndexCondition, publi /// Builds the OR-list of token sets for a `match`-style regexp, folding the required substring /// into every alternative. Returns an empty list when the regexp imposes no token requirement. std::vector> regexpToTokensForQueries(const String & regexp_string) const; + /// Supports '%needle%', 'needle%' and '%needle'. See isInfixPattern for which of them is exact. std::vector stringLikeToPatterns(const Field & field, bool case_insensitive = false) const; bool tryPrepareSetForTextSearch(const RPNBuilderTreeNode & lhs, const RPNBuilderTreeNode & rhs, const String & function_name, RPNElement & out) const; diff --git a/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp b/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp index f58e8a1200c47..2bab2319f7134 100644 --- a/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp @@ -134,16 +134,16 @@ void MergeTreeReaderTextIndex::initializeFallbackReader(const IMergeTreeReader * /// - Pattern queries (LIKE): fallback when dictionary scan is abandoned. /// - Phrase queries (hasPhrase with Exact mode): fallback when estimated cardinality is too high /// and reading position data would be slower than evaluating directly. - bool has_fallback_candidates = condition_text->hasSearchPatterns() - || std::ranges::any_of( - search_queries, - [](const auto & search_query) - { - return search_query && search_query->getSearchMode() == TextSearchMode::Phrase - && search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact; - }); + /// Only exact direct read needs it: a hint keeps the original predicate, so it can just be always true. + auto needs_fallback_for_query = [](const auto & search_query) + { + if (!search_query || search_query->getDirectReadMode() != TextIndexDirectReadMode::Exact) + return false; - if (!has_fallback_candidates) + return !search_query->getPatterns().empty() || search_query->getSearchMode() == TextSearchMode::Phrase; + }; + + if (std::ranges::none_of(search_queries, needs_fallback_for_query)) return; /// Build a fallback evaluation path. Compile each virtual column's default expression @@ -167,12 +167,7 @@ void MergeTreeReaderTextIndex::initializeFallbackReader(const IMergeTreeReader * { const auto & column = columns_to_read[i]; const auto & search_query = search_queries[i]; - if (!search_query) - continue; - - bool needs_fallback = !search_query->getPatterns().empty() - || (search_query->getSearchMode() == TextSearchMode::Phrase && search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact); - if (!needs_fallback) + if (!needs_fallback_for_query(search_query)) continue; /// Compile the virtual column's default expression (the original search predicate). diff --git a/src/Storages/MergeTree/TextIndexAnalyzer.cpp b/src/Storages/MergeTree/TextIndexAnalyzer.cpp index c1bbb05fd9d97..c99d0db329e69 100644 --- a/src/Storages/MergeTree/TextIndexAnalyzer.cpp +++ b/src/Storages/MergeTree/TextIndexAnalyzer.cpp @@ -318,6 +318,7 @@ void TextIndexAnalyzer::bypassPatternQueries() { auto & query_builder = query_builders.at(query_hash); query_builder.markBypassed(); + query_builder.is_analysis_incomplete = true; for (const auto & [query_token, _] : query_builder.tokens) queries_by_token[query_token].erase(query_hash); @@ -327,7 +328,7 @@ void TextIndexAnalyzer::bypassPatternQueries() double TextIndexAnalyzer::estimateQueryCardinality(const QueryBuilder & query_builder, size_t total_rows) const { const auto & query = *query_builder.query; - chassert(!query.getTokens().empty()); + chassert(!query.getTokens().empty() || !query.getPatterns().empty()); const double n = static_cast(total_rows); switch (query.getSearchMode()) @@ -367,6 +368,20 @@ double TextIndexAnalyzer::estimateQueryCardinality(const QueryBuilder & query_bu ? 1.0 - static_cast(query_builder.postings->cardinality()) / n : 1.0; + /// A pattern query declares no tokens, it owns the ones the dictionary scan matched. + if (query.getTokens().empty()) + { + for (const auto & [token, token_info] : query_builder.tokens) + { + if (hasReadPostings(token)) + continue; + + not_in_any *= (1.0 - static_cast(token_info->cardinality) / n); + } + + return n * (1.0 - not_in_any); + } + for (const auto & token : query.getTokens()) { auto it = query_builder.tokens.find(token); @@ -404,10 +419,8 @@ void TextIndexAnalyzer::analyzeCardinalitiesAndBypassHints(double selectivity_th if (query.getDirectReadMode() != TextIndexDirectReadMode::Hint) continue; - /// Pure-pattern queries have no declared tokens at parse time; their tokens are - /// discovered dynamically during dictionary scan. Skip the cardinality check in - /// that case — it would have no inputs to work with. - if (query.getTokens().empty()) + /// A pure-pattern query is estimated from the tokens the dictionary scan discovered. + if (query.getTokens().empty() && query_builder.tokens.empty()) continue; double estimated_cardinality = estimateQueryCardinality(query_builder, total_rows); diff --git a/src/Storages/MergeTree/TextIndexAnalyzer.h b/src/Storages/MergeTree/TextIndexAnalyzer.h index 03eba3c3958c1..fc199918aaeb5 100644 --- a/src/Storages/MergeTree/TextIndexAnalyzer.h +++ b/src/Storages/MergeTree/TextIndexAnalyzer.h @@ -42,6 +42,8 @@ class TextIndexAnalyzer bool is_failed = false; /// Query was discarded (low-selectivity hint, pattern bypass). bool is_bypassed = false; + /// The dictionary scan stopped early, so the matched tokens are incomplete and nothing can be pruned. + bool is_analysis_incomplete = false; /// Number of tokens whose posting list has already been folded into `postings`. size_t num_read_postings = 0; /// Declared tokens (`query->getTokens`) that may still contribute to an `Any` query. diff --git a/tests/performance/text_index_like.xml b/tests/performance/text_index_like.xml index 5f38d81015702..935b025a51654 100644 --- a/tests/performance/text_index_like.xml +++ b/tests/performance/text_index_like.xml @@ -1,4 +1,9 @@ + + 0 + 4 + + CREATE TABLE tab ( @@ -14,14 +19,23 @@ INSERT INTO tab SELECT number, - if(number % 100 = 0, + trimRight(if(number < 100000, repeat('clickhouse is a fast column oriented database system ', 4), - repeat('the quick brown fox jumps over the lazy dog near the river ', 4)) + repeat('the quick brown fox jumps over the lazy dog near the river ', 4))) FROM numbers(10000000) SELECT count() FROM tab WHERE message LIKE '%clickhouse%' SELECT count() FROM tab WHERE message ILIKE '%CLICKHOUSE%' + SELECT count() FROM tab WHERE message LIKE 'clickhouse%' + SELECT count() FROM tab WHERE message ILIKE 'CLICKHOUSE%' + + SELECT count() FROM tab WHERE message LIKE '%system' + SELECT count() FROM tab WHERE message ILIKE '%SYSTEM' + + SELECT count() FROM tab WHERE startsWith(message, 'clickhouse') + SELECT count() FROM tab WHERE endsWith(message, 'system') + DROP TABLE IF EXISTS tab diff --git a/tests/queries/0_stateless/02346_text_index_function_like_affix.reference b/tests/queries/0_stateless/02346_text_index_function_like_affix.reference new file mode 100644 index 0000000000000..89bb276d2e317 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_function_like_affix.reference @@ -0,0 +1,123 @@ +Test results are same with/without the optimization +-- without optimization +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[] +[] +[1] +[1,2,3,5] +[1] +[1,3] +[2,5] +[1] +[6] +[1,3] +[2,5] +[1,3] +[2,5] +-- with optimization +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[] +[] +[1] +[1,2,3,5] +[1] +[1,3] +[2,5] +[1] +[6] +[1,3] +[2,5] +[1,3] +[2,5] +-- with optimization but without hints +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[1,3] +[2,5] +Prefix and suffix patterns keep the original condition, infix patterns do not +prefix 1 1 +suffix 1 1 +prefix, no rewrite 1 1 +suffix, no rewrite 1 1 +infix 1 0 +ilike prefix 1 1 +Needles shorter than text_index_like_min_pattern_length are not evaluated by a dictionary scan +prefix 0 1 +[1,3] +[1] +Text index analysis +-- Prefix pattern should choose 1 part and 1024 granules out of 4 parts and 4096 granules +Description: text GRANULARITY 100000000 +Parts: 1/4 +Granules: 1024/4096 +-- Suffix pattern should choose 3 parts and 3072 granules out of 4 parts and 4096 granules +Description: text GRANULARITY 100000000 +Parts: 3/4 +Granules: 3072/4096 +-- Prefix pattern with a non-existent token should choose none +Description: text GRANULARITY 100000000 +Parts: 0/4 +Granules: 0/4096 +1024 +1024 +2048 +2048 +Test results are same with/without the optimization with array tokenizer +-- without optimization +[1,3] +[3] +[2,4] +[1,3] +[3] +[1,2,3,4] +[3] +-- with optimization +[1,3] +[3] +[2,4] +[1,3] +[3] +[1,2,3,4] +[3] +With the array tokenizer a token is the whole value, so an affix is exact +prefix 1 0 +suffix 1 0 +ilike prefix 1 0 +A nullable value is left to the original condition +[1,3] +[1,3] +[2] +[2] +[1,2] +[1,2] +[] +[] +[2] +[2] +[] +[] +An affix hint that prunes nothing is discarded, granule pruning is kept +1000 +0 +affix_hint_nonselective 0 1 1 +affix_hint_selective 1 0 1 +A nullable value reached through mapValues is left to the original condition +499 +499 +499 +499 +499 +499 +1 +1 diff --git a/tests/queries/0_stateless/02346_text_index_function_like_affix.sql b/tests/queries/0_stateless/02346_text_index_function_like_affix.sql new file mode 100644 index 0000000000000..bf57731ff6808 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_function_like_affix.sql @@ -0,0 +1,338 @@ +-- Tags: no-parallel-replicas +-- Tests that affix LIKE/ILIKE patterns, i.e. prefix ('value%') and suffix ('%value'), use the text index as a hint. +-- By default the analyzer rewrites such patterns into startsWith/endsWith (optimize_rewrite_like_perfect_affix), +-- so both spellings are covered here. +SET explain_query_plan_default = 'legacy'; + +SET enable_analyzer = 1; +SET use_skip_indexes_on_data_read = 1; +SET query_plan_direct_read_from_text_index = 1; +SET query_plan_text_index_add_hint = 1; +-- Pinned because the queries below assert which of the two spellings the plan ends up with. +SET optimize_rewrite_like_perfect_affix = 1; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + id UInt32, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY (id); + +-- The dictionary of this table contains tokens that match the prefix/suffix patterns below in rows +-- where the whole value does not match them, so a hint that is not verified would return extra rows. +INSERT INTO tab(id, message) VALUES + (1, 'foobar baz'), + (2, 'baz foobar'), + (3, 'foobarqux end'), + (4, 'end foobarqux'), + (5, 'quuxfoobar'), + (6, 'quuxfoobar tail'), + (7, 'nothing here'); + +SELECT 'Test results are same with/without the optimization'; + +SELECT '-- without optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 0; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'nonexistent%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%nonexistent'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND message LIKE '%baz'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' OR message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND hasToken(message, 'baz'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar baz'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'quuxfoobar tail'); +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0; + +SELECT '-- with optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 1; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'nonexistent%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%nonexistent'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND message LIKE '%baz'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' OR message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND hasToken(message, 'baz'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar baz'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'quuxfoobar tail'); +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0; + +SELECT '-- with optimization but without hints'; + +SET query_plan_text_index_add_hint = 0; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; + +SET query_plan_text_index_add_hint = 1; + +SELECT 'Prefix and suffix patterns keep the original condition, infix patterns do not'; + +-- The columns are: whether the query plan contains a text index virtual column, and whether it still +-- evaluates the original search function. The latter is absent only for an exact direct read. +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foobar%'); + +SELECT 'suffix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION endsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar'); + +SELECT 'prefix, no rewrite', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0); + +SELECT 'suffix, no rewrite', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0); + +SELECT 'infix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar%'); + +SELECT 'ilike prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION ilike(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message ILIKE 'FOOBAR%'); + +SELECT 'Needles shorter than text_index_like_min_pattern_length are not evaluated by a dictionary scan'; + +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foo%'); + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foo%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%baz'; + +DROP TABLE tab; + +SELECT 'Text index analysis'; + +CREATE TABLE tab +( + id UInt32, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY (id) +SETTINGS index_granularity = 1; + +INSERT INTO tab SELECT number, 'Hello ClickHouse' FROM numbers(1024); +INSERT INTO tab SELECT number, 'Hello World, ClickHouse is fast!' FROM numbers(1024); +INSERT INTO tab SELECT number, 'Hallo xClickHouse' FROM numbers(1024); +INSERT INTO tab SELECT number, 'ClickHousez rocks' FROM numbers(1024); + +SELECT '-- Prefix pattern should choose 1 part and 1024 granules out of 4 parts and 4096 granules'; +-- Only 'ClickHousez rocks' has a token starting with 'ClickHousez'. +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE 'ClickHousez%' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +SELECT '-- Suffix pattern should choose 3 parts and 3072 granules out of 4 parts and 4096 granules'; +-- 'ClickHouse' and 'xClickHouse' are tokens ending with 'ClickHouse', only 'ClickHousez' is not. +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE '%ClickHouse' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +SELECT '-- Prefix pattern with a non-existent token should choose none'; +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE 'random%' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +-- Three parts have a token starting with 'ClickHouse' but only one of them has rows starting with it, +-- so the hint must be verified by the original condition. +SELECT count() FROM tab WHERE message LIKE 'ClickHouse%'; +SELECT count() FROM tab WHERE message LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; + +SELECT count() FROM tab WHERE message LIKE '%ClickHouse'; +SELECT count() FROM tab WHERE message LIKE '%ClickHouse' SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; + +SELECT 'Test results are same with/without the optimization with array tokenizer'; + +CREATE TABLE tab +( + id UInt32, + tag String, + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, 'CLICKHOUSE_SQL'); + +SELECT '-- without optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 0; + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE '%Cloud'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE startsWith(tag, 'ClickHouse'); +SELECT groupArray(id) FROM tab WHERE endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE tag ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag ILIKE '%cloud'; + +SELECT '-- with optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 1; + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE '%Cloud'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE startsWith(tag, 'ClickHouse'); +SELECT groupArray(id) FROM tab WHERE endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE tag ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag ILIKE '%cloud'; + +DROP TABLE tab; + +SELECT 'With the array tokenizer a token is the whole value, so an affix is exact'; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + id UInt32, + tag String, + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, 'CLICKHOUSE_SQL'); + +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag LIKE 'ClickHouse%'); + +SELECT 'suffix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION endsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag LIKE '%Cloud'); + +SELECT 'ilike prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION ilike(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag ILIKE 'clickhouse%'); + +DROP TABLE tab; + +SELECT 'A nullable value is left to the original condition'; + +CREATE TABLE tab +( + id UInt32, + tag Nullable(String), + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, NULL); + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE NOT endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE NOT endsWith(tag, 'Cloud') SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE 'clickhouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE '%ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE '%ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE '%clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE '%clickhouse%' SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; + +SELECT 'An affix hint that prunes nothing is discarded, granule pruning is kept'; + +CREATE TABLE tab +( + id UInt64, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO tab SELECT number, multiIf(number < 1000, 'clickhouse is fast', number < 30000, 'bank of the river', 'alpha beta gamma') FROM numbers(100000); + +SELECT count() FROM tab WHERE message LIKE 'clickhouse%' SETTINGS log_comment = 'affix_hint_selective'; +SELECT count() FROM tab WHERE message LIKE 'river%' SETTINGS log_comment = 'affix_hint_nonselective'; + +SYSTEM FLUSH LOGS query_log; + +SELECT log_comment, + max(ProfileEvents['TextIndexUseHint'] > 0) AS hint_used, + max(ProfileEvents['TextIndexDiscardHint'] > 0) AS hint_discarded, + max(read_rows < 100000) AS granules_pruned +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 AND current_database = currentDatabase() + AND type = 'QueryFinish' AND log_comment IN ('affix_hint_selective', 'affix_hint_nonselective') +GROUP BY log_comment +ORDER BY log_comment; + +DROP TABLE tab; + +SELECT 'A nullable value reached through mapValues is left to the original condition'; + +CREATE TABLE tab +( + id UInt64, + m Map(String, Nullable(String)), + INDEX idx mapValues(m) TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO tab SELECT number, map('k', if(number = 0, 'foobar value', 'zulu yankee')) FROM numbers(500); +INSERT INTO tab VALUES (1000, map('k', NULL)); + +SELECT count() FROM tab WHERE NOT startsWith(m['k'], 'foobar'); +SELECT count() FROM tab WHERE NOT startsWith(m['k'], 'foobar') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE NOT m['k'] LIKE 'foobar%'; +SELECT count() FROM tab WHERE NOT m['k'] LIKE 'foobar%' SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE NOT endsWith(m['k'], 'value'); +SELECT count() FROM tab WHERE NOT endsWith(m['k'], 'value') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE startsWith(m['k'], 'foobar'); +SELECT count() FROM tab WHERE startsWith(m['k'], 'foobar') SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference b/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference index 51f83ff27583f..6090a2c07c4d5 100644 --- a/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference +++ b/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference @@ -6,7 +6,13 @@ Expression ((Project names + Projection)) Condition: true Parts: 3/3 Granules: 3/3 - Ranges: 3 + Skip + Name: idx_name_fts + Description: text GRANULARITY 100000000 + Condition: (mode: All; tokens: []) + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -15,7 +21,13 @@ Expression ((Project names + Projection)) Condition: true Parts: 3/3 Granules: 3/3 - Ranges: 3 + Skip + Name: idx_name_fts + Description: text GRANULARITY 100000000 + Condition: (mode: All; tokens: []) + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -112,9 +124,9 @@ Expression ((Project names + Projection)) Name: idx_name_fts Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) - Parts: 3/3 - Granules: 3/3 - Ranges: 3 + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -127,9 +139,9 @@ Expression ((Project names + Projection)) Name: idx_name_fts Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) - Parts: 3/3 - Granules: 3/3 - Ranges: 3 + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) diff --git a/tests/queries/0_stateless/04061_text_index_json_all_values.reference b/tests/queries/0_stateless/04061_text_index_json_all_values.reference index 494d66d4afc24..1bb69101acef6 100644 --- a/tests/queries/0_stateless/04061_text_index_json_all_values.reference +++ b/tests/queries/0_stateless/04061_text_index_json_all_values.reference @@ -22,8 +22,8 @@ SELECT id FROM tab WHERE startsWith(data.key1, 'lazy') ORDER BY id 1 Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) -Parts: 4/4 -Granules: 4/4 +Parts: 1/4 +Granules: 1/4 SELECT id FROM tab WHERE endsWith(data.key1, 'fox') ORDER BY id 0 Description: text GRANULARITY 100000000 @@ -61,8 +61,8 @@ SELECT id FROM tab WHERE startsWith(data.key1::String, 'lazy') ORDER BY id 1 Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) -Parts: 4/4 -Granules: 4/4 +Parts: 1/4 +Granules: 1/4 SELECT id FROM tab WHERE hasToken(data.key1::String, 'quick') ORDER BY id 0 2 From 28fcece7a061b30a8c8ff7e68a46797daaf0afb0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 15:14:56 +0000 Subject: [PATCH 075/296] Backport #121325 to 26.8: Bump `cctz` from 2026c to 2026d --- contrib/cctz | 2 +- ...imezone_name_file_prefix_rejected.reference | 1 + ...5233_timezone_name_file_prefix_rejected.sql | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05233_timezone_name_file_prefix_rejected.reference create mode 100644 tests/queries/0_stateless/05233_timezone_name_file_prefix_rejected.sql diff --git a/contrib/cctz b/contrib/cctz index 8e694da054a9a..bfd13df99e265 160000 --- a/contrib/cctz +++ b/contrib/cctz @@ -1 +1 @@ -Subproject commit 8e694da054a9a31d98392bf03ee188b04f810d0a +Subproject commit bfd13df99e26583c1a00edc5032964b01bab551b diff --git a/tests/queries/0_stateless/05233_timezone_name_file_prefix_rejected.reference b/tests/queries/0_stateless/05233_timezone_name_file_prefix_rejected.reference new file mode 100644 index 0000000000000..22377f01772d0 --- /dev/null +++ b/tests/queries/0_stateless/05233_timezone_name_file_prefix_rejected.reference @@ -0,0 +1 @@ +1970-01-01 00:00:00 diff --git a/tests/queries/0_stateless/05233_timezone_name_file_prefix_rejected.sql b/tests/queries/0_stateless/05233_timezone_name_file_prefix_rejected.sql new file mode 100644 index 0000000000000..7d2d21aa5244f --- /dev/null +++ b/tests/queries/0_stateless/05233_timezone_name_file_prefix_rejected.sql @@ -0,0 +1,18 @@ +-- Time zone names are untrusted input. `cctz` strips the `file:` prefix before it builds the path, +-- so `file:/abs/path` used to bypass the check that rejects names starting with `/` and opened an +-- arbitrary file. See https://github.com/ClickHouse/ClickHouse/issues/121367. + +SELECT toDateTime(0, 'file:UTC'); -- { serverError BAD_ARGUMENTS } +SELECT toDateTime(0, 'file:/usr/share/zoneinfo/UTC'); -- { serverError BAD_ARGUMENTS } +SELECT toDateTime(0, 'file:../zoneinfo/UTC'); -- { serverError BAD_ARGUMENTS } +SELECT toDateTime(0, '/usr/share/zoneinfo/UTC'); -- { serverError BAD_ARGUMENTS } +SELECT toDateTime(0, './UTC'); -- { serverError BAD_ARGUMENTS } +SELECT toDateTime(0, '~/UTC'); -- { serverError BAD_ARGUMENTS } +SELECT toDateTime(0, '../zoneinfo/UTC'); -- { serverError BAD_ARGUMENTS } +SELECT toDateTime(0, 'Etc/../Etc/UTC'); -- { serverError BAD_ARGUMENTS } +SELECT toTimeZone(toDateTime(0), 'file:UTC'); -- { serverError BAD_ARGUMENTS } +SELECT CAST(0 AS DateTime('file:UTC')); -- { serverError BAD_ARGUMENTS } +SELECT toDateTime(0, 'UTC') SETTINGS session_timezone = 'file:UTC'; -- { clientError BAD_ARGUMENTS } + +-- A regular name still works. +SELECT toDateTime(0, 'UTC'); From 4a03bb318cf3c84c914a7df4a48f33117ee99e9f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 17:28:23 +0000 Subject: [PATCH 076/296] Backport #119585 to 26.8: Detach a patch part whose index lost its content to a zeroed block --- src/Common/FailPoint.cpp | 1 + src/Storages/MergeTree/IMergeTreeDataPart.cpp | 19 ++++ .../MergeTree/MergedBlockOutputStream.cpp | 18 ++- .../MergeTree/PatchParts/PatchPartIndex.cpp | 4 + .../tests/gtest_patch_part_index_read.cpp | 82 ++++++++++++++ ..._patch_part_empty_index_detached.reference | 7 ++ .../05229_patch_part_empty_index_detached.sh | 104 ++++++++++++++++++ 7 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 src/Storages/MergeTree/PatchParts/tests/gtest_patch_part_index_read.cpp create mode 100644 tests/queries/0_stateless/05229_patch_part_empty_index_detached.reference create mode 100755 tests/queries/0_stateless/05229_patch_part_empty_index_detached.sh diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 1cc541d930982..15d565cee3fb4 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -75,6 +75,7 @@ static struct InitFiu ONCE(rmt_lightweight_update_sleep_after_block_allocation) \ ONCE(rmt_merge_task_sleep_in_prepare) \ ONCE(merge_tree_refresh_parts_throw_once) \ + REGULAR(patch_part_index_write_empty) \ ONCE(s3_read_buffer_throw_expired_token) \ ONCE(s3_send_request_throw_expired_token) \ REGULAR(s3_read_inject_etag_mismatch) \ diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 81ff0ca34e5bf..d22df1d1f9ae8 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1516,6 +1516,17 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks if (auto * constant_granularity = dynamic_cast(index_granularity.get())) constant_granularity->fixFromRowsCount(rows_count); + /// A patch part that holds rows names the parts it patches, so an index without source + /// parts is not a patch that applies to nothing - it is a file that lost its content. + /// Failing here is what keeps the acknowledged update recoverable: an empty index reports + /// data version 0, so `clearUnusedPatchParts` would find the patch materialized everywhere + /// and delete the only copy of it. An empty index belongs to an empty part alone (the + /// covering parts `cloneEmpty` creates). + if (info.isPatch() && rows_count > 0 && patch_part_index && patch_part_index->empty()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "Patch part {} has {} rows, but its index in {} references no source parts", + name, rows_count, PatchPartIndex::FILENAME); + loadExistingRowsCount(); /// Must be called after loadRowsCount() as it uses the value of `rows_count`. loadPartitionAndMinMaxIndex(); @@ -2001,7 +2012,15 @@ void IMergeTreeDataPart::loadPatchPartIndex() return; if (auto in = readFileIfExists(PatchPartIndex::FILENAME)) + { patch_part_index = PatchPartIndex::readBinary(*in); + + /// The file holds nothing but this index, so bytes left over mean its content is not what was + /// written. One corruption shape makes this check the difference between a loud and a silent + /// failure: a zeroed block parses as an index of format version `V1` with no source parts at + /// all, and everything after those nine bytes would otherwise be ignored. + assertEOF(*in); + } else throw Exception(ErrorCodes::CORRUPTED_DATA, "Missing file {} in patch part {}", PatchPartIndex::FILENAME, name); } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 87693b515eccf..167c6a53ef736 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,11 @@ namespace MergeTreeSetting extern const MergeTreeSettingsBool enable_index_granularity_compression; } +namespace FailPoints +{ + extern const char patch_part_index_write_empty[]; +} + MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeMutableDataPartPtr & data_part, MergeTreeSettingsPtr data_settings, @@ -380,9 +386,19 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis /// throws `CORRUPTED_DATA` otherwise, including for empty covering parts. if (new_part->info.isPatch()) { + /// Writes an index without source parts, which is the corruption shape the load path + /// rejects: a patch part that holds rows but names no part it patches. Only for tests. + bool write_empty_index = false; + fiu_do_on(FailPoints::patch_part_index_write_empty, { write_empty_index = true; }); + write_hashed_file(PatchPartIndex::FILENAME, [&](auto & buffer) { - new_part->getPatchPartIndex().writeBinary(buffer); + const auto & patch_part_index = new_part->getPatchPartIndex(); + + if (write_empty_index) + patch_part_index.cloneEmpty().writeBinary(buffer); + else + patch_part_index.writeBinary(buffer); }); } } diff --git a/src/Storages/MergeTree/PatchParts/PatchPartIndex.cpp b/src/Storages/MergeTree/PatchParts/PatchPartIndex.cpp index 61bf1bb38a49e..2ebcc2b6f0cb6 100644 --- a/src/Storages/MergeTree/PatchParts/PatchPartIndex.cpp +++ b/src/Storages/MergeTree/PatchParts/PatchPartIndex.cpp @@ -271,6 +271,10 @@ PatchPartIndex PatchPartIndex::readBinary(ReadBuffer & in) } res.buildSourcePartsByVersion(); + + /// Consumes exactly the bytes of the index and nothing after them: the index is also embedded in + /// larger streams, so whether anything may follow it is for the caller to decide (see + /// `IMergeTreeDataPart::loadPatchPartIndex` for the file that holds nothing else). return res; } diff --git a/src/Storages/MergeTree/PatchParts/tests/gtest_patch_part_index_read.cpp b/src/Storages/MergeTree/PatchParts/tests/gtest_patch_part_index_read.cpp new file mode 100644 index 0000000000000..92c3b967e9b8b --- /dev/null +++ b/src/Storages/MergeTree/PatchParts/tests/gtest_patch_part_index_read.cpp @@ -0,0 +1,82 @@ +#include + +#include +#include +#include +#include + +using namespace DB; + +/// A patch part carries the index of the parts it patches in `source_parts.dat`, and that file holds +/// nothing else. A zeroed block of the same size used to parse as a valid index: the first byte `0` is +/// the `V1` format version, the next eight zero bytes are `num_parts = 0`, and everything after them +/// was ignored. The load path now asserts that the file ends where the index ends, which relies on +/// `readBinary` consuming exactly the bytes of the index: the same parser reads the index out of larger +/// streams (the in-memory part data exchanged between replicas), where bytes do follow it. +TEST(PatchPartIndexRead, ConsumesExactlyTheIndex) +{ + PatchPartIndex index(MergeTreePatchPartsVersion::V1, ""); + index.addSourcePart("all_1_1_0", 2); + index.addSourcePart("all_2_2_0", 3); + + String written; + { + WriteBufferFromString out(written); + index.writeBinary(out); + } + + { + ReadBufferFromString in(written); + auto read_index = PatchPartIndex::readBinary(in); + EXPECT_FALSE(read_index.empty()); + EXPECT_EQ(read_index.getMinDataVersion("all_1_1_0"), 2); + EXPECT_EQ(read_index.getMaxDataVersion("all_2_2_0"), 3); + EXPECT_TRUE(in.eof()); + } + + /// The corruption shape from the issue: the file keeps its size, but its content is gone. The + /// parser stops after the nine bytes it understands, and what the loader does next is what turns + /// the rest of the block into a loud failure instead of an accepted empty index. + String zero_filled(written.size(), '\0'); + ASSERT_GT(zero_filled.size(), 9u); + { + ReadBufferFromString in(zero_filled); + auto read_index = PatchPartIndex::readBinary(in); + EXPECT_TRUE(read_index.empty()); + EXPECT_EQ(in.count(), 9u); + EXPECT_FALSE(in.eof()); + EXPECT_ANY_THROW(assertEOF(in)); + } + + /// Bytes after a well-formed index are left in the stream for the caller. + { + /// `ReadBufferFromString` only borrows the bytes, so the string has to outlive the buffer. + String with_trailing_bytes = written + String("tail"); + ReadBufferFromString in(with_trailing_bytes); + auto read_index = PatchPartIndex::readBinary(in); + EXPECT_FALSE(read_index.empty()); + EXPECT_EQ(in.count(), written.size()); + + String rest; + readStringUntilEOF(rest, in); + EXPECT_EQ(rest, "tail"); + } +} + +/// An index without source parts is still well-formed on its own - it is what an empty covering part +/// carries - so the parser accepts it and the load path is what rejects it for a part that holds rows. +TEST(PatchPartIndexRead, AcceptsAnEmptyIndex) +{ + PatchPartIndex index(MergeTreePatchPartsVersion::V1, ""); + + String written; + { + WriteBufferFromString out(written); + index.writeBinary(out); + } + + ReadBufferFromString in(written); + auto read_index = PatchPartIndex::readBinary(in); + EXPECT_TRUE(read_index.empty()); + EXPECT_TRUE(in.eof()); +} diff --git a/tests/queries/0_stateless/05229_patch_part_empty_index_detached.reference b/tests/queries/0_stateless/05229_patch_part_empty_index_detached.reference new file mode 100644 index 0000000000000..973ca647e6455 --- /dev/null +++ b/tests/queries/0_stateless/05229_patch_part_empty_index_detached.reference @@ -0,0 +1,7 @@ +updated sum: 999500 +active patch parts: 1 +active patch parts after reload: 0 +detached patch parts: 1 broken-on-start +sum after reload: 499500 +reloaded sum: 999500 +reloaded detached parts: 0 diff --git a/tests/queries/0_stateless/05229_patch_part_empty_index_detached.sh b/tests/queries/0_stateless/05229_patch_part_empty_index_detached.sh new file mode 100755 index 0000000000000..dc15bec6c9af2 --- /dev/null +++ b/tests/queries/0_stateless/05229_patch_part_empty_index_detached.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Tags: no-replicated-database, no-shared-merge-tree, no-parallel +# no-replicated-database, no-shared-merge-tree: the test reloads the table and reads +# `system.detached_parts`, which a replicated table recovers from another replica. +# no-parallel: `patch_part_index_write_empty` is server-global, so a concurrent lightweight +# `UPDATE` would write a corrupted patch part too. + +# A patch part carries the index of the parts it patches in `source_parts.dat`. An index without +# source parts belongs to an empty covering part alone: for a patch part that holds rows it means the +# file lost its content, which used to load clean and silently unapply an acknowledged update - and +# then let `clearUnusedPatchParts` delete the only copy of it, because an empty index reports data +# version 0. The part is detached as broken on load now, so the rows stay recoverable. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS t_patch_empty_index SYNC; + CREATE TABLE t_patch_empty_index (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; + + INSERT INTO t_patch_empty_index SELECT number, number FROM numbers(1000); +" + +# The failpoint is server-global, so it is cleared even if a query below fails. +trap '${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT patch_part_index_write_empty"' EXIT + +${CLICKHOUSE_CLIENT} --query "SYSTEM ENABLE FAILPOINT patch_part_index_write_empty" + +${CLICKHOUSE_CLIENT} --query " + SET enable_lightweight_update = 1; + UPDATE t_patch_empty_index SET v = v + 1000 WHERE id < 500; +" + +${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT patch_part_index_write_empty" + +# The update is acknowledged and visible: the in-memory index of the patch part is intact, only its +# file is not. +echo -n 'updated sum: ' +${CLICKHOUSE_CLIENT} --query "SELECT sum(v) FROM t_patch_empty_index" + +echo -n 'active patch parts: ' +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM system.parts + WHERE database = currentDatabase() AND table = 't_patch_empty_index' + AND active AND startsWith(name, 'patch-') +" + +${CLICKHOUSE_CLIENT} --query "DETACH TABLE t_patch_empty_index SYNC" +# Loading the corrupted part logs the `CORRUPTED_DATA` exception it is detached for, and the test +# harness fails a test whose client writes to stderr. +${CLICKHOUSE_CLIENT} --send_logs_level=fatal --query "ATTACH TABLE t_patch_empty_index" + +# Before the fix the patch part loaded as an index with no source parts: still active, applying to +# nothing, and eligible for cleanup. +echo -n 'active patch parts after reload: ' +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM system.parts + WHERE database = currentDatabase() AND table = 't_patch_empty_index' + AND active AND startsWith(name, 'patch-') +" + +echo -n 'detached patch parts: ' +${CLICKHOUSE_CLIENT} --query " + SELECT count(), any(reason) FROM system.detached_parts + WHERE database = currentDatabase() AND table = 't_patch_empty_index' + AND startsWith(name, 'broken') +" + +# The update is not applied any more - but it is reported and its rows are in \`detached/\`, instead +# of being deleted by the cleanup of a patch that looks materialized everywhere. +echo -n 'sum after reload: ' +${CLICKHOUSE_CLIENT} --query "SELECT sum(v) FROM t_patch_empty_index" + +# An uncorrupted patch part still loads and still applies after a reload. +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS t_patch_good_index SYNC; + CREATE TABLE t_patch_good_index (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; + + INSERT INTO t_patch_good_index SELECT number, number FROM numbers(1000); + SET enable_lightweight_update = 1; + UPDATE t_patch_good_index SET v = v + 1000 WHERE id < 500; +" + +${CLICKHOUSE_CLIENT} --query "DETACH TABLE t_patch_good_index SYNC" +${CLICKHOUSE_CLIENT} --query "ATTACH TABLE t_patch_good_index" + +echo -n 'reloaded sum: ' +${CLICKHOUSE_CLIENT} --query "SELECT sum(v) FROM t_patch_good_index" + +echo -n 'reloaded detached parts: ' +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM system.detached_parts + WHERE database = currentDatabase() AND table = 't_patch_good_index' +" + +${CLICKHOUSE_CLIENT} --query " + DROP TABLE t_patch_empty_index SYNC; + DROP TABLE t_patch_good_index SYNC; +" From 0f439b33d1ce51d8ec483c3ca1f82db29e8fe413 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 18:56:59 +0000 Subject: [PATCH 077/296] Backport #120488 to 26.8: Fix `INSERT ... SELECT FROM url()`/`s3()` with parallel replicas inserting the data once per replica --- src/Interpreters/InterpreterInsertQuery.cpp | 29 ++++-- src/Storages/IStorageCluster.h | 9 +- src/TableFunctions/TableFunctionURL.cpp | 14 ++- .../configs/config.d/parallel_replicas.xml | 23 +++++ tests/integration/test_s3_style_link/test.py | 53 +++++++++++ ...uster_function_parallel_replicas.reference | 8 ++ ...auto_cluster_function_parallel_replicas.sh | 91 +++++++++++++++++++ 7 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml create mode 100644 tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference create mode 100755 tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index afbd4a86e5521..9e05cac8341c6 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -1107,12 +1107,30 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica /// query will be executed on all nodes of the cluster auto src_cluster = src_storage_cluster->getCluster(local_context); - /// Actually the query doesn't change, we just serialize it to string. Strip the initiator-only - /// settings from the forwarded query text (both `changes` and `default_settings`, across the INSERT - /// and its source SELECT) so those names — including the new HTTP table-as-file settings — do not reach - /// the shards and trip `UNKNOWN_SETTING` on a rolling upgrade; the per-shard context is stripped below. + src_storage_cluster->updateExternalDynamicMetadataIfExists(local_context); + + const auto src_metadata_snapshot = src_storage_cluster->getInMemoryMetadataPtr(local_context, false); + const auto src_snapshot = src_storage_cluster->getStorageSnapshot(src_metadata_snapshot, local_context); + + /// Strip the initiator-only settings from the forwarded query text (both `changes` and `default_settings`, + /// across the INSERT and its source SELECT) so those names — including the new HTTP table-as-file settings — + /// do not reach the shards and trip `UNKNOWN_SETTING` on a rolling upgrade; the per-shard context is + /// stripped below. auto query_to_send = query.clone(); ClusterProxy::stripInitiatorOnlySettingsFromQuery(query_to_send); + + /// The source storage may have been created by `parallel_replicas_for_cluster_engines` from a plain table + /// function (`url`, `s3`, ...), while the query text still names that plain function. A node that runs + /// the forwarded query as a secondary query does not convert it again: it creates a plain storage that + /// expands the globs and reads every file on its own instead of taking its share of the read tasks from + /// the initiator, so N nodes insert the data N times. Rewrite the source the same way `IStorageCluster::read` + /// does for a `SELECT`: the function becomes its `*Cluster` variant with the cluster name argument, and the + /// structure and format arguments are added so that the nodes do not infer the schema again. + { + auto & select_to_send = query_to_send->as().select->as(); + src_storage_cluster->updateQueryToSendIfNeeded(select_to_send.list_of_selects->children.at(0), src_snapshot, local_context); + } + String query_str; { WriteBufferFromOwnString buf; @@ -1134,8 +1152,6 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica query_context->setSettings(stripped_settings); } - src_storage_cluster->updateExternalDynamicMetadataIfExists(local_context); - std::optional filter_dag; const ActionsDAG::Node * predicate = nullptr; if (select_query) @@ -1171,7 +1187,6 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica } } } - const auto src_metadata_snapshot = src_storage_cluster->getInMemoryMetadataPtr(local_context, false); auto extension = src_storage_cluster->getTaskIteratorExtension( predicate, filter_dag ? &*filter_dag : nullptr, local_context, src_cluster, src_metadata_snapshot); diff --git a/src/Storages/IStorageCluster.h b/src/Storages/IStorageCluster.h index 43b2d690955d1..6ecfdd22dc193 100644 --- a/src/Storages/IStorageCluster.h +++ b/src/Storages/IStorageCluster.h @@ -54,9 +54,16 @@ class IStorageCluster : public IStorage const String & getClusterName() const { return cluster_name; } + /// Prepare the `SELECT ... FROM f(...)` query (`f` is a table function) for the other nodes of the cluster: add the + /// structure and format arguments so that the nodes do not infer the schema again, and turn a plain table + /// function (`url`, `s3`, ...) that `parallel_replicas_for_cluster_engines` converted into this cluster + /// storage into its `*Cluster` variant with the cluster name argument, so that the nodes take their read + /// tasks from the initiator instead of reading every file on their own. Called by `read` and by the + /// distributed `INSERT ... SELECT` in `InterpreterInsertQuery`, which forwards the query the same way. + virtual void updateQueryToSendIfNeeded(ASTPtr & /*query*/, const StorageSnapshotPtr & /*storage_snapshot*/, const ContextPtr & /*context*/) {} + protected: virtual void updateBeforeRead(const ContextPtr &) {} - virtual void updateQueryToSendIfNeeded(ASTPtr & /*query*/, const StorageSnapshotPtr & /*storage_snapshot*/, const ContextPtr & /*context*/) {} virtual void updateConfigurationIfNeeded(ContextPtr /* context */) {} diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index b58a0b842505a..1e95ac17dee1c 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -267,15 +267,25 @@ StoragePtr TableFunctionURL::executeImpl( /// reports the delegate's engine name and access URI, so the outer check (or the caller that /// explicitly disabled it and took over) has already covered exactly the delegate's source. if (delegate) + { + /// The query text still names `url`, while the delegate is a different backend. If the delegate + /// created its `*Cluster` storage for `parallel_replicas_for_cluster_engines`, the forwarded query + /// would be rewritten from the surface AST name into `urlCluster(...)` - a function that rejects + /// every non-HTTP scheme - and with the argument grammar of the delegate rather than of `url`. + /// Scheme dispatch is therefore resolved on this node: the delegate builds its plain storage. + ContextMutablePtr delegate_context = Context::createCopy(context); + delegate_context->setSetting("parallel_replicas_for_cluster_engines", false); + return delegate->execute( ast_function, - context, + delegate_context, table_name, std::move(cached_columns), /*use_global_context=*/false, is_insert_query, /*check_create_temporary_table=*/false, /*check_source_access=*/false); + } /// Stored columns accompany a table definition rather than an ad-hoc query, so creation and /// replay must resolve to the same storage. @@ -554,6 +564,8 @@ SELECT * FROM url('s3://clickhouse-public-datasets/hits_compatible/hits.csv'); Scheme dispatch is not yet wired through [`urlCluster`](/reference/functions/table-functions/urlCluster): a non-`http(s)` scheme passed to `urlCluster` is rejected with an error. Use the corresponding cluster function (`s3Cluster`, `azureBlobStorageCluster`, `hdfsCluster`, …) for those backends instead. +For the same reason, a dispatched `url` call is read on the node that received the query: the [parallel_replicas_for_cluster_engines](/reference/settings/session-settings/parallel-replicas#parallel_replicas_for_cluster_engines) fan-out is not applied to it. Use the corresponding cluster function directly when you want the read distributed across replicas. + ## Globs in URL {#globs-in-url} Patterns in `{ }` are used to generate a set of shards or to specify failover addresses. Supported pattern types and examples see in the description of the [remote](/reference/functions/table-functions/remote#globs-in-addresses) function. diff --git a/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml b/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml new file mode 100644 index 0000000000000..f8e015f0b761f --- /dev/null +++ b/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml @@ -0,0 +1,23 @@ + + + + + + + http://minio1:9001/root/ + minio + ClickHouse_Minio_P@ssw0rd + + + + + + + node9000 + node9000 + node9000 + + + + diff --git a/tests/integration/test_s3_style_link/test.py b/tests/integration/test_s3_style_link/test.py index da7e64676d173..353c3a0764bf0 100644 --- a/tests/integration/test_s3_style_link/test.py +++ b/tests/integration/test_s3_style_link/test.py @@ -10,6 +10,7 @@ "node", main_configs=[ "configs/config.d/minio.xml", + "configs/config.d/parallel_replicas.xml", ], user_configs=[ "configs/users.d/users.xml", @@ -139,3 +140,55 @@ def test_s3_question_mark_wildcards(started_cluster): assert result_s3_scheme == result_http_scheme assert result_s3_scheme.startswith('20\t') assert "['a1','a2']" in result_s3_scheme or "['a2','a1']" in result_s3_scheme + + +def test_url_s3_scheme_with_parallel_replicas(started_cluster): + """ + `url('s3://...')` is delegated to the `s3` backend, but the query text still names `url`. + The cluster fan-out of `parallel_replicas_for_cluster_engines` rewrites the forwarded query + from that surface name, so it used to send `urlCluster('s3://...')` - a shape `urlCluster` + rejects - both for a plain `SELECT` and for the distributed `INSERT ... SELECT`. + """ + node.query( + f""" + INSERT INTO FUNCTION s3 + ( + 'minio://data/parallel_replicas_url.csv', 'minio', '{minio_secret_key}', + 'CSV', 'a UInt32' + ) SETTINGS s3_truncate_on_insert=1 + SELECT number FROM numbers(10); + """ + ) + + parallel_replicas_settings = """ + SETTINGS cluster_for_parallel_replicas = 'parallel_replicas', + enable_parallel_replicas = 1, + max_parallel_replicas = 3, + parallel_replicas_for_cluster_engines = 1 + """ + + assert ( + node.query( + f""" + SELECT count() FROM url('s3://data/parallel_replicas_url.csv', 'CSV', 'a UInt32') + {parallel_replicas_settings} + """ + ) + == "10\n" + ) + + node.query("DROP TABLE IF EXISTS url_s3_parallel_replicas SYNC") + node.query( + "CREATE TABLE url_s3_parallel_replicas (a UInt32) ENGINE = MergeTree ORDER BY a" + ) + node.query( + f""" + INSERT INTO url_s3_parallel_replicas + SELECT * FROM url('s3://data/parallel_replicas_url.csv', 'CSV', 'a UInt32') + {parallel_replicas_settings}, parallel_distributed_insert_select = 2 + """ + ) + + # The rows must be inserted exactly once, not once per replica. + assert node.query("SELECT count() FROM url_s3_parallel_replicas") == "10\n" + node.query("DROP TABLE url_s3_parallel_replicas SYNC") diff --git a/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference new file mode 100644 index 0000000000000..4cd3790c6b883 --- /dev/null +++ b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference @@ -0,0 +1,8 @@ +--- url --- +60 30 +--- s3 --- +60 30 +--- forwarded queries of url --- +3 3 60 +--- forwarded queries of s3 --- +3 3 60 diff --git a/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh new file mode 100755 index 0000000000000..c956ed1d537a3 --- /dev/null +++ b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, zookeeper +# Tag no-fasttest: needs MinIO and a Replicated table + +# https://github.com/ClickHouse/ClickHouse/issues/120485 +# +# `INSERT INTO SELECT * FROM url(...)` (or `s3(...)`) with parallel replicas enabled: +# `parallel_replicas_for_cluster_engines` converts the plain table function into a cluster storage on the +# initiator, so the INSERT takes the distributed `parallel_distributed_insert_select` path and forwards the +# query to every replica of the cluster. The forwarded query text still named the plain function, and a replica +# running it as a secondary query created a plain storage that expanded the globs and read every file on its +# own instead of taking its share of the read tasks from the initiator, so N replicas inserted the data N times. +# The forwarded query must name the `*Cluster` variant, the same way the SELECT path does. +# +# The three "replicas" of `test_cluster_one_shard_three_replicas_localhost` are all this server, so the +# secondary queries are visible in the local query log. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -eu + +S3_DIR="http://localhost:11111/test/${CLICKHOUSE_DATABASE}/05218" + +# Three files of different sizes: 10 + 20 + 30 = 60 rows, so a duplicated file changes the count. +for i in 1 2 3 +do + $CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/part_${i}.tsv', 'TSV', 'x UInt32') SELECT number FROM numbers(${i} * 10)" +done + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS dst_05218 SYNC; + CREATE TABLE dst_05218 (x UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/dst_05218', 'r1') ORDER BY x; +" + +SETTINGS="enable_parallel_replicas = 1, automatic_parallel_replicas_mode = 0, max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_cluster_engines = 1, parallel_distributed_insert_select = 2, log_queries = 1" + +# The query ids and the query log lookup below must isolate this run: a re-run on the same server +# must not pick up the secondary queries of a previous run. +QUERY_ID_SUFFIX="${CLICKHOUSE_DATABASE}_$(date +%s%N)_${RANDOM}" +QUERY_ID_URL="05218_url_${QUERY_ID_SUFFIX}" +QUERY_ID_S3="05218_s3_${QUERY_ID_SUFFIX}" + +echo "--- url ---" +$CLICKHOUSE_CLIENT --query_id "${QUERY_ID_URL}" -q " + INSERT INTO dst_05218 SELECT * FROM url('${S3_DIR}/part_{1..3}.tsv', 'TSV', 'x UInt32') SETTINGS ${SETTINGS}" +$CLICKHOUSE_CLIENT -q "SELECT count(), uniqExact(x) FROM dst_05218" + +echo "--- s3 ---" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE dst_05218" +$CLICKHOUSE_CLIENT --query_id "${QUERY_ID_S3}" -q " + INSERT INTO dst_05218 SELECT * FROM s3('${S3_DIR}/part_{1..3}.tsv', 'TSV', 'x UInt32') SETTINGS ${SETTINGS}" +$CLICKHOUSE_CLIENT -q "SELECT count(), uniqExact(x) FROM dst_05218" + +# The INSERT must really have been distributed: every replica ran the forwarded INSERT, the forwarded +# query names the `*Cluster` function, and the replicas together read every file exactly once. +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +for pair in "${QUERY_ID_URL} urlCluster" "${QUERY_ID_S3} s3Cluster" +do + query_id="${pair%% *}" + cluster_function="${pair##* }" + echo "--- forwarded queries of ${cluster_function%Cluster} ---" + $CLICKHOUSE_CLIENT -q " + WITH initial AS + ( + SELECT query_id + FROM system.query_log + WHERE current_database = currentDatabase() + AND query_id = '${query_id}' + AND is_initial_query = 1 + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - INTERVAL 10 MINUTE + ) + SELECT + count() AS replicas, + countIf(query ILIKE '%${cluster_function}(%') AS cluster_function_queries, + sum(read_rows) AS rows_read_by_replicas + FROM system.query_log + WHERE initial_query_id IN (SELECT query_id FROM initial) + AND is_initial_query = 0 + AND query_kind = 'Insert' + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - INTERVAL 10 MINUTE" +done + +$CLICKHOUSE_CLIENT -q "DROP TABLE dst_05218 SYNC" From 25b0c2abd71160422501bbdc832073a98ad6ef2f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 19:21:29 +0000 Subject: [PATCH 078/296] Backport #121037 to 26.8: Fix segmentation fault in `RefreshTask::startup()` racing a parallel shutdown --- src/Storages/MaterializedView/RefreshSet.cpp | 20 ++-- src/Storages/MaterializedView/RefreshSet.h | 2 + src/Storages/MaterializedView/RefreshTask.cpp | 28 ++++- ...st_refresh_task_startup_after_shutdown.cpp | 105 ++++++++++++++++++ 4 files changed, 137 insertions(+), 18 deletions(-) create mode 100644 src/Storages/MaterializedView/tests/gtest_refresh_task_startup_after_shutdown.cpp diff --git a/src/Storages/MaterializedView/RefreshSet.cpp b/src/Storages/MaterializedView/RefreshSet.cpp index c65838c16a730..9346f0c8bd353 100644 --- a/src/Storages/MaterializedView/RefreshSet.cpp +++ b/src/Storages/MaterializedView/RefreshSet.cpp @@ -85,18 +85,14 @@ RefreshSet::RefreshSet() = default; void RefreshSet::emplace(StorageID id, std::optional inner_table_id, const std::vector & dependencies, RefreshTaskPtr task) { - { - std::lock_guard guard(mutex); - const auto iter = addTaskLocked(id, task); - RefreshTaskList::iterator inner_table_iter; - if (inner_table_id) - inner_table_iter = addInnerTableLocked(*inner_table_id, task); - addDependenciesLocked(task, dependencies); - - task->setRefreshSetHandleUnlock(Handle(this, id, inner_table_id, iter, inner_table_iter, dependencies)); - } - - notifyDependents(id); + std::lock_guard guard(mutex); + const auto iter = addTaskLocked(id, task); + RefreshTaskList::iterator inner_table_iter; + if (inner_table_id) + inner_table_iter = addInnerTableLocked(*inner_table_id, task); + addDependenciesLocked(task, dependencies); + + task->setRefreshSetHandleUnlock(Handle(this, id, inner_table_id, iter, inner_table_iter, dependencies)); } RefreshTaskList::iterator RefreshSet::addTaskLocked(StorageID id, RefreshTaskPtr task) diff --git a/src/Storages/MaterializedView/RefreshSet.h b/src/Storages/MaterializedView/RefreshSet.h index a0ac155138822..7fa91d2ed2442 100644 --- a/src/Storages/MaterializedView/RefreshSet.h +++ b/src/Storages/MaterializedView/RefreshSet.h @@ -52,6 +52,8 @@ class RefreshSet RefreshSet(); + /// Caller should then also call notifyDependents, because dependent views need to know when + /// their dependencies appear/disappear. void emplace(StorageID id, std::optional inner_table_id, const std::vector & dependencies, RefreshTaskPtr task); /// Finds active refreshable view(s) by database and table name. diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index faed43facb7cf..2a3d98b5af9f7 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -315,13 +315,29 @@ bool RefreshTask::canCreateOrDropOtherTables() const void RefreshTask::startup() { - if (start_paused || view->getContext()->getSettingsRef()[Setting::stop_refreshable_materialized_views_on_startup]) - scheduling.stop_requested = true; - auto inner_table_id = refresh_append ? std::nullopt : std::make_optional(view->getTargetTableId()); - view->getContext()->getRefreshSet().emplace(view->getStorageID(), inner_table_id, initial_dependencies, shared_from_this()); + ContextMutablePtr context; + StorageID view_id = StorageID::createEmpty(); + { + std::lock_guard guard(mutex); - std::lock_guard guard(mutex); - scheduleRefresh(guard); + /// shutdown() is allowed to run before or during startup() (see its declaration) and nulls `view`. + if (!view) + return; + + if (start_paused || view->getContext()->getSettingsRef()[Setting::stop_refreshable_materialized_views_on_startup]) + scheduling.stop_requested = true; + context = view->getContext(); + view_id = view->getStorageID(); + auto inner_table_id = refresh_append ? std::nullopt : std::make_optional(view->getTargetTableId()); + + /// `set_handle` is not thread safe and shutdown() resets it under `mutex`. + context->getRefreshSet().emplace(view_id, inner_table_id, initial_dependencies, shared_from_this()); + + scheduleRefresh(guard); + } + + /// Outside `mutex`: notifying a dependent view locks that view's own task mutex. + context->getRefreshSet().notifyDependents(view_id); } void RefreshTask::finalizeRestoreFromBackup() diff --git a/src/Storages/MaterializedView/tests/gtest_refresh_task_startup_after_shutdown.cpp b/src/Storages/MaterializedView/tests/gtest_refresh_task_startup_after_shutdown.cpp new file mode 100644 index 0000000000000..01991dac8ab38 --- /dev/null +++ b/src/Storages/MaterializedView/tests/gtest_refresh_task_startup_after_shutdown.cpp @@ -0,0 +1,105 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// `DatabaseCatalog` is process-wide and shared by every suite in `unit_tests_dbms`, so the +/// database name has to be unique across the whole binary, not just readable here. +constexpr auto database_name = "refresh_task_startup_after_shutdown_db"; + +struct State +{ + State(const State &) = delete; + + ContextMutablePtr context; + + static const State & instance() + { + static State state; + return state; + } + +private: + State() : context(Context::createCopy(getContext().context)) + { + tryRegisterFunctions(); + tryRegisterAggregateFunctions(); + + DatabasePtr database = std::make_shared(database_name, context); + const ColumnsDescription columns{{"x", std::make_shared()}}; + for (const auto * table_name : {"src", "target"}) + database->attachTable( + context, + table_name, + std::make_shared( + StorageID(database_name, table_name), columns, ConstraintsDescription{}, String{}, MemorySettings{}), + {}); + + DatabaseCatalog::instance().attachDatabase(database_name, database); + context->setCurrentDatabase(database_name); + } +}; + +/// An explicit `TO` target and `LoadingStrictnessLevel::ATTACH` keep this to a plain constructor +/// call: neither creates an inner table, and a non-Replicated database leaves the refresh +/// uncoordinated, so no Keeper is needed. +std::shared_ptr attachRefreshableView(const ContextMutablePtr & context, const String & view_name) +{ + const String db = database_name; + const String query = "ATTACH MATERIALIZED VIEW " + db + "." + view_name + " REFRESH EVERY 1 YEAR TO " + db + + ".target AS SELECT x FROM " + db + ".src"; + + ParserCreateQuery parser; + ASTPtr ast = parseQuery(parser, query, 100000, 1000, 1000000); + + return std::make_shared( + StorageID(database_name, view_name), + context, + ast->as(), + ColumnsDescription{{"x", std::make_shared()}}, + LoadingStrictnessLevel::ATTACH, + /*comment=*/String{}, + /*is_restore_from_backup=*/false); +} + +} + +/// shutdown() is documented to be callable before or during startup(), and it nulls the `view` +/// back-pointer that startup() reads, so this order is sanctioned rather than a misuse. +/// +/// The two calls are made directly because no query sequence produces that order: ATTACH TABLE +/// builds a fresh RefreshTask, and a database sweep joins the outstanding startup jobs before it +/// begins (DatabaseOnDisk::shutdown() calls stopLoading()). +TEST(RefreshTaskStartupAfterShutdown, StartupAfterShutdownDoesNotDereferenceNullView) +{ + const auto & state = State::instance(); + + auto view = attachRefreshableView(state.context, "mv"); + /// Without a refresh task there is no `view` pointer to null and the test below is vacuous. + ASSERT_TRUE(view->isRefreshable()); + const StorageID view_id = view->getStorageID(); + + view->flushAndPrepareForShutdown(); + view->startup(); + + /// startup() must also decline to register the task: `RefreshSet` membership is what + /// `system.view_refreshes` reports and what schedules refreshes, and this view is shut down. + EXPECT_TRUE(state.context->getRefreshSet().findTasks(view_id).empty()); +} From 443fd99757c07200b73bc295dcd5499570d3a4f4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 23 Sep 2026 20:51:23 +0000 Subject: [PATCH 079/296] Backport #121693 to 26.8: Guard the JSON AST deserialization recursions with checkStackSize --- src/Parsers/ASTFromJSON.cpp | 35 +++++++++++++++---- src/Parsers/ASTJSONReadHelpers.cpp | 3 ++ .../0_stateless/04056_ast_json_limits.sql | 14 ++++++++ ...n_depth_guard_max_ast_depth_zero.reference | 2 +- ...ast_json_depth_guard_max_ast_depth_zero.sh | 6 ++-- 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/Parsers/ASTFromJSON.cpp b/src/Parsers/ASTFromJSON.cpp index 40ceb1cf95836..bb4d0b6dd8f18 100644 --- a/src/Parsers/ASTFromJSON.cpp +++ b/src/Parsers/ASTFromJSON.cpp @@ -78,12 +78,14 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -309,14 +311,27 @@ size_t computeJSONNestingDepth(const String & json) return max_depth; } +/// `Poco::JSON::Parser` descends recursively over the whole document before any AST node is built. +class StackCheckingParseHandler : public Poco::JSON::ParseHandler +{ +public: + void startObject() override + { + checkStackSize(); + Poco::JSON::ParseHandler::startObject(); + } + + void startArray() override + { + checkStackSize(); + Poco::JSON::ParseHandler::startArray(); + } +}; + } ASTPtr IAST::createFromJSON(const String & json) { - /// `Poco::JSON::Parser::setDepth` does not actually bound recursion in our Poco fork - /// (`ParserImpl` stores `_depth` but `handle`/`handleObject`/`handleArray` never read it), - /// so a hostile deeply-nested payload would recurse through the parser and overflow the - /// stack before any AST-level depth check runs. Enforce a raw-text bracket budget first. /// The budget is a safe multiple of the effective depth limit (the JSON encoding adds bracket /// levels per AST/`Field` level), not the limit itself, so a valid serialized AST is never /// rejected here — the constructed AST depth is still bounded by the counter check below. @@ -328,14 +343,19 @@ ASTPtr IAST::createFromJSON(const String & json) "JSON nesting depth exceeds the limit derived from max_ast_depth ({}) during JSON AST deserialization", json_nesting_budget); Poco::JSON::Parser parser; - /// Also request the parser-level bound (kept for forward compatibility if the fork starts - /// honouring it); the pre-scan above is the actual enforcement. + parser.setHandler(new StackCheckingParseHandler); + /// Poco's own default bound is 1000 levels (JSON_DEFAULT_DEPTH). parser.setDepth(json_nesting_budget); Poco::Dynamic::Var result; try { result = parser.parse(json); } + /// `DB::Exception` derives from `Poco::Exception`, so this clause must precede the one below. + catch (const Exception &) + { + throw; + } catch (const Poco::Exception & e) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to parse JSON for AST deserialization: {}", e.displayText()); @@ -365,6 +385,9 @@ ASTPtr IAST::createFromJSON(const Poco::JSON::Object & json) throw Exception(ErrorCodes::TOO_DEEP_AST, "JSON AST deserialization exceeded maximum depth limit ({})", max_depth); + /// The limit above counts nodes, which is not a stack budget at any value. + checkStackSize(); + /// Check element count limit. if (json_deser_max_elements && json_deser_current_elements >= json_deser_max_elements) throw Exception(ErrorCodes::TOO_BIG_AST, diff --git a/src/Parsers/ASTJSONReadHelpers.cpp b/src/Parsers/ASTJSONReadHelpers.cpp index db816fc4d5f42..deed0dc9fddf0 100644 --- a/src/Parsers/ASTJSONReadHelpers.cpp +++ b/src/Parsers/ASTJSONReadHelpers.cpp @@ -128,6 +128,9 @@ Field JSONObjectReader::readFieldFromObjectImpl(const Poco::JSON::Object & obj, "Structured Field value exceeds maximum AST depth limit ({}) during JSON AST deserialization", max_depth); + /// The limit above counts `Field` levels, which is not a stack budget at any value. + checkStackSize(); + /// Count every `Field` value (scalar or structured) against the element-count budget too, so a /// wide literal payload (e.g. one huge `Array`) cannot bypass `max_ast_elements` while adding no /// AST nodes. diff --git a/tests/queries/0_stateless/04056_ast_json_limits.sql b/tests/queries/0_stateless/04056_ast_json_limits.sql index e448e8ff8bb7c..6aac3b4200510 100644 --- a/tests/queries/0_stateless/04056_ast_json_limits.sql +++ b/tests/queries/0_stateless/04056_ast_json_limits.sql @@ -91,3 +91,17 @@ SELECT formatQueryFromJSON('{"type":"RefreshStrategy","schedule_kind":"EVERY"}') -- `clickhouse_json` client/server entry points (so a huge shallow document is rejected before Poco -- materializes it). Here `parseQueryToJSON('SELECT 1')` is well over 10 bytes. SELECT formatQueryFromJSON(parseQueryToJSON('SELECT 1')) SETTINGS max_query_size = 10; -- { serverError SYNTAX_ERROR } + +-- Reading an AST back from JSON descends over the document, so raising `max_ast_depth` must not turn +-- a deep payload into a stack overflow: each shape below must fail with a controlled error, never +-- crash. The depths sit far past every bound so that holds in any build; which of the descents +-- reports it first follows the build's stack budget, so no arm asserts a particular one. +-- A chain of nested AST nodes. +SELECT formatQueryFromJSON(concat(repeat('{"type":"ExpressionList","children":[', 20000), '{"type":"Literal","value":{"field_type":"UInt64","value":1}}', repeat(']}', 20000))) SETTINGS max_ast_depth = 100000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- A nested `field_type` value inside a single literal, which adds no AST nodes of its own. +SELECT formatQueryFromJSON(concat('{"type":"Literal","value":', repeat('{"field_type":"Array","value":[', 25000), '{"field_type":"UInt64","value":1}', repeat(']}', 25000), '}')) SETTINGS max_ast_depth = 200000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- The JSON text is parsed before any AST node exists, so a deep member that the reader never descends +-- into still has to be bounded while the text itself is being read. +SELECT formatQueryFromJSON(concat('{"type":"Literal","value":{"field_type":"UInt64","value":1},"junk":', repeat('[', 200000), repeat(']', 200000), '}')) SETTINGS max_ast_depth = 100000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } diff --git a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference index 0fb27808c3ca1..73b717c7d2550 100644 --- a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference +++ b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference @@ -1,6 +1,6 @@ TOO_DEEP_AST TOO_DEEP_AST -Structured Field value exceeds maximum AST depth limit +Structured Field value rejected Field dump payload exceeds maximum AST depth limit 1 TOO_DEEP_AST diff --git a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh index 455c50c4a2d5f..67c08b16c6ba5 100755 --- a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh +++ b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh @@ -27,14 +27,16 @@ ${CLICKHOUSE_CLIENT} --max_ast_depth 0 --param_json "$BRACKET_BOMB" \ --query "SELECT formatQueryFromJSON({json:String})" 2>&1 | grep -om1 'TOO_DEEP_AST' # 3. A deeply nested structured `Field` value adds no AST nodes and stays under the bracket -# budget, so only the `Field` depth bound rejects it. +# budget, so it is rejected by its own depth bound or by the recursion's stack check. Which of +# the two reports it depends on the build's stack budget, so assert only that it is rejected. ${CLICKHOUSE_CLIENT} --max_ast_depth 0 --query " SELECT formatQueryFromJSON(concat( '{\"type\":\"Literal\",\"value\":', repeat('{\"field_type\":\"Array\",\"value\":[', 2000), '{\"field_type\":\"UInt64\",\"value\":1}', repeat(']}', 2000), '}'))" 2>&1 | - grep -om1 'Structured Field value exceeds maximum AST depth limit' + grep -qE 'Structured Field value exceeds maximum AST depth limit|TOO_DEEP_RECURSION' && + echo 'Structured Field value rejected' # 4. A deeply nested `Field` dump hides its nesting inside a JSON string, so the bracket # pre-scan does not see it either; `Field::restoreFromDump` must not recurse unbounded. From 4ccf093ab6b9a3e5cfc18a63d2c8e72c9379c29d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 24 Sep 2026 02:53:45 +0000 Subject: [PATCH 080/296] Backport #108626 to 26.8: Avoid vertical TTL delete for TTLDrop merges --- src/Storages/MergeTree/MergeTask.cpp | 4 + .../04411_ttl_drop_not_vertical.reference | 4 + .../04411_ttl_drop_not_vertical.sh | 154 ++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference create mode 100755 tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 7138a2685bb35..fc33ad1cc3a83 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -3530,6 +3530,10 @@ MergeAlgorithm MergeTask::ExecuteAndFinalizeHorizontalPart::chooseMergeAlgorithm return MergeAlgorithm::Horizontal; if (ctx->need_remove_expired_values) { + /// `TTLTransform` stops reading after the first block when the rows TTL has expired for the whole part, + /// while a vertical merge reads every key row to write `rows_sources`. + if (global_ctx->future_part->merge_type == MergeType::TTLDrop && global_ctx->metadata_snapshot->hasRowsTTL()) + return MergeAlgorithm::Horizontal; if (!canVerticalTTLDelete(*global_ctx)) return MergeAlgorithm::Horizontal; } diff --git a/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference new file mode 100644 index 0000000000000..876d94d1857a2 --- /dev/null +++ b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference @@ -0,0 +1,4 @@ +TTLDropMerge Horizontal 0 +0 +TTLDropMerge Horizontal 0 +0 diff --git a/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh new file mode 100755 index 0000000000000..d34ea2f74dc6a --- /dev/null +++ b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +TABLE=t_ttl_drop_not_vertical + +function wait_for_ttl_drop_merge() +{ + for _ in $(seq 1 600); do + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS part_log" + local merge_count + merge_count=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge'") + + if [ "${merge_count}" -gt "0" ]; then + return + fi + sleep 0.1 + done + + echo "Timed out waiting for a TTLDropMerge of ${TABLE}" >&2 + exit 1 +} + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} + ( + id UInt64, + d DateTime DEFAULT '2000-01-01 00:00:00', + c1 UInt64, + c2 UInt64, + c3 UInt64, + c4 UInt64 + ) + ENGINE = MergeTree + ORDER BY id + TTL d + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_block_number_column = 0, + enable_block_offset_column = 0, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_optimize_ttl_delete = 1, + ratio_of_defaults_for_sparse_serialization = 1.0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number + 1000, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM START TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM START MERGES ${TABLE}" + +wait_for_ttl_drop_merge + +${CLICKHOUSE_CLIENT} -q " + SELECT + merge_reason, + merge_algorithm, + rows + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge' + ORDER BY event_time_microseconds DESC + LIMIT 1" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" + +TABLE=t_ttl_drop_not_vertical_mixed_ttl + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} + ( + id UInt64, + d DateTime DEFAULT '2000-01-01 00:00:00', + c1 UInt64, + c2 UInt64, + c3 UInt64, + c4 UInt64 + ) + ENGINE = MergeTree + ORDER BY id + TTL d + INTERVAL 1 DAY, d + INTERVAL 2 DAY RECOMPRESS CODEC(ZSTD) + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_block_number_column = 0, + enable_block_offset_column = 0, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_optimize_ttl_delete = 1, + ratio_of_defaults_for_sparse_serialization = 1.0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number + 1000, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM START TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM START MERGES ${TABLE}" + +wait_for_ttl_drop_merge + +${CLICKHOUSE_CLIENT} -q " + SELECT + merge_reason, + merge_algorithm, + rows + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge' + ORDER BY event_time_microseconds DESC + LIMIT 1" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" From cff4c0021e2136cca1c664f66572abb55182441f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 24 Sep 2026 08:35:40 +0000 Subject: [PATCH 081/296] Backport #118917 to 26.8: Bump `croaring` from v4.5.1 to v5.1.1 --- contrib/croaring | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/croaring b/contrib/croaring index 025ae3f7add16..342463b31b909 160000 --- a/contrib/croaring +++ b/contrib/croaring @@ -1 +1 @@ -Subproject commit 025ae3f7add169bc820dcfd46fa9304f382ec40a +Subproject commit 342463b31b909737bd6295c69b2f8e0ed9497424 From a7aa824c2e54f6c2733231a67402e7b0df63715e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 24 Sep 2026 09:04:58 +0000 Subject: [PATCH 082/296] Include checkStackSize for the checkStackSize() call added by the backport 26.8 lacked the earlier fix that added this include (and its own checkStackSize() call) to ASTJSONReadHelpers.cpp, so the cherry-picked checkStackSize() call in readFieldFromObjectImpl failed to compile. Co-Authored-By: Claude Sonnet 5 --- src/Parsers/ASTJSONReadHelpers.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Parsers/ASTJSONReadHelpers.cpp b/src/Parsers/ASTJSONReadHelpers.cpp index deed0dc9fddf0..ae4111a79448a 100644 --- a/src/Parsers/ASTJSONReadHelpers.cpp +++ b/src/Parsers/ASTJSONReadHelpers.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include From 0653b9ea269820006c1aee957f5b31847b89fdfd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 24 Sep 2026 09:58:14 +0000 Subject: [PATCH 083/296] Backport #121864 to 26.8: Small performance optimizations --- .../Iceberg/ManifestFileIterator.cpp | 15 ++++++----- .../DataLakes/Iceberg/ManifestFileIterator.h | 1 + .../DataLakes/Iceberg/SchemaProcessor.cpp | 27 +++++++++++++------ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index 9c0d2db914de9..9fed28a864966 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -464,13 +464,14 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) /// those manifests still carry the original snapshot_id. The manifest file's own Avro header /// records the correct schema_id for the data files it describes, so falling back to /// manifest_schema_id is safe and correct in this case. - LOG_DEBUG( - getLogger("ManifestFileIterator"), - "Manifest file '{}' has entry with snapshot_id '{}' whose snapshot metadata is not present " - "(snapshot may have been expired by the catalog). Falling back to manifest schema_id {}.", - path_to_manifest_file, - resolved_snapshot_id, - manifest_schema_id); + if (!logged_missing_snapshot_metadata.exchange(true, std::memory_order_relaxed)) + LOG_TEST( + getLogger("ManifestFileIterator"), + "Manifest file '{}' has entry with snapshot_id '{}' whose snapshot metadata is not present " + "(snapshot may have been expired by the catalog). Falling back to manifest schema_id {}.", + path_to_manifest_file, + resolved_snapshot_id, + manifest_schema_id); } const auto resolved_schema_id = schema_id_opt.has_value() ? *schema_id_opt : manifest_schema_id; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h index 40bb9b4c45c19..d6e90f78590e2 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h @@ -160,6 +160,7 @@ class ManifestFileIterator : public boost::noncopyable std::atomic current_row_index{0}; std::atomic fully_initialized{false}; std::atomic active_fetchers{0}; + std::atomic logged_missing_snapshot_metadata{false}; /// Cached results accumulated during iteration mutable SharedMutex files_mutex; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index c154a025ab3a8..30c5177ce13c7 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -395,8 +395,6 @@ std::string IcebergSchemaProcessor::default_link{}; void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr) { - std::lock_guard lock(mutex); - Int32 schema_id = schema_ptr->getValue(f_schema_id); /// Databricks UniForm writes a degenerate placeholder schema (e.g. {"schema-id":0,"fields":[]}) @@ -404,16 +402,29 @@ void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schem if (!schema_ptr->isArray(f_fields) || schema_ptr->getArray(f_fields)->size() == 0) return; + std::unordered_map type_mapping; + if (allow_geo_parser) + { + type_mapping[f_geography] = f_binary; + type_mapping[f_geometry] = f_binary; + } + + Poco::JSON::Object::Ptr registered_schema; + { + SharedLockGuard lock(mutex); + auto it = iceberg_table_schemas_by_ids.find(schema_id); + if (it != iceberg_table_schemas_by_ids.end()) + registered_schema = it->second; + } + if (registered_schema && schemasAreIdentical(*registered_schema, *schema_ptr, type_mapping)) + return; + + std::lock_guard lock(mutex); + current_schema_id = schema_id; if (iceberg_table_schemas_by_ids.contains(schema_id)) { chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); - std::unordered_map type_mapping; - if (allow_geo_parser) - { - type_mapping[f_geography] = f_binary; - type_mapping[f_geometry] = f_binary; - } /// A schema-id is immutable per the Iceberg spec: re-binding it to different fields is malformed metadata. if (!schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr, type_mapping)) throw Exception( From 87208465d5fd9cbb0a0c9e1306f0ed5bd0d2cb58 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 24 Sep 2026 10:29:53 +0000 Subject: [PATCH 084/296] Backport #121856 to 26.8: Do not exhaust the stack when serializing a deeply nested query to JSON --- src/Parsers/ASTJSONHelpers.cpp | 2 ++ src/Parsers/ASTJSONHelpers.h | 3 +++ ...uery_to_json_deep_query_stack_guard.reference | 2 ++ ...arse_query_to_json_deep_query_stack_guard.sql | 16 ++++++++++++++++ 4 files changed, 23 insertions(+) create mode 100644 tests/queries/0_stateless/05254_parse_query_to_json_deep_query_stack_guard.reference create mode 100644 tests/queries/0_stateless/05254_parse_query_to_json_deep_query_stack_guard.sql diff --git a/src/Parsers/ASTJSONHelpers.cpp b/src/Parsers/ASTJSONHelpers.cpp index 1d8e3e28cdadf..63cad18b82c88 100644 --- a/src/Parsers/ASTJSONHelpers.cpp +++ b/src/Parsers/ASTJSONHelpers.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -22,6 +23,7 @@ void JSONObjectWriter::writeAlias(const ASTWithAlias & node) static void writeFieldJSON(WriteBuffer & out, const FormatSettings & fs, const Field & field) { + checkStackSize(); out << "{\"field_type\":"; writeJSONString(field.getTypeName(), out, fs); diff --git a/src/Parsers/ASTJSONHelpers.h b/src/Parsers/ASTJSONHelpers.h index 68afb51b2aaed..7cc6c5d7d4672 100644 --- a/src/Parsers/ASTJSONHelpers.h +++ b/src/Parsers/ASTJSONHelpers.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -30,6 +31,8 @@ class JSONObjectWriter JSONObjectWriter(WriteBuffer & out_, const char * type_name) : out(out_) { + /// One of these is constructed per node by the AST-to-JSON walk, which recurses as deep as the query nests. + checkStackSize(); out << "{\"type\":"; writeJSONString(std::string_view(type_name), out, fs); } diff --git a/tests/queries/0_stateless/05254_parse_query_to_json_deep_query_stack_guard.reference b/tests/queries/0_stateless/05254_parse_query_to_json_deep_query_stack_guard.reference new file mode 100644 index 0000000000000..6ed281c757a96 --- /dev/null +++ b/tests/queries/0_stateless/05254_parse_query_to_json_deep_query_stack_guard.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/05254_parse_query_to_json_deep_query_stack_guard.sql b/tests/queries/0_stateless/05254_parse_query_to_json_deep_query_stack_guard.sql new file mode 100644 index 0000000000000..6501d6bb073a1 --- /dev/null +++ b/tests/queries/0_stateless/05254_parse_query_to_json_deep_query_stack_guard.sql @@ -0,0 +1,16 @@ +-- `parseQueryToJSON` walks the parsed query recursively, so a query nested deeper than that walk can +-- follow must be reported rather than crash the server. `max_parser_depth` is raised far above the +-- nesting so that the parser's own limit does not answer first, and the refused depths are far past +-- what any build's stack can hold, so they do not depend on the build. The accepted depths are kept +-- small instead: under TSan only 5% of a thread's stack may be used, which this walk exhausts at +-- under a hundred levels of nesting. + +SELECT length(parseQueryToJSON(concat('SELECT ', repeat('abs(', 10), '1', repeat(')', 10)))) > 0; +SELECT length(parseQueryToJSON(concat('SELECT ', repeat('abs(', 40000), '1', repeat(')', 40000)))) +SETTINGS max_parser_depth = 200000, max_ast_depth = 200000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- A single nested value nests the same way while adding one node to the query, so the depth of the +-- query does not bound it. +SELECT length(parseQueryToJSON(concat('SELECT ', repeat('[', 10), '1', repeat(']', 10)))) > 0; +SELECT length(parseQueryToJSON(concat('SELECT ', repeat('[', 500000), '1', repeat(']', 500000)))) +SETTINGS max_parser_depth = 2000000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } From cac86dcf32e00c796da943572917f1c3c284e7f3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 24 Sep 2026 14:22:37 +0000 Subject: [PATCH 085/296] Backport #117421 to 26.8: Fix SQL injection through identifiers when inserting into a SQLite table --- src/Common/quoteString.cpp | 11 +++ src/Common/quoteString.h | 2 + src/IO/WriteHelpers.h | 13 +++ src/Storages/StorageSQLite.cpp | 4 +- ...qlite_insert_identifier_escaping.reference | 20 +++++ ...05055_sqlite_insert_identifier_escaping.sh | 81 +++++++++++++++++++ 6 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference create mode 100755 tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh diff --git a/src/Common/quoteString.cpp b/src/Common/quoteString.cpp index 0559e85d8a502..cc3dbaac0e621 100644 --- a/src/Common/quoteString.cpp +++ b/src/Common/quoteString.cpp @@ -39,6 +39,17 @@ String doubleQuoteString(std::string_view x) } +String doubleQuoteStringSQLite(std::string_view x) +{ + String res(2 + x.size(), '\0'); + { + WriteBufferFromString wb(res); + writeDoubleQuotedStringSQLite(x, wb); + } + return res; +} + + String backQuote(std::string_view x) { String res(2 + x.size(), '\0'); diff --git a/src/Common/quoteString.h b/src/Common/quoteString.h index b6ee14c0b8783..15733c7c2d6a6 100644 --- a/src/Common/quoteString.h +++ b/src/Common/quoteString.h @@ -30,6 +30,8 @@ namespace DB /// Double quote the string. String doubleQuoteString(std::string_view x); +String doubleQuoteStringSQLite(std::string_view x); + /// Quote the identifier with backquotes. String backQuote(std::string_view x); diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 185c22b38caf8..1afee1a496e0d 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -642,6 +642,19 @@ inline void writeQuotedStringSQLite(std::string_view ref, WriteBuffer & buf) writeChar('\'', buf); } +/// SQLite identifiers: a " is escaped by doubling it; every other byte, backslash included, is literal. +inline void writeDoubleQuotedStringSQLite(std::string_view ref, WriteBuffer & buf) +{ + writeChar('"', buf); + for (char c : ref) + { + if (c == '"') + writeChar('"', buf); + writeChar(c, buf); + } + writeChar('"', buf); +} + inline void writeDoubleQuotedString(const String & s, WriteBuffer & buf) { writeAnyQuotedString<'"'>(s, buf); diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp index aef8d35af6039..a5964106745e8 100644 --- a/src/Storages/StorageSQLite.cpp +++ b/src/Storages/StorageSQLite.cpp @@ -184,14 +184,14 @@ class SQLiteSink final : public SinkToStorage WriteBufferFromOwnString sqlbuf; sqlbuf << "INSERT INTO "; - sqlbuf << doubleQuoteString(remote_table_name); + sqlbuf << doubleQuoteStringSQLite(remote_table_name); sqlbuf << " ("; for (auto it = block.begin(); it != block.end(); ++it) { if (it != block.begin()) sqlbuf << ", "; - sqlbuf << quoteString(it->name); + sqlbuf << doubleQuoteStringSQLite(it->name); } sqlbuf << ") VALUES "; diff --git a/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference new file mode 100644 index 0000000000000..d50220788b0b6 --- /dev/null +++ b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference @@ -0,0 +1,20 @@ +--- A1 control: the boundary the injection bypasses is enforced when the path is named directly +PATH_ACCESS_DENIED +--- A2 injection through the remote table name +no such table +objects created in the anchor: none +canary read from outside user_files: none +objects written outside user_files: none +--- A3 injection through a column name +no column named +objects created in the anchor: none +canary read from outside user_files: none +objects written outside user_files: none +--- A4 control: a table and column name that legitimately contain a double quote round-trip +NO_ERROR +11 +--- A5 control: a table name that legitimately contains a backslash round-trips +NO_ERROR +22 +--- A6 a NUL in the remote table name fails loudly, it is never silently truncated +unrecognized token diff --git a/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh new file mode 100755 index 0000000000000..9ec9abb0b9bae --- /dev/null +++ b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Fast tests don't build external libraries (SQLite) + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +ANCHOR="${USER_FILES_PATH}/05055_anchor_${CLICKHOUSE_DATABASE}.db" +# Outside the user_files confinement. +OUTSIDE="${CLICKHOUSE_TMP}/05055_outside_${CLICKHOUSE_DATABASE}.db" + +cleanup() +{ + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_name_inj" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_col_inj" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_quote_id" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_backslash_id" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_nul_name" + rm -f "${ANCHOR}" "${OUTSIDE}" +} +trap cleanup EXIT +cleanup + +# Odd names: the old backslash escaping produced them, so the injected script's first statement succeeds and the rest runs. +sqlite3 "${ANCHOR}" 'CREATE TABLE "target\"(id INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE col_anchor("c\" INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE "ta""ble" ("c""1" INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE "a\b" (x INTEGER)' +chmod ugo+rw "${ANCHOR}" + +# A database outside user_files, holding a canary the user must never be able to reach. +sqlite3 "${OUTSIDE}" "CREATE TABLE secrets(s TEXT); INSERT INTO secrets VALUES ('CANARY')" +chmod ugo+rw "${OUTSIDE}" + +# One stable token per failure, so the reference embeds no path and no server version. +classify() +{ + local out + out=$(${CLICKHOUSE_CLIENT} --query="$1" 2>&1 \ + | grep -oF -e 'no such table' -e 'no column named' -e 'unrecognized token' \ + -e 'syntax error' -e 'PATH_ACCESS_DENIED' \ + | sed -n 1p) + echo "${out:-NO_ERROR}" +} + +# Read out of band: the ClickHouse read path cannot address these names, so an in-band readback would be a false negative. +injected_state() +{ + sqlite3 "${ANCHOR}" "SELECT 'objects created in the anchor: ' || coalesce((SELECT group_concat(name) FROM (SELECT name FROM sqlite_master WHERE name LIKE '%\_marker' ESCAPE '\' ORDER BY name)), 'none')" + sqlite3 "${ANCHOR}" "SELECT 'canary read from outside user_files: ' || coalesce((SELECT group_concat(s) FROM stolen_marker), 'none')" 2>/dev/null \ + || echo 'canary read from outside user_files: none' + sqlite3 "${OUTSIDE}" "SELECT 'objects written outside user_files: ' || coalesce((SELECT group_concat(name) FROM (SELECT name FROM sqlite_master WHERE name LIKE '%\_marker' ESCAPE '\' ORDER BY name)), 'none')" +} + +echo '--- A1 control: the boundary the injection bypasses is enforced when the path is named directly' +classify "CREATE TABLE t_ctl (s String) ENGINE = SQLite('${OUTSIDE}', 'secrets')" + +echo '--- A2 injection through the remote table name' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_name_inj (id UInt32) ENGINE = SQLite('${ANCHOR}', \$\$target\" (id) VALUES (999); ATTACH DATABASE '${OUTSIDE}' AS v; CREATE TABLE stolen_marker AS SELECT s FROM v.secrets; CREATE TABLE v.written_marker(z); DETACH v; --\$\$)" +classify "INSERT INTO t_name_inj VALUES (0)" +injected_state + +echo '--- A3 injection through a column name' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_col_inj (\`c') VALUES (1); CREATE TABLE col_marker(z); --\` UInt32) ENGINE = SQLite('${ANCHOR}', 'col_anchor')" +classify "INSERT INTO t_col_inj VALUES (0)" +injected_state + +echo '--- A4 control: a table and column name that legitimately contain a double quote round-trip' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_quote_id (\`c\"1\` UInt32) ENGINE = SQLite('${ANCHOR}', 'ta\"ble')" +classify "INSERT INTO t_quote_id VALUES (11)" +sqlite3 "${ANCHOR}" 'SELECT "c""1" FROM "ta""ble"' + +echo '--- A5 control: a table name that legitimately contains a backslash round-trips' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_backslash_id (x UInt32) ENGINE = SQLite('${ANCHOR}', 'a\\\\b')" +classify "INSERT INTO t_backslash_id VALUES (22)" +sqlite3 "${ANCHOR}" 'SELECT x FROM "a\b"' + +echo '--- A6 a NUL in the remote table name fails loudly, it is never silently truncated' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_nul_name (id UInt32) ENGINE = SQLite('${ANCHOR}', 'a\0b')" +classify "INSERT INTO t_nul_name VALUES (0)" From 6b68f29f4e46aaf493f3f3ee9a148d956972c449 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 24 Sep 2026 16:28:59 +0000 Subject: [PATCH 086/296] Backport #118360 to 26.8: Require the local privilege for KILL and SYSTEM statements over ON CLUSTER --- .../InterpreterKillQueryQuery.cpp | 36 ++++-- src/Interpreters/InterpreterSystemQuery.cpp | 13 +- ...d_system_on_cluster_access_types.reference | 12 ++ ...kill_and_system_on_cluster_access_types.sh | 111 ++++++++++++++++++ 4 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference create mode 100755 tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 177ca1a051c0c..32b725ce3474c 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -474,17 +474,31 @@ AccessRightsElements InterpreterKillQueryQuery::getRequiredAccessForDDLOnCluster { const auto & query = query_ptr->as(); AccessRightsElements required_access; - if (query.type == ASTKillQueryQuery::Type::Query) - required_access.emplace_back(AccessType::KILL_QUERY); - else if (query.type == ASTKillQueryQuery::Type::Mutation) - required_access.emplace_back( - AccessType::ALTER_UPDATE - | AccessType::ALTER_DELETE - | AccessType::ALTER_MATERIALIZE_INDEX - | AccessType::ALTER_MATERIALIZE_COLUMN - | AccessType::ALTER_MATERIALIZE_TTL - | AccessType::ALTER_REWRITE_PARTS - ); + /// This switch has no `default:`, so a new Type has to be mapped here to compile. + switch (query.type) + { + case ASTKillQueryQuery::Type::Query: + required_access.emplace_back(AccessType::KILL_QUERY); + break; + case ASTKillQueryQuery::Type::Mutation: + required_access.emplace_back( + AccessType::ALTER_UPDATE + | AccessType::ALTER_DELETE + | AccessType::ALTER_MATERIALIZE_INDEX + | AccessType::ALTER_MATERIALIZE_COLUMN + | AccessType::ALTER_MATERIALIZE_TTL + | AccessType::ALTER_REWRITE_PARTS + ); + break; + case ASTKillQueryQuery::Type::PartMoveToShard: + required_access.emplace_back(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "part_moves_between_shards"); + required_access.emplace_back(AccessType::ALTER_MOVE_PARTITION | AccessType::MOVE_PARTITION_BETWEEN_SHARDS); + break; + case ASTKillQueryQuery::Type::Transaction: + required_access.emplace_back(AccessType::KILL_TRANSACTION); + required_access.emplace_back(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "transactions"); + break; + } return required_access; } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9efabe1c7877f..9a9ff9e467858 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -3229,11 +3229,22 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() } case Type::STOP_THREAD_FUZZER: case Type::START_THREAD_FUZZER: + { + required_access.emplace_back(AccessType::SYSTEM_THREAD_FUZZER); + break; + } + case Type::RESET_COVERAGE: + { + required_access.emplace_back(AccessType::SYSTEM); + break; + } + /// The parser cases of the failpoint statements and of SYSTEM SET COVERAGE TEST never read an + /// ON CLUSTER clause, so those cluster spellings do not parse and reach no host. UNKNOWN and + /// END are not statements. case Type::ENABLE_FAILPOINT: case Type::WAIT_FAILPOINT: case Type::NOTIFY_FAILPOINT: case Type::DISABLE_FAILPOINT: - case Type::RESET_COVERAGE: case Type::SET_COVERAGE_TEST: case Type::UNKNOWN: case Type::END: break; diff --git a/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference new file mode 100644 index 0000000000000..e9efacb33c69d --- /dev/null +++ b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference @@ -0,0 +1,12 @@ +KILL TRANSACTION -> KILL TRANSACTION ON *.* +KILL PART_MOVE_TO_SHARD -> SELECT ON system.part_moves_between_shards +SYSTEM STOP THREAD FUZZER -> SYSTEM THREAD FUZZER ON *.* +SYSTEM START THREAD FUZZER -> SYSTEM THREAD FUZZER ON *.* +SYSTEM RESET COVERAGE -> SYSTEM ON *.* +KILL TRANSACTION without system.transactions -> SELECT ON system.transactions +KILL PART_MOVE_TO_SHARD without move privileges -> ALTER MOVE PARTITION, MOVE PARTITION BETWEEN SHARDS ON *.* +no CLUSTER grant -> CLUSTER ON *.* +KILL TRANSACTION local -> allowed +KILL TRANSACTION on cluster -> allowed +KILL PART_MOVE_TO_SHARD local -> allowed +KILL PART_MOVE_TO_SHARD on cluster -> allowed diff --git a/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh new file mode 100755 index 0000000000000..9f246c25da6d2 --- /dev/null +++ b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Holds CLUSTER and nothing else. +cluster_user="cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Holds every statement privilege but not CLUSTER. +no_cluster_user="no_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Hold the complete privilege set of one statement each. +kill_txn_user="kill_txn_$CLICKHOUSE_TEST_UNIQUE_NAME" +move_user="move_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Hold all but one privilege of one statement each. +partial_txn_user="partial_txn_$CLICKHOUSE_TEST_UNIQUE_NAME" +partial_move_user="partial_move_$CLICKHOUSE_TEST_UNIQUE_NAME" + +function cleanup() +{ + $CLICKHOUSE_CLIENT -mq " + DROP USER IF EXISTS $cluster_user; + DROP USER IF EXISTS $no_cluster_user; + DROP USER IF EXISTS $kill_txn_user; + DROP USER IF EXISTS $move_user; + DROP USER IF EXISTS $partial_txn_user; + DROP USER IF EXISTS $partial_move_user; + " +} +cleanup +trap cleanup EXIT + +# CLUSTER is a global privilege, so it cannot share a GRANT with a table-scoped one. +$CLICKHOUSE_CLIENT -mq " + CREATE USER $cluster_user, $no_cluster_user, $kill_txn_user, $move_user, + $partial_txn_user, $partial_move_user IDENTIFIED WITH no_password; + + GRANT CLUSTER ON *.* TO $cluster_user; + + GRANT KILL TRANSACTION, SYSTEM THREAD FUZZER, SYSTEM ON *.* TO $no_cluster_user; + + GRANT CLUSTER, KILL TRANSACTION ON *.* TO $kill_txn_user; + GRANT SELECT ON system.transactions TO $kill_txn_user; + + GRANT CLUSTER, ALTER MOVE PARTITION, MOVE PARTITION BETWEEN SHARDS ON *.* TO $move_user; + GRANT SELECT ON system.part_moves_between_shards TO $move_user; + + GRANT CLUSTER, KILL TRANSACTION ON *.* TO $partial_txn_user; + + GRANT CLUSTER ON *.* TO $partial_move_user; + GRANT SELECT ON system.part_moves_between_shards TO $partial_move_user; +" + +cluster="test_shard_localhost" +# Match no live transaction and no live part move, so every allowed arm below is a no-op. The +# transaction predicate deliberately does not compare `tid`: its tuple shape is not the same in +# every build, so a literal tuple fails type analysis before the access check this test is about. +txn_predicate="tid_hash = 0" +task_uuid="'00000000-0000-0000-0000-000000000000'" + +# Report the privilege the server asked for and let the reference hold the expected mapping. The +# privilege name is what discriminates: an ACCESS_DENIED-only assertion would also pass when the +# CLUSTER check, which runs before the required-access check, is what fired. +required_privilege() { + $CLICKHOUSE_CLIENT --distributed_ddl_output_mode none --user "$1" --query "$2" 2>&1 | + sed -n "/necessary to have the grant/{s/.*grant \(.*\)\. (ACCESS_DENIED).*/\1/p;q;}" +} + +# Each statement must name over ON CLUSTER the same privilege its local spelling names. +while IFS= read -r statement; do + echo "${statement%% ON CLUSTER*} -> $(required_privilege "$cluster_user" "$statement")" +done < $(required_privilege "$partial_txn_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate")" +echo "KILL PART_MOVE_TO_SHARD without move privileges -> $(required_privilege "$partial_move_user" "KILL PART_MOVE_TO_SHARD ON CLUSTER $cluster WHERE task_uuid = $task_uuid")" + +# In-range control: holding the statement privileges without CLUSTER is refused by the earlier check, +# so a mapping that refuses everything would not produce the five lines above. +echo "no CLUSTER grant -> $(required_privilege "$no_cluster_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate")" + +# The two statements whose privileges can be granted in full are allowed in both spellings, which +# proves the new elements are a gate rather than an unconditional refusal. The local half is asserted +# too: a user who is cluster-allowed while locally refused is the bypass this test exists to catch. +# The thread fuzzer and coverage statements get no allowed arm, because executing them would change +# server-global state that concurrent tests read and neither takes an argument that lets the host +# reject them harmlessly. +allowed() { + local out + out=$($CLICKHOUSE_CLIENT --distributed_ddl_output_mode none --user "$1" --query "$2" 2>&1) + if [ -z "$out" ]; then + echo "$3 -> allowed" + else + echo "$3 -> FAIL: $out" + fi +} + +allowed "$kill_txn_user" "KILL TRANSACTION WHERE $txn_predicate" "KILL TRANSACTION local" +allowed "$kill_txn_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate" "KILL TRANSACTION on cluster" +allowed "$move_user" "KILL PART_MOVE_TO_SHARD WHERE task_uuid = $task_uuid" "KILL PART_MOVE_TO_SHARD local" +allowed "$move_user" "KILL PART_MOVE_TO_SHARD ON CLUSTER $cluster WHERE task_uuid = $task_uuid" "KILL PART_MOVE_TO_SHARD on cluster" From f6351a498d1084bdba618af41021d798f689bf22 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 23 Sep 2026 20:27:00 +0000 Subject: [PATCH 087/296] Fix a bug that MySQL/Postgres queries may run in an expired session --- src/Interpreters/Session.cpp | 9 +- src/Interpreters/Session.h | 2 +- .../__init__.py | 0 .../configs/protocols.xml | 4 + .../test.py | 112 ++++++++++++++++++ 5 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_auth_method_valid_until_stateful_protocols/__init__.py create mode 100644 tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml create mode 100644 tests/integration/test_auth_method_valid_until_stateful_protocols/test.py diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 6b6cc000b61fd..17521ee3d1bc6 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -413,7 +413,7 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So prepared_client_info->connection_address = Poco::Net::SocketAddress(connection_address ? *connection_address : address); } -void Session::checkIfUserIsStillValid() +void Session::checkIfUserIsStillValid() const { if (const auto valid_until = user_authenticated_with.getValidUntil()) { @@ -696,6 +696,13 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t if (!user_id && getClientInfo().interface != ClientInfo::Interface::TCP_INTERSERVER) throw Exception(ErrorCodes::LOGICAL_ERROR, "Query context must be created after authentication"); + /// The authentication method's `VALID UNTIL` must be enforced per query, not only at login: + /// stateful protocols (MySQL, PostgreSQL, native TCP, gRPC, Arrow Flight) authenticate once and + /// then create a query context per command, so an expired credential must stop working here. + /// Interserver connections replay an already-checked initiator identity, so they are exempt. + if (getClientInfo().interface != ClientInfo::Interface::TCP_INTERSERVER) + checkIfUserIsStillValid(); + /// We can create a query context either from a session context or from a global context. const bool from_session_context = static_cast(session_context) && !detached; diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 055c8b8bcbd99..4a8bf55154f32 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -62,7 +62,7 @@ class Session // Verifies whether the user's validity extends beyond the current time. // Throws an exception if the user's validity has expired. - void checkIfUserIsStillValid(); + void checkIfUserIsStillValid() const; /// Writes a row about login failure into session log (if enabled) void onAuthenticationFailure(const std::optional & user_name, const Poco::Net::SocketAddress & address_, const Exception & e); diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/__init__.py b/tests/integration/test_auth_method_valid_until_stateful_protocols/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml b/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml new file mode 100644 index 0000000000000..65eafc59c3f85 --- /dev/null +++ b/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml @@ -0,0 +1,4 @@ + + 9001 + 5433 + diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py b/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py new file mode 100644 index 0000000000000..1faf74afbbebd --- /dev/null +++ b/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py @@ -0,0 +1,112 @@ +import time + +import psycopg2 +import pymysql.connections +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# The authentication method's `VALID UNTIL` must be enforced per query, not only at login. +# Stateful protocols (MySQL, PostgreSQL) authenticate once at connection startup and then run +# every later command through `Session::makeQueryContext`, so without a per-query re-check a +# credential that expires after login would keep working for the lifetime of the connection. +# The check lives in `Session::makeQueryContextImpl`, so every protocol shares it. +node = cluster.add_instance("node", main_configs=["configs/protocols.xml"]) + +MYSQL_PORT = 9001 +POSTGRES_PORT = 5433 + +# Lifetime of the expiring credential, measured from user creation. The first query on each +# connection runs within a fraction of a second of creation (well inside the lifetime); the second +# query runs after sleeping past the expiry. +EXPIRING_LIFETIME_S = 6 + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def create_users(): + node.query("DROP USER IF EXISTS u_expiring, u_lasting") + expiry = node.query(f"SELECT toString(now() + INTERVAL {EXPIRING_LIFETIME_S} SECOND)").strip() + # Plaintext-stored passwords so both the MySQL and PostgreSQL frontends can verify them. + node.query(f"CREATE USER u_expiring IDENTIFIED WITH plaintext_password BY 'pw' VALID UNTIL '{expiry}'") + node.query("CREATE USER u_lasting IDENTIFIED WITH plaintext_password BY 'pw' VALID UNTIL '2999-01-01 00:00:00'") + node.query("GRANT SELECT ON system.one TO u_expiring, u_lasting") + return expiry + + +def sleep_past(expiry): + while int(node.query(f"SELECT now() > toDateTime('{expiry}')").strip()) != 1: + time.sleep(0.5) + # The check is `now > valid_until` with second precision, so cross the boundary decisively. + time.sleep(1.5) + + +def test_mysql_connection_stops_working_after_expiry(started_cluster): + expiry = create_users() + host = started_cluster.get_instance_ip("node") + + def connect(user): + return pymysql.connections.Connection(host=host, user=user, password="pw", database="default", port=MYSQL_PORT) + + expiring = connect("u_expiring") + lasting = connect("u_lasting") + + # Both credentials are valid at connection time and for the first query. + for client in (expiring, lasting): + cursor = client.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == ((1,),) + + sleep_past(expiry) + + # The same, still-open connection must stop working once the method has expired ... + with pytest.raises(pymysql.err.MySQLError, match="expired"): + expiring.cursor().execute("SELECT 1") + + # ... while a connection under a non-expired method keeps working. + cursor = lasting.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == ((1,),) + + lasting.close() + + +def test_postgresql_connection_stops_working_after_expiry(started_cluster): + expiry = create_users() + host = started_cluster.get_instance_ip("node") + + def connect(user): + conn = psycopg2.connect(host=host, user=user, password="pw", dbname="default", port=POSTGRES_PORT) + conn.autocommit = True + return conn + + expiring = connect("u_expiring") + lasting = connect("u_lasting") + + for client in (expiring, lasting): + cursor = client.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == [(1,)] + + sleep_past(expiry) + + # The PostgreSQL handler sends an ErrorResponse and then terminates the connection on an expired + # credential, and psycopg2 may surface either the message or the connection loss - both are + # fail-close; the point is that the query must not succeed. + with pytest.raises(psycopg2.Error, match="expired|closed the connection"): + expiring.cursor().execute("SELECT 1") + + cursor = lasting.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == [(1,)] + + lasting.close() From 8fa70cde6514bd319633ad16e3329a1f1233269e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 24 Sep 2026 20:12:00 +0000 Subject: [PATCH 088/296] Backport #114344 to 26.8: Validate a verbatim codec's declared size in the passthrough read shortcut --- src/Compression/CompressedReadBufferBase.cpp | 6 ++ ...mpressed_decompressed_size_bound.reference | 13 +++ ...4848_compressed_decompressed_size_bound.sh | 85 +++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference create mode 100755 tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index b32d51ef27e4a..d116c7baa1d04 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -310,6 +310,12 @@ void CompressedReadBufferBase::decompress(BufferBase::Buffer & to, size_t size_d "Can't decompress data: the compressed data size ({}, this should include header size) is less than the header size ({})", size_compressed_without_checksum, static_cast(header_size)); + if (size_compressed_without_checksum - header_size != size_decompressed) + throw Exception(external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CORRUPTED_DATA, + "Can't decompress data: the compressed data size without header ({}) does not match size_decompressed ({}) " + "for a codec that stores data uncompressed", + size_compressed_without_checksum - header_size, size_decompressed); + to = BufferBase::Buffer(compressed_buffer + header_size, compressed_buffer + size_compressed_without_checksum); } else diff --git a/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference new file mode 100644 index 0000000000000..74b069ccd930a --- /dev/null +++ b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference @@ -0,0 +1,13 @@ +-- a valid frame still executes (proves the arms below fail for the intended reason) +1 +-- a codec that stores data uncompressed must not lie about the uncompressed size +1 +-- and neither may the other verbatim codec +1 +-- nor may it understate the uncompressed size +1 +-- engines keep working on ordinary data +1000 100000 +1000 100000 +1 +1 diff --git a/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh new file mode 100755 index 0000000000000..a0248bf8eb067 --- /dev/null +++ b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -o pipefail + +# Wire layout of a compressed frame: +# [16B checksum][1B method][4B LE size_compressed][4B LE size_decompressed][payload] +# size_compressed counts the 9-byte header. The checksum is an unkeyed CityHash128 of everything +# after it, so a peer can compute it: these frames are accepted at default settings, and +# http_native_compression_disable_checksumming_on_decompress is deliberately NOT used. +# +# checksum-for-compressed-block prints CityHash128 of every single-bit mutation of its input, so +# feeding it the body with bit 0 flipped yields CityHash128(body) on the line labelled "0, 0". The +# wire order is low64 then high64, each little-endian, i.e. the reverse of the printed hex. +emit() { # $1 = frame body as hex, checksum excluded + local checksum + checksum=$(python3 -c " +import sys +b = bytearray.fromhex(sys.argv[1]); b[0] ^= 1 +sys.stdout.buffer.write(bytes(b)) +" "$1" | $CLICKHOUSE_BINARY checksum-for-compressed-block | awk -F'\t' '$2 == "0, 0" { print $1; exit }') + python3 -c " +import sys +sys.stdout.buffer.write(bytearray.fromhex(sys.argv[1])[::-1] + bytearray.fromhex(sys.argv[2])) +" "$checksum" "$1" +} + +# Method bytes are from CompressionInfo.h: NONE is 0x02 = 2 and Quantized is 0x9e = 158. +frame() { # $1 = method byte (decimal), $2 = size_decompressed, $3 = payload + emit "$(python3 -c " +import struct, sys +payload = sys.argv[3].encode() +sys.stdout.buffer.write(bytes([int(sys.argv[1])]) + struct.pack('&1 | grep -c '(8) does not match size_decompressed (999)' +# Quantized is the other codec reporting isNone(), so it takes the same shortcut. The read path +# builds it from the method byte alone, without the enable_quantized_codec setting its DDL requires. +echo '-- and neither may the other verbatim codec' +frame 158 999 'SELECT 1' | post 2>&1 | grep -c '(8) does not match size_decompressed (999)' + +# The comparison is for inequality, so arms that only ever declare more than the body pin one side of +# it: narrowed to "body shorter than declared", both frames above would still be refused. This one +# declares less instead, and without the check it is not refused at all, it executes its real body. +echo '-- nor may it understate the uncompressed size' +frame 2 3 'SELECT 1' | post 2>&1 | grep -c '(8) does not match size_decompressed (3)' + +# No-regression control: only a NONE-coded frame takes the shortcut, so the column, the marks and +# the primary key are all stored uncompressed here. StripeLog is absent because it compresses its +# whole stream with the default codec and ignores the column codec, so its reads never take it. +echo '-- engines keep working on ordinary data' +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS t_log_bound; + DROP TABLE IF EXISTS t_mt_bound; + + CREATE TABLE t_log_bound (s String CODEC(NONE)) ENGINE = Log; + CREATE TABLE t_mt_bound (k UInt64, s String CODEC(NONE), INDEX idx_s s TYPE minmax GRANULARITY 1) + ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8, compress_marks = 1, compress_primary_key = 1, + min_bytes_for_wide_part = 0, packed_skip_index_max_bytes = 0, + marks_compression_codec = 'NONE', primary_key_compression_codec = 'NONE'; + + INSERT INTO t_log_bound SELECT repeat('a', 100) FROM numbers(1000); + INSERT INTO t_mt_bound SELECT number, repeat('a', 100) FROM numbers(1000); + + SELECT count(), sum(length(s)) FROM t_log_bound; + SELECT count(), sum(length(s)) FROM t_mt_bound WHERE s LIKE '%a%'; + SELECT count() FROM t_mt_bound WHERE k = 42; + -- packed_skip_index_max_bytes = 0 keeps the index in its own file, which is the skip-index class + -- checkDataPart iterates over; a packed one carries no per-file checksum entry to visit. + CHECK TABLE t_mt_bound SETTINGS check_query_single_value_result = 1; + + DROP TABLE t_log_bound; + DROP TABLE t_mt_bound; +" From 0963256af9c256b82cd7e6083911c0b60b33ee82 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 24 Sep 2026 20:44:46 +0000 Subject: [PATCH 089/296] Backport #120282 to 26.8: Bound column count in inserted RowBinary header --- .../Formats/Impl/BinaryRowInputFormat.cpp | 11 +++++ ...owbinary_header_too_many_columns.reference | 3 ++ ...04603_rowbinary_header_too_many_columns.sh | 48 +++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference create mode 100755 tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh diff --git a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp index 3b10b98a27831..bc380233d0cc0 100644 --- a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace DB @@ -14,8 +15,12 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_SKIP_UNKNOWN_FIELD; + extern const int TOO_LARGE_ARRAY_SIZE; } +/// Bound number of columns in header so user cannot reserve() arbitrarily large amount of memory +static constexpr auto TOO_MANY_COLUMNS_MESSAGE = "Suspiciously many columns in RowBinary header: {}"; + template BinaryRowInputFormat::BinaryRowInputFormat(ReadBuffer & in_, SharedHeader header, IRowInputFormat::Params params_, bool with_names_, bool with_types_, const FormatSettings & format_settings_) : RowInputFormatWithNamesAndTypes>( @@ -88,6 +93,8 @@ template std::vector BinaryFormatReader::readNames() { readVarUInt(read_columns, *in); + if (read_columns > DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, TOO_MANY_COLUMNS_MESSAGE, read_columns); return readHeaderRow(); } @@ -150,6 +157,8 @@ template void BinaryFormatReader::skipNames() { readVarUInt(read_columns, *in); + if (read_columns > DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, TOO_MANY_COLUMNS_MESSAGE, read_columns); skipHeaderRow(); } @@ -160,6 +169,8 @@ void BinaryFormatReader::skipTypes() { /// It's possible only when with_names = false and with_types = true readVarUInt(read_columns, *in); + if (read_columns > DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, TOO_MANY_COLUMNS_MESSAGE, read_columns); } skipHeaderRow(); diff --git a/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference new file mode 100644 index 0000000000000..6cc8f3258bf9e --- /dev/null +++ b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference @@ -0,0 +1,3 @@ +1 +TOO_LARGE_ARRAY_SIZE +TOO_LARGE_ARRAY_SIZE diff --git a/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh new file mode 100755 index 0000000000000..627efc72b50a2 --- /dev/null +++ b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tests that a RowBinaryWithNames[AndTypes] header with a suspiciously large column +# count is rejected with TOO_LARGE_ARRAY_SIZE instead of amplifying into a huge +# allocation. See https://github.com/ClickHouse/clickhouse-private/issues/69219 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_rowbinary_header" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_rowbinary_header (x UInt64) ENGINE = Memory" + +# LEB128 encoder. +varint() { + python3 -c ' +import sys +n = int(sys.argv[1]) +out = bytearray() +while True: + b = n & 0x7F + n >>= 7 + if n: + b |= 0x80 + out.append(b) + if not n: + break +sys.stdout.buffer.write(bytes(out)) +' "$1" +} + +# Negative control: a well-formed one-column header (name "x", type "UInt64") plus one +# 8-byte UInt64 row inserts fine. +{ varint 1; printf '\x01x\x06UInt64'; printf '\x01\x00\x00\x00\x00\x00\x00\x00'; } \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNamesAndTypes" --data-binary @- +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_rowbinary_header" + +# Attack: a header claiming far more columns than the ceiling (1'000'000). +# The tiny body must be rejected up-front, not amplified. +varint 2000000 \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNamesAndTypes" --data-binary @- 2>&1 \ + | grep -o "TOO_LARGE_ARRAY_SIZE" | head -n1 + +# Same for the WithNames variant (the names header is a separate read path). +varint 2000000 \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNames" --data-binary @- 2>&1 \ + | grep -o "TOO_LARGE_ARRAY_SIZE" | head -n1 + +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_rowbinary_header" From a8a37bb99b61b2db79fb711bf3ebcefc7891ecc3 Mon Sep 17 00:00:00 2001 From: George Larionov Date: Thu, 24 Sep 2026 23:36:09 +0000 Subject: [PATCH 090/296] Fix settings history in backport of #121424 to 26.8 Drop the leaked `26.10` and `26.9` blocks; record the changes in the `26.8` block. `ai_function_max_api_calls_per_query` no longer changes in 26.8, so its entry is removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/Core/SettingsChangesHistory.cpp | 105 +----------------- ...functions_compatibility_defaults.reference | 2 - ...28_ai_functions_compatibility_defaults.sql | 15 +-- 3 files changed, 6 insertions(+), 116 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 97a310296201d..867808a27809c 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -41,108 +41,6 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. - addSettingsChanges(settings_changes_history, "26.10", - { - {"allow_executable_tables", true, true, "New setting to disable reading through the `executable` table function and from `Executable` and `ExecutablePool` tables."}, - {"ai_function_max_input_tokens_per_query", 1000000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, - {"ai_function_max_output_tokens_per_query", 500000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, - {"ai_function_max_api_calls_per_query", 1000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, - {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema and log a warning instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.10 restores the previous strict behavior."}, - {"legacy_join_size_limits_trigger_spilling", true, false, "`max_rows_in_join` / `max_bytes_in_join` are now hard caps for every hash join, including the ones that spill to disk, where they used to act as the spill trigger. Spilling is driven by `max_bytes_before_external_join` / `max_bytes_ratio_before_external_join` alone."}, - {"prefer_optimize_projection", false, false, "New setting: choose a usable projection regardless of its estimated cost, like `force_optimize_projection`, but without failing the query when no projection is used."}, - {"query_plan_optimize_join_order_conflict_detector", "", "", "New setting selecting the conflict detector that decides join reordering validity in the DPsub join order algorithm: `a` for the (correct but incomplete) CD-A, `c` for the (correct and complete) CD-C, empty for none."}, - {"use_text_index_postings_cache", false, true, "Enabled the text index posting lists cache globally. Previously each query used a small private cache, which caused posting lists and phrase search results to be recomputed within a single query on large tables."}, - {"output_format_arrow_unsupported_types", "binary", "binary", "New setting superseding `output_format_arrow_unsupported_types_as_binary`, adding a `text` mode. Its default matches the previous behavior, so `compatibility` must not change it."}, - }); - addSettingsChanges(settings_changes_history, "26.9", - { - {"max_bytes_before_external_distinct", 0, 0, "New setting to enable spilling of `DISTINCT` to disk when memory usage exceeds the given threshold in bytes. If 0, only `max_bytes_ratio_before_external_distinct` applies."}, - {"max_bytes_ratio_before_external_distinct", 0., 0.5, "New setting to enable spilling of `DISTINCT` to disk when memory usage exceeds the given ratio of available memory. If 0, only `max_bytes_before_external_distinct` applies."}, - {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7 or 26.8: the previous value is deliberately equal to the new one, because those versions already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, - {"allow_delta_lake_create_table", false, false, "New setting: allow creating a new DeltaLake table using delta-kernel-rs or registering an existing one into a catalog."}, - {"delta_lake_accurate_write_cast", false, true, "New setting: cast written values to the Delta write-schema type with an accurate cast that throws on a value that does not fit the target type instead of silently truncating; `compatibility` below 26.9 uses the plain, non-throwing cast."}, - {"allow_experimental_nullable_tuple_type", false, true, "`Nullable(Tuple)` is now GA"}, - {"enable_nullable_tuple_type", false, true, "`Nullable(Tuple)` is now GA"}, - {"allow_nullable_tuple_in_extracted_subcolumns", false, true, "`Nullable(Tuple)` is now GA: a `Tuple` subcolumn extracted from a `Tuple`, `Variant`, `Dynamic` or `JSON` column is `Nullable(Tuple)` and is NULL in the rows where the subcolumn is missing. The setting is read once at server startup, so `compatibility` restores the previous behavior only from the startup profile (for example, users.xml), not from a session-level `SET`."}, - {"workload_admission_timeout_ms", 0, 0, "New setting bounding how long a query waits to be admitted by workload scheduling (acquiring its query slot and memory reservation) before failing; 0 (default) preserves the previous unbounded wait."}, - {"s3_disable_checksum", false, false, "Obsolete setting: checksum calculation no longer re-reads the source"}, - {"session_query_ids_history_size", 0, 1000, "New setting limiting the size of the session-local query id history exposed through the new `system.session_query_ids` system table. The previous value `0` (recording disabled) reproduces the pre-26.9 behavior."}, - {"query_plan_optimize_join_order_conflict_detector", "", "", "New setting selecting the conflict detector that decides join reordering validity in the DPsub join order algorithm: `a` for the (correct but incomplete) CD-A, `c` for the (correct and complete) CD-C, empty for none."}, - {"reader_executor_window_size", 4194304, 8388608, "Raised the default read window of the experimental `ReaderExecutor` from 4 MiB to 8 MiB. Under memory pressure the window is reduced from this base, floored at 128 KiB."}, - {"webassembly_udf_input_split_memory_ratio", 0.0, 0.5, "New setting controlling the fraction of a WebAssembly UDF instance's linear memory that one call's serialized input may occupy, which also enables the dynamic splitting of that input by its serialized size; `compatibility` below 26.9 sets it to 0 and restores the previous behavior, where `webassembly_udf_max_input_block_size = 0` meant one call per pipeline block."}, - {"cascades_aggregation_pushdown", false, true, "New setting to consider pushing partial aggregation below a join (eager aggregation) in the Cascades optimizer."}, - {"optimize_read_in_reverse_order_final", false, true, "New setting to enable the read-in-order optimization when reading in reverse order of the sorting key with the `FINAL` modifier from `ReplacingMergeTree` tables."}, - {"load_marks_asynchronously", false, true, "Load marks of all streams in parallel by default. On remote disks, synchronous loading of marks of columns with many substreams (such as `JSON`) took one network round trip per stream."}, - {"ast_fuzzer_oracle", false, false, "New setting to enable correctness oracle checks in the server-side AST fuzzer."}, - {"create_token_default_ttl_seconds", 1800, 1800, "New setting giving a lifetime to a token created by `CREATE TOKEN` without an explicit `VALID UNTIL` or `VALID FOR` clause. The statement is new, so there is no earlier behavior to restore and the previous value is the default itself: a `compatibility` with an older version must not turn tokens into never-expiring ones."}, - {"enable_hash_join_row_store", false, true, "New setting to enable transforming the payload of a hash join into a row-major layout."}, - {"min_rows_ratio_for_hash_join_row_store", 5.0, 5.0, "New setting to control the minimum estimated ratio of join output rows to build-side rows to enable transforming hash join payload to row-major. 0 means the transformation is always allowed."}, - {"enable_sharding_aggregator", false, false, "Obsolete setting, the sharded aggregator has been removed in favor of the adaptive aggregator (`enable_adaptive_aggregator`)."}, - {"allow_preliminary_distinct_abandoning", false, true, "New setting that lets the preliminary `DISTINCT` give up deduplicating mostly-unique input, because the final `DISTINCT` deduplicates its output again."}, - {"query_plan_fuse_filter_into_array_join", false, true, "New optimization to fuse a filter on ARRAY JOINed columns into the ARRAY JOIN step, enabled by default."}, - {"iceberg_file_entries_queue_size", 100, 100, "New setting for the previously hardcoded capacity of the queue between the Iceberg data manifest decode tasks and the query."}, - {"iceberg_manifest_decode_concurrency", 2, 4, "New setting bounding how many Iceberg manifest files are decoded concurrently, for delete and data manifests alike. It replaces `iceberg_delete_manifest_decode_concurrency` (kept as an alias). `2` approximates the pre-26.9 data path, which decoded one manifest at a time with the next one's fetch already in flight; under `compatibility` at or below 26.8 the delete decode therefore also runs at 2 rather than its released default of 4, preserving the older data-path memory envelope at the cost of some delete-decode overlap."}, - {"query_plan_lower_array_join_function", false, false, "New optimization to lower an arrayJoin function into a real ARRAY JOIN step; disabled by default."}, - {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, - {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, - {"allow_experimental_analyzer", true, true, "The setting is obsolete: the analyzer is mandatory and the old query analysis is no longer supported. Disabling it is refused instead of being ignored, and `compatibility` with a version below 24.3 no longer reverts it."}, - {"allow_url_wildcard_from_index_pages", false, false, "Added an alias for setting `allow_experimental_url_wildcard_from_index_pages`."}, - {"allow_kafka_offsets_storage_in_keeper", false, false, "Added an alias for setting `allow_experimental_kafka_offsets_storage_in_keeper`."}, - {"allow_correlated_subqueries", true, true, "Added an alias for setting `allow_experimental_correlated_subqueries`."}, - {"allow_geo_types_in_iceberg", false, false, "Added an alias for setting `allow_experimental_geo_types_in_iceberg`."}, - {"enable_materialized_postgresql_table", false, false, "Added an alias for setting `allow_experimental_materialized_postgresql_table`."}, - {"enable_funnel_functions", false, false, "Added an alias for setting `allow_experimental_funnel_functions`."}, - {"enable_unique_key", false, false, "Added an alias for setting `allow_experimental_unique_key`."}, - {"allow_join_right_table_sorting", false, false, "Added an alias for setting `allow_experimental_join_right_table_sorting`."}, - {"enable_json_lazy_type_hints", false, false, "Added an alias for setting `allow_experimental_json_lazy_type_hints`."}, - {"enable_join_runtime_filters", true, true, "The JOIN runtime filters became a Production tier feature."}, - {"join_runtime_filter_exact_values_limit", 10000, 10000, "The JOIN runtime filters became a Production tier feature."}, - {"join_runtime_bloom_filter_bytes", 512_KiB, 512_KiB, "The JOIN runtime filters became a Production tier feature."}, - {"join_runtime_bloom_filter_hash_functions", 3, 3, "The JOIN runtime filters became a Production tier feature."}, - {"join_runtime_filter_pass_ratio_threshold_for_disabling", 0.7, 0.7, "The JOIN runtime filters became a Production tier feature."}, - {"join_runtime_filter_blocks_to_skip_before_reenabling", 30, 30, "The JOIN runtime filters became a Production tier feature."}, - {"join_runtime_bloom_filter_max_ratio_of_set_bits", 0.7, 0.7, "The JOIN runtime filters became a Production tier feature."}, - {"join_runtime_filter_min_probe_rows", 1000, 1000, "The JOIN runtime filters became a Production tier feature."}, - {"enable_join_runtime_filters_index_analysis", false, false, "The JOIN runtime filters became a Production tier feature."}, - {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, - {"query_plan_aggregation_bucket_top_k", false, true, "New setting to toggle the plan optimization that materializes only each two-level bucket's best n groups when a final aggregation feeds ORDER BY over its outputs with LIMIT n and the per-bucket selection is provably exact."}, - {"enable_trino_dialect", false, false, "New setting to enable the `trino` value of the `dialect` setting, which translates Trino SQL syntax and maps Trino function names to ClickHouse equivalents."}, - {"enable_join_key_only_hash_tables", false, true, "New setting to store the join keys alone, without a reference to a right row, in the hash tables of joins whose result can never contain a value taken from a right row (`LEFT ANTI`, and `LEFT SEMI` when no right column is selected)."}, - {"distributed_plan_read_in_order", false, false, "New setting to allow the read-in-order optimization for `ORDER BY` in a distributed query plan, so a sorted read of the table's sorting key can skip the sort and stop early. Off by default: only shapes where no exchange survives between the read and the sort are safe today."}, - {"distributed_cache_client_id", "", "", "New setting (CI tests only) to override the distributed cache client id per query."}, - {"query_plan_propagate_predicate_across_join", false, true, "New setting that lifts filter conjuncts across equi-join keys so primary-key pruning fires on both sides."}, - {"read_through_distributed_cache", false, false, "The setting moved to the server configuration and is ignored as a profile setting: reading from the distributed cache is now switched by the server setting `enable_read_through_distributed_cache`, which is applied without a restart (so that merges, mutations and `Buffer` flushes follow it too, instead of being pinned to the value the server started with). Use `force_read_through_distributed_cache` to deviate from the server setting per query."}, - {"write_through_distributed_cache", false, false, "The setting moved to the server configuration and is ignored as a profile setting: writing to the distributed cache is now switched by the server setting `enable_write_through_distributed_cache`, which is applied without a restart (so that merges, mutations and `Buffer` flushes follow it too, instead of being pinned to the value the server started with). Use `force_write_through_distributed_cache` to deviate from the server setting per query."}, - {"force_read_through_distributed_cache", "auto", "auto", "New setting overriding the server setting `enable_read_through_distributed_cache` for a single query."}, - {"force_write_through_distributed_cache", "auto", "auto", "New setting overriding the server setting `enable_write_through_distributed_cache` for a single query."}, - {"distributed_cache_min_inflight_bytes_to_discard_connection_on_seek", 0, 4 * 1024 * 1024, "New setting to drop and reopen a distributed cache connection on a seek when too many in-flight bytes would otherwise be discarded. Defaults to 4 MiB; 0 restores the previous behavior (always reuse the connection via the read range id)."}, - {"distributed_plan_workers_provisioning_timeout_ms", 10000, 10000, "New setting bounding how long a query waits for leased stateless workers to become reachable before execution."}, - {"distributed_plan_fallback_to_local_execution", false, true, "New setting to fall back to local execution when a plan cannot be distributed (only takes effect under `make_distributed_plan`)."}, - {"query_plan_optimize_lazy_materialization_for_object_storage", false, true, "New setting to use lazy materialization for `ORDER BY ... LIMIT n` queries reading Parquet files from object storage (including Iceberg tables)."}, - {"iceberg_compaction_commit_batch_size", 100, 100, "New setting"}, - {"iceberg_compaction_max_rows_in_data_file", std::numeric_limits::max(), std::numeric_limits::max(), "New setting for the max rows of an iceberg data file produced by compaction, separate from the insert-time limit."}, - {"iceberg_compaction_max_bytes_in_data_file", std::numeric_limits::max(), std::numeric_limits::max(), "New setting for the max bytes of an iceberg data file produced by compaction, separate from the insert-time limit."}, - {"enable_json_lazy_type_hints", false, false, "Lazy JSON type hints are now Beta. An alias for setting 'allow_experimental_json_lazy_type_hints'."}, - {"s3_upload_checksum_algorithm", "", "", "New setting to choose the checksum algorithm for S3 uploads."}, - {"network_compression_method", "LZ4", "ZSTD", "Switched the default compression method for client/server and server/server communication from `LZ4` to `ZSTD` to reduce network traffic."}, - {"network_zstd_compression_level", 1, 3, "Aligned the default network `ZSTD` compression level with the new default on-disk `ZSTD(3)` compression."}, - {"use_statistics_for_min_max_aggregation", false, true, "New setting to answer `min`, `max` and `count` aggregations without `GROUP BY` and filters from per-part column statistics for parts that have them materialized, reading only the remaining parts. previous_value=false so `compatibility` with versions before 26.9 keeps the optimization disabled and restores the pre-existing plan."}, - {"parallel_replicas_allow_merge_tables", false, false, "New setting to allow reading from a `Merge` table with plan-based parallel replicas, by expanding the `Merge` read into a union of the reads from the underlying `MergeTree` tables. It only has an effect together with `parallel_replicas_plan_based`."}, - {"iceberg_compaction_max_bytes_in_data_file", std::numeric_limits::max(), 512 * 1024 * 1024, "New setting for the max bytes of an iceberg data file produced by compaction, separate from the insert-time limit. The default is aligned with the documented default of the Iceberg table property `write.target-file-size-bytes` (512 MiB), see https://iceberg.apache.org/docs/1.5.2/configuration/. Previously compaction merged all eligible files of a partition into a single output file."}, - {"iceberg_insert_max_bytes_in_data_file", 1024 * 1024 * 1024, 512 * 1024 * 1024, "Aligned with the documented default of the Iceberg table property `write.target-file-size-bytes` (512 MiB), see https://iceberg.apache.org/docs/1.5.2/configuration/."}, - {"iceberg_data_file_size_lower_threshold_compaction", 10 * 1024 * 1024, 384 * 1024 * 1024, "Aligned with how the Iceberg `rewrite_data_files` procedure derives `min-file-size-bytes`: 0.75 of the target file size (512 MiB). Compaction now selects files below 384 MiB instead of below 10 MiB."}, - {"iceberg_data_file_size_upper_threshold_compaction", 10ULL * 1024 * 1024 * 1024, 512ULL * 1024 * 1024 * 9 / 5, "Aligned with how the Iceberg `rewrite_data_files` procedure derives `max-file-size-bytes`: 1.8 of the target file size (512 MiB)."}, - {"iceberg_manifest_min_count_to_compact", 30, 100, "Aligned with the documented default of the Iceberg table property `commit.manifest.min-count-to-merge` (100), see https://iceberg.apache.org/docs/1.5.2/configuration/."}, - {"parallel_replicas_allow_merge_tables", false, false, "New setting to allow reading from a `Merge` table with plan-based parallel replicas, by expanding the `Merge` read into a union of the reads from the underlying `MergeTree` tables. It only has an effect together with `parallel_replicas_plan_based`."}, - {"optimize_mutations_with_partition_pruning", false, true, "New setting to automatically prune partitions for mutations based on WHERE clause"}, - {"statistics_max_set_size_for_exact_selectivity_estimation", 10000, 10000, "The bound on the cost of estimating the selectivity of `IN` with a large set is kept under `compatibility` with an earlier version: the previous value is deliberately equal to the new one, so that the uncapped estimation, which could add hundreds of milliseconds to the planning of a single query, is not restored."}, - {"type_json_skip_null_typed_paths", false, false, "New setting to treat NULL values in typed JSON paths as absent"}, - {"use_iceberg_manifest_list_partition_pruning", false, true, "New setting to skip Iceberg manifest files whose manifest-list partition summaries cannot match the query filter, without reading them."}, - {"enable_time_series_table", false, false, "The `TimeSeries` table engine and the `promql` dialect were moved to the private preview tier. Added an alias for setting `allow_experimental_time_series_table`."}, - {"enable_time_series_aggregate_functions", false, false, "The `timeSeries*` aggregate functions were moved to the private preview tier. Added an alias for setting `allow_experimental_time_series_aggregate_functions`."}, - {"output_format_arrow_record_batch_size", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target row count. The default `0` preserves one record batch per block."}, - {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, - }); addSettingsChanges(settings_changes_history, "26.8", { {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, @@ -204,7 +102,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"shrink_over_allocated_columns_min_waste_ratio", 1.0, 1.0, "New setting to shrink over-allocated columns to fit on INSERT to reduce peak memory usage. Disabled by default (1.0)."}, {"shrink_over_allocated_columns_min_waste_bytes", 16 * 1024 * 1024, 16 * 1024 * 1024, "New setting: minimum absolute wasted memory in a column for it to be shrunk to fit on INSERT."}, {"ai_function_allow_insecure_endpoint", true, false, "AI functions now reject insecure (http) endpoints to remote hosts by default."}, - {"ai_function_max_api_calls_per_query", 0, 1000, "Bound outbound AI function HTTP calls per query by default (previously 0 - unlimited)."}, + {"ai_function_max_input_tokens_per_query", 1000000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, + {"ai_function_max_output_tokens_per_query", 500000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, {"join_runtime_filter_min_probe_rows", 0, 1000, "New setting to control minimum probe side size for installing JOIN runtime filters. It wasn't limited before, so previous value is 0 meaning always install."}, {"read_in_order_use_virtual_row", false, true, "Enable the virtual row optimization by default. When reading in order of the primary key over many parts, it lets `MergingSortedTransform` reprioritize sources using primary key values from the sparse index, so parts that are not relevant for the query are not read, plus a bounded read-ahead window of at most `max_threads` parts that keeps reads parallel. This significantly reduces peak memory consumption (see https://github.com/ClickHouse/ClickHouse/issues/52624)."}, {"page", 0., 0., "New setting for paginated HTTP responses, equivalent to offset = limit * (page - 1). Float so it can hold negative or fractional values (passed through to SQL `LIMIT`/`OFFSET`)."}, diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference index b4345fecb8359..078656c7dc9d3 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference @@ -1,7 +1,5 @@ -- Current defaults false 0 1 --- compatibility = 26.9 restores the API call quota -false 1000 1 -- compatibility = 26.6 restores the legacy defaults true 0 0 0 diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql index 1881d8b85a6f5..1fe196405fc4c 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql @@ -3,12 +3,10 @@ -- no-replicated-database: named collections are server-global, not database-scoped -- ============================================================================= --- Four AI function default flips: `ai_function_allow_insecure_endpoint` from 1 to 0 and --- `ai_function_max_api_calls_per_query` from 0 (unlimited) to 1000 in 26.8, then --- `ai_function_max_retries` from 0 to 1 in 26.9, and `ai_function_max_api_calls_per_query` --- back to 0 (unlimited) in 26.10. `compatibility = 26.6` predates all of them and --- `compatibility = 26.9` reverts only the last one, which pins the previous_value/new_value --- pairs in `SettingsChangesHistory`. +-- Two AI function defaults were flipped in 26.8: `ai_function_allow_insecure_endpoint` from 1 +-- to 0 and `ai_function_max_retries` from 0 to 1. `ai_function_max_api_calls_per_query` stays 0 +-- (unlimited). `compatibility = 26.6` predates the flips and restores them, which pins the +-- previous_value/new_value pairs in `SettingsChangesHistory`. -- -- The endpoint check runs in `resolveAIParams`, before the zero-row early return -- in `executeImpl`, so an empty source table exercises it without any real HTTP @@ -26,11 +24,6 @@ SELECT '-- Current defaults'; SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); SELECT aiGenerate(x, map('credentials', 'ai_compat_remote_http')) FROM tab; -- { serverError BAD_ARGUMENTS } -SELECT '-- compatibility = 26.9 restores the API call quota'; -SET compatibility = '26.9'; -SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); -SELECT aiGenerate(x, map('credentials', 'ai_compat_remote_http')) FROM tab; -- { serverError BAD_ARGUMENTS } - SELECT '-- compatibility = 26.6 restores the legacy defaults'; SET compatibility = '26.6'; SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); From 9801cd0c7891285a89d69816b6d8a9ca8c4a60d4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 08:26:43 +0000 Subject: [PATCH 091/296] Backport #118674 to 26.8: Keep a `Backup` database loadable after `ALTER DATABASE ... MODIFY COMMENT` --- src/Databases/DatabaseBackup.cpp | 71 +++++++++-- src/Parsers/FunctionSecretArgumentsFinder.cpp | 14 +++ .../configs/backups.xml | 9 ++ .../test_backup_source_grants/test.py | 113 ++++++++++++++---- .../integration/test_database_backup/test.py | 102 ++++++++++++++++ ...base_backup_database_disk_engine.reference | 2 +- ...3_explicit_url_named_secret_mask.reference | 1 + ...4510_s3_explicit_url_named_secret_mask.sql | 7 ++ ...8_database_backup_modify_comment.reference | 4 + .../05148_database_backup_modify_comment.sh | 48 ++++++++ ...kup_quoted_locator_parallel_with.reference | 5 + ...ase_backup_quoted_locator_parallel_with.sh | 80 +++++++++++++ 12 files changed, 419 insertions(+), 37 deletions(-) create mode 100644 tests/queries/0_stateless/05148_database_backup_modify_comment.reference create mode 100755 tests/queries/0_stateless/05148_database_backup_modify_comment.sh create mode 100644 tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference create mode 100755 tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh diff --git a/src/Databases/DatabaseBackup.cpp b/src/Databases/DatabaseBackup.cpp index 0fa7ae4fe9c75..c46a7a663b1f6 100644 --- a/src/Databases/DatabaseBackup.cpp +++ b/src/Databases/DatabaseBackup.cpp @@ -43,6 +43,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -71,6 +72,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int INCORRECT_FILE_NAME; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; extern const int CANNOT_GET_CREATE_TABLE_QUERY; } @@ -464,8 +466,11 @@ ASTPtr DatabaseBackup::getCreateDatabaseQueryImpl() const { const auto & settings = getContext()->getSettingsRef(); + /// The locator is emitted as the function it is, not as a string literal holding its text: this + /// definition is what `ALTER DATABASE ... MODIFY COMMENT` writes back into the metadata file, and + /// the load path parses the second argument with `BackupInfo::fromAST`, which takes a function. const String query = fmt::format("CREATE DATABASE {} ENGINE = Backup({}, {})", - backQuoteIfNeed(database_name), quoteString(config.database_name), quoteString(config.backup_info.toString())); + backQuoteIfNeed(database_name), quoteString(config.database_name), config.backup_info.toString()); ParserCreateQuery parser; ASTPtr ast = parseQuery(parser, @@ -493,7 +498,7 @@ std::vector> DatabaseBackup::getTablesForBackup(co namespace { -DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr) +DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr, bool allow_locator_in_string_literal) { if (engine_args.size() != 2) throw Exception::createRuntime(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, @@ -502,6 +507,33 @@ DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr) DatabaseBackup::Configuration result; result.database_name = checkAndGetLiteralArgument(engine_args[0], "database_name"); + + /** A locator held in a string literal (`Backup('db', 'File(\'backup.zip\')')`) is the form that + * metadata rewritten by an older server carries, so it has to keep loading - a server that cannot + * parse its own metadata does not start at all. + * + * Only there: in a statement a user writes the locator must be the function it is, because that is + * the form `FunctionSecretArgumentsFinder` knows how to redact, and a quoted one would carry its + * credentials verbatim into `query_log`, `SHOW PROCESSLIST` and the distributed DDL payload. + */ + if (allow_locator_in_string_literal) + { + if (const auto * locator = engine_args[1]->as(); locator && locator->value.getType() == Field::Types::String) + { + result.backup_info = BackupInfo::fromString(locator->value.safeGet()); + return result; + } + } + + /// `BackupInfo::fromAST` puts the offending argument into its message, and that message reaches the + /// error log and the `exception` column of `query_log`. A locator held in a string literal is exactly + /// the text that can carry credentials, so refuse it here without echoing it. + if (!engine_args[1]->as()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Expected function as the backup destination of a `Backup` database. It must be spelled as the " + "function it is, such as `File('backup')` or `S3(...)`. The text given is not shown, because a " + "destination held in a string literal cannot be redacted and may carry credentials"); + result.backup_info = BackupInfo::fromAST(*engine_args[1]); return result; @@ -511,11 +543,20 @@ DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr) void DatabaseBackup::parseAndAuthorizeLocator(const ASTs & engine_args, ContextPtr query_context) { - /// A locator we cannot parse opens nothing: creation rejects it, so there is nothing to authorize. - if (engine_args.size() == 2 && !engine_args[1]->as()) - return; - - auto config = parseArguments(engine_args, query_context); + /** A locator that is not a function is refused right here rather than left to creation time. + * + * This preflight is the last point that still runs as the real user, and the creation it would + * otherwise rely on does not always run: `RESTORE DATABASE` issues `CREATE DATABASE IF NOT EXISTS`, + * which returns from `InterpreterCreateQuery::createDatabase` before `DatabaseFactory::get` when the + * target database already exists. With `allow_different_database_def = 1` the definition mismatch is + * waived as well, so a manifest an older server left holding a quoted locator would pass through the + * whole restore with its embedded source never authorized, while the function form of the same + * manifest is checked. + * + * Only `parseArguments` may formulate the refusal: the offending text can carry credentials and must + * not be echoed. + */ + auto config = parseArguments(engine_args, query_context, /*allow_locator_in_string_literal=*/ false); BackupFactory::instance().checkSourceAccess(config.backup_info, query_context, IBackup::OpenMode::READ); } @@ -531,13 +572,23 @@ void registerDatabaseBackup(DatabaseFactory & factory) if (engine->arguments) engine_args = engine->arguments->children; - auto config = parseArguments(engine_args, args.context); - /// Authorize only a newly introduced definition: one read back from this server's metadata was /// already validated, and a context with no user cannot be checked per user. + /// + /// Metadata is read back on three paths: the short `ATTACH DATABASE db`, a load under `force_restore_data`, + /// and the replay of the stored full `ATTACH DATABASE db ENGINE = Backup(...)` statement at server start. + /// The last one runs in plain `ATTACH` mode, so neither of the first two conditions covers it - and it is + /// exactly the path that has to load metadata an older server rewrote. + /// + /// The loader flag, not `internal`, is the discriminator: wrappers such as `PARALLEL WITH` run user + /// statements as internal ones, and a user's `ATTACH DATABASE ... ENGINE = Backup(...)` must neither + /// skip the source authorization nor get its locator accepted in the form the secret masker cannot redact. const bool has_real_user = args.context->getAccess()->getUserID().has_value(); + const bool is_internal_metadata_replay = args.is_metadata_replay && args.mode >= LoadingStrictnessLevel::ATTACH; const bool from_existing_metadata - = isLoadingFromExistingMetadata(args.mode) || args.create_query.attach_short_syntax; + = isLoadingFromExistingMetadata(args.mode) || args.create_query.attach_short_syntax || is_internal_metadata_replay; + + auto config = parseArguments(engine_args, args.context, /*allow_locator_in_string_literal=*/ from_existing_metadata); if (has_real_user && !from_existing_metadata) BackupFactory::instance().checkSourceAccess(config.backup_info, args.context, IBackup::OpenMode::READ); diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 0275cd8165089..8a6138d23552e 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -1181,6 +1181,20 @@ void FunctionSecretArgumentsFinder::findBackupDatabaseSecretArguments() auto storage_arg = function->arguments->at(1); auto storage_function = storage_arg->getFunction(); + /// A locator that is not a function - a string literal holding its text, or an expression - carries + /// the destination as text this finder cannot parse, and that text can hold an access key, a secret + /// access key or a presigned URL. The engine accepts such a locator only while replaying its own + /// metadata, but a statement carrying it is formatted before the engine rejects it: by `PARALLEL WITH`, + /// by the distributed DDL queue, and by `query_log`. Hide it whole rather than let it through verbatim. + if (!storage_function) + { + result.start = 1; + result.count = 1; + result.replacement = "'[HIDDEN]'"; + result.quote_replacement = false; + return; + } + /// The nested S3 destination is not recognized as an S3 engine when the formatter recurses into it, /// so its secrets must be masked here. Handle both forms: /// Backup('', S3('url', 'access_key_id', 'secret_access_key' [, ...])) diff --git a/tests/integration/test_backup_source_grants/configs/backups.xml b/tests/integration/test_backup_source_grants/configs/backups.xml index 2d99cb1d1ef92..acf77f033a2f2 100644 --- a/tests/integration/test_backup_source_grants/configs/backups.xml +++ b/tests/integration/test_backup_source_grants/configs/backups.xml @@ -1,5 +1,14 @@ + + + + local + /var/lib/clickhouse/disks/backup_disk/ + + + /var/lib/clickhouse/backups + backup_disk diff --git a/tests/integration/test_backup_source_grants/test.py b/tests/integration/test_backup_source_grants/test.py index 42b44da713dcb..ff8efd1037bdd 100644 --- a/tests/integration/test_backup_source_grants/test.py +++ b/tests/integration/test_backup_source_grants/test.py @@ -256,9 +256,8 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d # the check is skipped here and the restricted user reaches the embedded locator unchecked. # # The outer locator is File and IS granted, so a denial naming READ ON S3 can only come from the - # embedded S3 locator. `BACKUP DATABASE` serializes the locator as a string that - # `BackupInfo::fromAST` rejects before any check, so the function form is written into the - # manifest directly - the attacker-controlled-manifest shape this authorization exists for. + # embedded S3 locator, which is why the manifest is rewritten to hold one - the + # attacker-controlled-manifest shape this authorization exists for. # Every locator here is credential-free (File locally, a 1-argument S3 URL in the manifest): the # later definition-mismatch error logs both definitions, and a masked credential in that line # trips the tests-only `throw_on_match` masking rule and aborts the server. @@ -289,7 +288,7 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d assert "READ ON S3" in error, error # With the grant, authorization passes: the restore proceeds past CHECKING_ACCESS_RIGHTS and - # fails later on the pre-existing string-vs-function definition mismatch instead. + # fails later on the mismatch between the pre-existing definition and the rewritten one instead. node.query(f"GRANT READ ON S3 TO {USER}") error = node.query_and_get_error( "RESTORE DATABASE dbembedded ON CLUSTER one_shard FROM File('outer10')", user=USER @@ -300,41 +299,103 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d node.query("DROP DATABASE dbembedded SYNC") -def test_restore_on_cluster_of_a_real_backup_engine_manifest_is_unchanged(started_cluster): - # The manifest is NOT crafted here, which is the point: `BACKUP DATABASE` serializes the inner - # locator as a string, and authorizing it would parse it and reject it with BAD_ARGUMENTS before - # any access decision. Only this shape can catch that, so it is a separate case from the crafted - # one - which asserts the security property but cannot see this class. +def test_restore_on_cluster_refuses_a_quoted_embedded_locator_over_an_existing_database( + started_cluster, +): + # The pre-fix carrier: metadata an older server rewrote holds the locator as a string literal, and + # such a manifest can end up inside a backup. On this path the creation-time refusal never runs - + # the target database exists, so `CREATE DATABASE IF NOT EXISTS` returns before `DatabaseFactory`, + # and `allow_different_database_def = 1` waives the definition mismatch that would stop it next. + # The preflight is therefore the only place that can see the quoted locator, and it must refuse it + # rather than wave it through unauthorized: the function form of the very same manifest is denied + # for a missing `READ ON S3` in the case above. + # + # The refusal must not print the locator, which is why the assertion below looks for the host that + # a quoted locator could carry credentials next to. + node.query("DROP DATABASE IF EXISTS dbquoted SYNC") + node.query("BACKUP DATABASE d67785 TO File('inner13') FORMAT Null") + node.query("CREATE DATABASE dbquoted ENGINE = Backup('d67785', File('inner13'))") + node.query("BACKUP DATABASE dbquoted TO File('outer13') FORMAT Null") + manifest = ( + "CREATE DATABASE dbquoted ENGINE = Backup('d67785', " + "'S3(\\'http://minio1:9001/root/data/denied/b13\\')')" + ) + node.exec_in_container( + [ + "bash", + "-c", + "cat > /var/lib/clickhouse/backups/outer13/metadata/dbquoted.sql <<'MANIFEST'\n" + f"{manifest}\nMANIFEST", + ], + user="root", + ) + + assert node.query("SELECT count() FROM system.databases WHERE name = 'dbquoted'") == "1\n" + node.query(f"GRANT READ ON FILE TO {USER}") + + error = node.query_and_get_error( + "RESTORE DATABASE dbquoted ON CLUSTER one_shard FROM File('outer13') " + "SETTINGS allow_different_database_def = 1", + user=USER, + ) + assert "BAD_ARGUMENTS" in error, error + assert "Expected function as the backup destination" in error, error + assert "minio1" not in error, error + + node.query("DROP DATABASE dbquoted SYNC") + + +def test_restore_on_cluster_of_a_real_backup_engine_manifest_authorizes_the_inner_locator( + started_cluster, +): + # The manifest is NOT crafted here, which is the point: `BACKUP DATABASE` writes the inner locator + # as the function it is, so a real manifest carries a locator that authorization can decode, and + # the embedded one is checked exactly as a rewritten one is. Only this shape can see that, so it is + # a separate case from the rewritten one - which asserts the security property against a locator + # the definition never held. The outer locator is a Disk backup, which needs READ ON DISK, so a + # denial naming READ ON FILE can only come from the inner one. node.query("DROP DATABASE IF EXISTS dbreal SYNC") node.query("BACKUP DATABASE d67785 TO File('inner11') FORMAT Null") node.query("CREATE DATABASE dbreal ENGINE = Backup('d67785', File('inner11'))") - node.query("BACKUP DATABASE dbreal TO File('outer11') FORMAT Null") + node.query("BACKUP DATABASE dbreal TO Disk('backup_disk', 'outer11') FORMAT Null") manifest = node.exec_in_container( - ["bash", "-c", "cat /var/lib/clickhouse/backups/outer11/metadata/dbreal.sql"], + [ + "bash", + "-c", + "cat /var/lib/clickhouse/disks/backup_disk/outer11/metadata/dbreal.sql", + ], user="root", ) - # The locator really is the string form: an ASTLiteral, not an ASTFunction. - assert "Backup('d67785', 'File(\\'inner11\\')')" in manifest, manifest + # The locator really is the function form: an ASTFunction, not an ASTLiteral holding its text. + assert "Backup('d67785', File('inner11'))" in manifest, manifest - # Only the outer locator's own grant; the string-form inner one must not be authorized at all. - node.query(f"GRANT READ ON FILE TO {USER}") + # Only the outer locator's own grant, so the embedded File locator is the one that is missing. + node.query(f"GRANT READ ON DISK TO {USER}") + + error = node.query_and_get_error( + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11')", + user=USER, + ) + assert "ACCESS_DENIED" in error, error + assert "READ ON FILE" in error, error - # Target exists, so nothing is created and the restore fully succeeds. Authorizing the string - # form turns this into `Code: 36` out of CHECKING_ACCESS_RIGHTS. + # Target exists and its definition matches the manifest, so nothing is created and the restore + # fully succeeds. + node.query(f"GRANT READ ON FILE TO {USER}") node.query( - "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM File('outer11') FORMAT Null", user=USER + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11') FORMAT Null", + user=USER, ) - # Target absent, so creation runs and rejects the string form - as it does without this feature. - # `While creating database` is what pins the failure to the creation stage rather than the - # access-check one, which is where the same code would report it. + # Target absent, so creation runs: the locator parses, and the created database reads the tables of + # the inner backup. node.query("DROP DATABASE dbreal SYNC") - error = node.query_and_get_error( - "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM File('outer11')", user=USER + node.query( + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11') FORMAT Null", + user=USER, ) - assert "ACCESS_DENIED" not in error, error - assert "BAD_ARGUMENTS" in error, error - assert "While creating database" in error, error + assert node.query("SELECT x FROM dbreal.secrets") == "42\n" + node.query("DROP DATABASE dbreal SYNC") def test_explicit_base_backup_locator_is_authorized_on_the_initiator(started_cluster): diff --git a/tests/integration/test_database_backup/test.py b/tests/integration/test_database_backup/test.py index 8316d798e14f2..40aa406a050e1 100644 --- a/tests/integration/test_database_backup/test.py +++ b/tests/integration/test_database_backup/test.py @@ -12,6 +12,17 @@ with_minio=True, ) +# `test_database_backup_metadata_with_quoted_locator_loads_on_restart` rewrites a database metadata file +# in place, and such a file only exists when the metadata lives on the local disk, so that test runs on an +# instance which keeps the local database disk. +instance_local_metadata = cluster.add_instance( + "instance_local_metadata", + main_configs=["configs/backups.xml"], + stay_alive=True, + with_minio=True, + with_remote_database_disk=False, +) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -271,3 +282,94 @@ def test_database_backup_unavailable_but_server_starts(backup_destination): instance.query("DROP DATABASE IF EXISTS test_database_backup SYNC") instance.query("DROP DATABASE IF EXISTS test_database SYNC") cleanup_backup_files(instance) + + +def test_database_backup_metadata_with_quoted_locator_loads_on_restart(): + # Regression test for https://github.com/ClickHouse/ClickHouse/issues/118349 + # An older server regenerated the definition of a `Backup` database with the locator quoted into a + # string literal, and `ALTER DATABASE ... MODIFY COMMENT` wrote that back into `metadata/.sql`. + # The next start replays the stored full `ATTACH DATABASE ... ENGINE = Backup(...)` statement, which + # is neither the short `ATTACH` nor a force-restore load, so it has to accept that form on its own. + # + # The metadata file is rewritten in place here, so this runs on the instance whose metadata is a file + # on the local disk rather than an object on a remote database disk. + instance = instance_local_metadata + + cleanup_backup_files(instance) + + instance.query( + """ + DROP DATABASE IF EXISTS test_database SYNC; + DROP DATABASE IF EXISTS test_database_backup SYNC; + + CREATE DATABASE test_database; + + CREATE TABLE test_database.test_table (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + INSERT INTO test_database.test_table VALUES (0, 'test_database.test_table'); + + BACKUP DATABASE test_database TO File('test_database_backup_file'); + CREATE DATABASE test_database_backup ENGINE = Backup('test_database', File('test_database_backup_file')); + """ + ) + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + + # The metadata file exactly as a pre-fix server left it after a comment change. + instance.stop_clickhouse() + metadata = ( + "ATTACH DATABASE test_database_backup\n" + "ENGINE = Backup('test_database', 'File(\\'test_database_backup_file\\')')\n" + "COMMENT 'written by an older server'\n" + ) + instance.exec_in_container( + [ + "bash", + "-c", + "cat > /var/lib/clickhouse/metadata/test_database_backup.sql <<'SQL'\n" + + metadata + + "SQL\n", + ], + user="root", + ) + assert "'File(\\'test_database_backup_file\\')'" in instance.exec_in_container( + ["cat", "/var/lib/clickhouse/metadata/test_database_backup.sql"] + ) + instance.start_clickhouse() + + # The server started and the database loaded with its tables and comment. + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + assert ( + instance.query( + "SELECT comment FROM system.databases WHERE name = 'test_database_backup'" + ) + == "written by an older server\n" + ) + # The definition is regenerated with the locator as the function it is. + assert ( + "ENGINE = Backup('test_database', File('test_database_backup_file'))" + in instance.query("SHOW CREATE DATABASE test_database_backup FORMAT TSVRaw") + ) + + # A comment change on this server writes the function form, and that survives a restart too. + instance.query( + "ALTER DATABASE test_database_backup MODIFY COMMENT 'written by this server'" + ) + instance.restart_clickhouse() + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + assert ( + instance.query( + "SELECT comment FROM system.databases WHERE name = 'test_database_backup'" + ) + == "written by this server\n" + ) + + instance.query("DROP DATABASE test_database_backup SYNC") + instance.query("DROP DATABASE test_database SYNC") diff --git a/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference b/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference index 4f8c3ae7dc13c..80083db7617a9 100644 --- a/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference +++ b/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference @@ -20,7 +20,7 @@ 8 1500 9 1500 -- -CREATE DATABASE default_inner_backup_database\nENGINE = Backup(\'default_inner\', \'Disk(\\\'backups\\\', \\\'default_inner\\\')\') +CREATE DATABASE default_inner_backup_database\nENGINE = Backup(\'default_inner\', Disk(\'backups\', \'default_inner\')) test_table_1 15000 test_table_2 15000 -- diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference index eb36b31ef3f96..030f5c5e60319 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference @@ -207,6 +207,7 @@ CREATE DATABASE db_04510_mixed ENGINE = Backup(\'\', S3(\'url_dbmixed\', access_ CREATE DATABASE db_04510_ncurl ENGINE = Backup(\'\', S3(nc_dburl_missing, url = \'[HIDDEN]\')) CREATE DATABASE db_04510_hdr ENGINE = Backup(\'\', S3(\'url_dbhdr\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_expr ENGINE = Backup(\'\', S3(\'url_dbexpr\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_quoted ENGINE = Backup(\'\', \'[HIDDEN]\') CREATE DATABASE db_04510_s3pos ENGINE = S3(\'url_dbs3pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\') DROP DATABASE IF EXISTS default_1 CREATE DATABASE default_1 ENGINE = S3(\'url_dbenv\', \'ak\', \'[HIDDEN]\', use_environment_credentials = 1) diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql index 9cded2e5e118a..569086cf03b2c 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql @@ -280,6 +280,13 @@ CREATE DATABASE db_04510_hdr ENGINE = Backup('', S3('url_dbhdr', 'ak', 'SEKRIT_S CREATE DATABASE db_04510_expr ENGINE = Backup('', S3('url_dbexpr', 'ak', 'SEKRIT_SAK', extra_credentials(concat('extern', 'al_id') = 'SEKRIT_EXPR'))); -- { serverError BAD_ARGUMENTS } +-- A locator held in a string literal is text this finder cannot parse, and that text can carry an +-- access key or a presigned URL. The `Backup` engine accepts that form only while replaying its own +-- metadata, but a statement carrying it is formatted - by `PARALLEL WITH`, by the distributed DDL +-- queue, by `query_log` - before the engine rejects it, so it must be masked whole. +CREATE DATABASE db_04510_quoted ENGINE = Backup('', + 'S3(\'https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG\', \'ak\', \'SEKRIT_QUOTED\')'); -- { serverError BAD_ARGUMENTS } + -- The S3 database engine accepts no positional beyond secret_access_key; an extra positional must -- be masked in the logged query text. CREATE DATABASE db_04510_s3pos ENGINE = S3('url_dbs3pos', 'ak', 'SEKRIT_SAK', diff --git a/tests/queries/0_stateless/05148_database_backup_modify_comment.reference b/tests/queries/0_stateless/05148_database_backup_modify_comment.reference new file mode 100644 index 0000000000000..b3863579a9540 --- /dev/null +++ b/tests/queries/0_stateless/05148_database_backup_modify_comment.reference @@ -0,0 +1,4 @@ +10 +10 +a comment +refused diff --git a/tests/queries/0_stateless/05148_database_backup_modify_comment.sh b/tests/queries/0_stateless/05148_database_backup_modify_comment.sh new file mode 100755 index 0000000000000..4af56ac37b84b --- /dev/null +++ b/tests/queries/0_stateless/05148_database_backup_modify_comment.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup +ATTACHED_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_attached + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $BACKUP_DATABASE_NAME; +CREATE DATABASE $BACKUP_DATABASE_NAME; + +CREATE TABLE $BACKUP_DATABASE_NAME.test_table (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO $BACKUP_DATABASE_NAME.test_table SELECT number FROM numbers(10); + +BACKUP DATABASE $BACKUP_DATABASE_NAME TO Disk('backups', '$BACKUP_DATABASE_NAME') FORMAT Null; + +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', Disk('backups', '$BACKUP_DATABASE_NAME')); + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +""" + +# A comment change rewrites the metadata file of the database, so the locator has to survive the +# round trip: loading it back is what the server does at every start. +$CLICKHOUSE_CLIENT -q """ +ALTER DATABASE $ATTACHED_DATABASE_NAME MODIFY COMMENT 'a comment'; +DETACH DATABASE $ATTACHED_DATABASE_NAME; +ATTACH DATABASE $ATTACHED_DATABASE_NAME; + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +SELECT comment FROM system.databases WHERE name = '$ATTACHED_DATABASE_NAME'; +""" + +# A locator serialized as a string literal is accepted only while loading metadata an older server +# rewrote: in a statement a user writes it must be the function it is, because that is the form the +# secret masker redacts. +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE $ATTACHED_DATABASE_NAME; +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE $BACKUP_DATABASE_NAME; +""" diff --git a/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference new file mode 100644 index 0000000000000..8b916e9e97123 --- /dev/null +++ b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference @@ -0,0 +1,5 @@ +refused +refused +refused +0 1 +10 diff --git a/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh new file mode 100755 index 0000000000000..454cce40ebff8 --- /dev/null +++ b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup +ATTACHED_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_attached +OTHER_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_other + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $BACKUP_DATABASE_NAME; +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME; +CREATE DATABASE $BACKUP_DATABASE_NAME; + +CREATE TABLE $BACKUP_DATABASE_NAME.test_table (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO $BACKUP_DATABASE_NAME.test_table SELECT number FROM numbers(10); + +BACKUP DATABASE $BACKUP_DATABASE_NAME TO Disk('backups', '$BACKUP_DATABASE_NAME') FORMAT Null; +""" + +# `PARALLEL WITH` runs its statements as internal queries, and the full `ATTACH DATABASE ... ENGINE = ...` +# form runs in `ATTACH` mode - the same pair the server's own replay of stored metadata shows. The quoted +# locator is accepted only on that replay, never in a statement a user writes, however it is wrapped. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +# Each block below must start from the same state: whether the sibling `CREATE DATABASE` of a +# `PARALLEL WITH` whose other statement threw is committed or rolled back depends on how the +# statements were scheduled, and with `max_threads = 1` it is committed. Drop it in between, so the +# refusal under test is what the block observes, not a leftover "database already exists". +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +# A quoted locator can carry credentials, and `PARALLEL WITH` formats its statements before the engine +# refuses them, so the formatted text must hide it: neither the logged query text (the same text an +# `ON CLUSTER` statement puts into the distributed DDL payload) nor the refusal message may carry it. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'S3(\\'http://localhost:11111/05218\\', \\'ak\\', \\'SEKRIT_05218\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT -q """ +SELECT countIf(query LIKE '%SEKRIT_05218%' OR exception LIKE '%SEKRIT_05218%'), countIf(query LIKE '%Backup(%[HIDDEN]%') > 0 +FROM system.query_log +WHERE current_database = currentDatabase() + AND type != 'QueryStart' + AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE; +""" + +# The function form goes through the same wrapper. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE IF NOT EXISTS $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', Disk('backups', '$BACKUP_DATABASE_NAME')); + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +""" + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME; +DROP DATABASE $BACKUP_DATABASE_NAME; +""" From 52129a0751d4daf45fe114a48aa4b2fd1d8b986e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 08:37:19 +0000 Subject: [PATCH 092/296] Backport #115693 to 26.8: Do not assert scan-progress values decided by an async race in test_early_return_limit --- tests/integration/test_storage_delta/test.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index cc8f7034012f3..efc207fa0d186 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -5358,24 +5358,16 @@ def test_early_return_limit(started_cluster, use_delta_kernel): assert first_check_hits > 0 or queue_check_hits > 0 - assert 1 == int(instance.query( - f"SELECT count() FROM system.text_log WHERE query_id = '{query_id}' AND message LIKE '%List batch size is 1/1, shutdown: true%'" - )) - # Early return should scan significantly fewer files # With s3_list_object_keys_size=1, queue pauses frequently forcing shutdown checks # Should stop very early after consuming just a few files assert scanned_files < full_scan_files, \ f"Early return should scan fewer files: {scanned_files} >= {full_scan_files}" - # 3 because: - # we have async reader creation with 2 existing readers at a moment of time, - # each calls next() and consumes 2 files from the scan. - # It takes 1 file for the query to stop because of LIMIT 1. - # But because scan is also asynchronous and continues once batch limit is not reached, - # we get +1 scanned file. - assert scanned_files == 3, \ - f"Early return should scan 3 files with LIMIT 1, but scanned {scanned_files}" + # At most 3: two async readers each consume one file, plus one the scan produces before + # it observes shutdown. Fewer is legal when shutdown lands earlier. + assert scanned_files <= 3, \ + f"Early return should scan at most 3 files with LIMIT 1, but scanned {scanned_files}" def test_struct_dotted_field_names(started_cluster): From cdbaf82420964d5c329cc964b86562c93f4e8214 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 08:56:30 +0000 Subject: [PATCH 093/296] Backport #119686 to 26.8: Fix flaky `01780_column_sparse`: order the `arrayFilter` result before limiting it --- tests/queries/0_stateless/01780_column_sparse.reference | 2 +- tests/queries/0_stateless/01780_column_sparse.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01780_column_sparse.reference b/tests/queries/0_stateless/01780_column_sparse.reference index 3393361a19fb0..7452be01648e6 100644 --- a/tests/queries/0_stateless/01780_column_sparse.reference +++ b/tests/queries/0_stateless/01780_column_sparse.reference @@ -132,7 +132,7 @@ SELECT id % 7, sum(u) FROM t_sparse GROUP BY id % 7 ORDER BY id % 7; 4 190 5 330 6 270 -SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] LIMIT 5; +SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] ORDER BY id LIMIT 5; [1] [1,3] [1,3,5] diff --git a/tests/queries/0_stateless/01780_column_sparse.sql b/tests/queries/0_stateless/01780_column_sparse.sql index 8e3c4372d05aa..c5b98b1fd422f 100644 --- a/tests/queries/0_stateless/01780_column_sparse.sql +++ b/tests/queries/0_stateless/01780_column_sparse.sql @@ -27,7 +27,7 @@ SELECT * FROM t_sparse WHERE arr2 != [] ORDER BY id; SELECT sum(u) FROM t_sparse; SELECT id % 7, sum(u) FROM t_sparse GROUP BY id % 7 ORDER BY id % 7; -SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] LIMIT 5; +SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] ORDER BY id LIMIT 5; CREATE TABLE t_sparse_1 (id UInt64, v Int64) ENGINE = MergeTree ORDER BY tuple() From fdec066dc3af8d90dfa2f28e934935ff36300818 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 09:04:27 +0000 Subject: [PATCH 094/296] Backport #119743 to 26.8: Tolerate Iceberg manifest header schemas that conflict with metadata.json --- src/Core/Settings.cpp | 5 + src/Core/SettingsChangesHistory.cpp | 4 + .../Iceberg/ManifestFileIterator.cpp | 12 +- .../DataLakes/Iceberg/SchemaProcessor.cpp | 132 ++++++++++++---- .../DataLakes/Iceberg/SchemaProcessor.h | 26 +++- .../tests/gtest_iceberg_schema_processor.cpp | 144 ++++++++++++++++++ ...tion_conflicting_manifest_schema.reference | 7 + ..._compaction_conflicting_manifest_schema.sh | 81 ++++++++++ 8 files changed, 376 insertions(+), 35 deletions(-) create mode 100644 tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference create mode 100755 tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 304d1177a1796..1a73f7833bd16 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8101,6 +8101,11 @@ Default partition strategy for file like engines. Applied only to `CREATE` queri )", 0) \ DECLARE(Bool, use_iceberg_partition_pruning, true, R"( Use Iceberg partition pruning for Iceberg tables +)", 0) \ + DECLARE(Bool, iceberg_tolerate_conflicting_manifest_schemas, true, R"( +If enabled and the `schema` key of an Iceberg manifest file header carries a schema that differs from the schema already registered for the same schema-id from metadata.json, the metadata.json schema is used and the manifest header copy is ignored with a warning. If disabled, such a conflict fails the query with an ICEBERG_SPECIFICATION_VIOLATION error. + +The manifest header schema is only a copy of the table schema at the time the manifest was written, and some writers (e.g. AWS S3 Tables maintenance jobs) have been observed storing degraded copies there. Other query engines resolve schemas from metadata.json and ignore divergent header copies, so the default follows them. A conflict between two metadata.json schema definitions still always fails the query. )", 0) \ DECLARE(Bool, optimize_distinct_in_order, true, R"( Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a13257bba3538..62d536511a0cc 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -41,6 +41,10 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "26.10", + { + {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema and log a warning instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.10 restores the previous strict behavior."}, + }); addSettingsChanges(settings_changes_history, "26.8", { {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index 9c0d2db914de9..189f1e700d1b3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -9,6 +9,7 @@ #include +#include #include #include @@ -17,6 +18,7 @@ #include #include +#include #include #include #include @@ -39,6 +41,11 @@ namespace DB::ErrorCodes extern const int BAD_ARGUMENTS; } +namespace DB::Setting +{ + extern const SettingsBool iceberg_tolerate_conflicting_manifest_schemas; +} + namespace ProfileEvents { extern const Event IcebergPartitionPrunedFiles; @@ -319,7 +326,10 @@ std::shared_ptr ManifestFileIterator::create( const Poco::JSON::Object::Ptr & schema_object = json.extract(); Int32 manifest_schema_id = schema_object->getValue(f_schema_id); - schema_processor.addIcebergTableSchema(schema_object); + schema_processor.addIcebergTableSchema( + schema_object, + IcebergSchemaProcessor::SchemaSource::ManifestFile, + context_->getSettingsRef()[Setting::iceberg_tolerate_conflicting_manifest_schemas]); PartitionSpecification partition_spec_vec; for (size_t i = 0; i != partition_specification->size(); ++i) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index c154a025ab3a8..978f863f9dfdc 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -393,7 +393,17 @@ namespace Iceberg std::string IcebergSchemaProcessor::default_link{}; -void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr) +void IcebergSchemaProcessor::dropCachedSchema(Int32 schema_id) +{ + iceberg_table_schemas_by_ids.erase(schema_id); + clickhouse_table_schemas_by_ids.erase(schema_id); + std::erase_if(transform_dags_by_ids, [schema_id](const auto & item) { return item.first.first == schema_id || item.first.second == schema_id; }); + std::erase_if(clickhouse_types_by_source_ids, [schema_id](const auto & item) { return item.first.first == schema_id; }); + std::erase_if(clickhouse_ids_by_source_names, [schema_id](const auto & item) { return item.first.first == schema_id; }); +} + +void IcebergSchemaProcessor::addIcebergTableSchema( + Poco::JSON::Object::Ptr schema_ptr, SchemaSource source, bool tolerate_conflicting_manifest_schemas) { std::lock_guard lock(mutex); @@ -404,7 +414,6 @@ void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schem if (!schema_ptr->isArray(f_fields) || schema_ptr->getArray(f_fields)->size() == 0) return; - current_schema_id = schema_id; if (iceberg_table_schemas_by_ids.contains(schema_id)) { chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); @@ -414,44 +423,101 @@ void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schem type_mapping[f_geography] = f_binary; type_mapping[f_geometry] = f_binary; } - /// A schema-id is immutable per the Iceberg spec: re-binding it to different fields is malformed metadata. - if (!schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr, type_mapping)) - throw Exception( - ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "Iceberg schema with schema-id {} is bound to two different schemas across metadata versions", - schema_id); - } - else - { - auto fields = schema_ptr->get(f_fields).extract(); - /// A field name is required per the Iceberg spec, and an empty column name is not representable in ClickHouse. - for (size_t i = 0; i != fields->size(); ++i) + if (schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr, type_mapping)) + { + /// An identical metadata.json copy confirms a copy that was registered from a manifest header. + if (source == SchemaSource::Metadata) + manifest_sourced_schema_ids.erase(schema_id); + return; + } + + /// The 'schema' key in a manifest file header is only a copy of the table schema at the + /// time the manifest was written; metadata.json is the authoritative source. Broken writers + /// have been observed storing degraded copies in manifest headers under an already-used + /// schema-id (e.g. AWS S3 Tables maintenance jobs writing `timestamp` instead of + /// `timestamptz`, or a schema containing only the partition source columns). Other engines + /// (Spark, Trino, PyIceberg, DuckDB) resolve schemas from metadata.json and ignore such + /// divergent copies that came from a manifest. + const bool registered_from_manifest = manifest_sourced_schema_ids.contains(schema_id); + if (registered_from_manifest) { - auto field = fields->getObject(static_cast(i)); - if (field->getValue(f_name).empty()) + if (source == SchemaSource::Metadata) + { + /// A read registers the metadata.json schemas before it walks any manifest, but the + /// maintenance entrypoints (`remove_orphan_files`, `expire_snapshots`, manifest + /// compaction, mutation validation) can reach a manifest header first on an empty + /// processor. A schema that came from a manifest is never authoritative, so the + /// metadata.json copy replaces it, along with everything that was derived from it. + LOG_WARNING( + getLogger("IcebergSchemaProcessor"), + "Schema-id {} was registered from a manifest file header and differs from the schema " + "metadata.json binds to that id; replacing the schema that came from the manifest", + schema_id); + dropCachedSchema(schema_id); + manifest_sourced_schema_ids.erase(schema_id); + } + else + { + /// Two manifest headers disagree on a schema-id that metadata.json has not defined: + /// there is no authoritative copy to decide which one the data was written with. throw Exception( ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "Iceberg schema with schema-id {} has a field with id {} whose name is empty", - schema_id, - field->getValue(f_id)); + "Iceberg schema with schema-id {} is bound to two different schemas by manifest file headers, " + "and metadata.json does not define it", + schema_id); + } } - - auto clickhouse_schema = std::make_shared(); - String current_full_name{}; - for (size_t i = 0; i != fields->size(); ++i) + else { - auto field = fields->getObject(static_cast(i)); - auto name = field->getValue(f_name); - bool required = field->getValue(f_required); - current_full_name = name; - auto type = getFieldType(field, f_type, required, current_full_name, true); - clickhouse_schema->push_back(NameAndTypePair{name, type}); - clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; - clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); + if (source == SchemaSource::ManifestFile && tolerate_conflicting_manifest_schemas) + { + LOG_WARNING( + getLogger("IcebergSchemaProcessor"), + "Manifest file header carries schema-id {} which differs from the schema already " + "registered for that id from metadata.json; ignoring the manifest header copy " + "(disable setting `iceberg_tolerate_conflicting_manifest_schemas` to make this an error)", + schema_id); + return; + } + /// A schema-id is immutable per the Iceberg spec: re-binding it to different fields is malformed metadata. + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Iceberg schema with schema-id {} is bound to two different schemas across metadata versions", + schema_id); } - clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; - iceberg_table_schemas_by_ids[schema_id] = schema_ptr; } + + auto fields = schema_ptr->get(f_fields).extract(); + /// A field name is required per the Iceberg spec, and an empty column name is not representable in ClickHouse. + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + if (field->getValue(f_name).empty()) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Iceberg schema with schema-id {} has a field with id {} whose name is empty", + schema_id, + field->getValue(f_id)); + } + + current_schema_id = schema_id; + auto clickhouse_schema = std::make_shared(); + String current_full_name{}; + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + auto name = field->getValue(f_name); + bool required = field->getValue(f_required); + current_full_name = name; + auto type = getFieldType(field, f_type, required, current_full_name, true); + clickhouse_schema->push_back(NameAndTypePair{name, type}); + clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; + clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); + } + clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; + iceberg_table_schemas_by_ids[schema_id] = schema_ptr; + if (source == SchemaSource::ManifestFile) + manifest_sourced_schema_ids.insert(schema_id); current_schema_id = std::nullopt; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h index a072f5ebc1276..396f244ec07c8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h @@ -88,9 +88,23 @@ class IcebergSchemaProcessor using Node = ActionsDAG::Node; public: + /// Where a schema copy being registered comes from. metadata.json is the authoritative source; + /// the 'schema' key of a manifest file header is only a snapshot of the table schema at the time + /// the manifest was written. A schema that came from a manifest is never authoritative: it is + /// replaced by the metadata.json copy of the same schema-id whenever that one is registered, and + /// it may be ignored if it conflicts with an already registered metadata.json copy. + enum class SchemaSource + { + Metadata, + ManifestFile, + }; + explicit IcebergSchemaProcessor(bool allow_geo_parser_ = false) : allow_geo_parser(allow_geo_parser_) {} - void addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr); + void addIcebergTableSchema( + Poco::JSON::Object::Ptr schema_ptr, + SchemaSource source = SchemaSource::Metadata, + bool tolerate_conflicting_manifest_schemas = false); std::shared_ptr getClickHouseTableSchemaById(Int32 id); std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id); NameAndTypePair getFieldCharacteristics(Int32 schema_version, Int32 source_id) const; @@ -125,6 +139,16 @@ class IcebergSchemaProcessor mutable std::map, Int32> clickhouse_ids_by_source_names TSA_GUARDED_BY(mutex); std::optional current_schema_id TSA_GUARDED_BY(mutex) = 0; std::unordered_map schema_id_by_snapshot TSA_GUARDED_BY(mutex); + /// Schema-ids whose registered copy came from a manifest file header and has not been confirmed + /// by an identical metadata.json copy yet. Such a copy is replaced when metadata.json binds the + /// schema-id to a different schema, and two manifest headers disagreeing on such a schema-id is + /// an error, because no authoritative copy is left to decide between them. + std::unordered_set manifest_sourced_schema_ids TSA_GUARDED_BY(mutex); + + /// Forget the schema registered for `schema_id` together with everything derived from it: the + /// per-field lookups and the cached schema transformation DAGs in either direction. They are + /// keyed by schema-id and never rebuilt once populated, so a stale entry would keep answering. + void dropCachedSchema(Int32 schema_id) TSA_REQUIRES(mutex); NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type, String & current_full_name, bool is_subfield_of_root); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp index 504f0f8fc7e9e..b79d3ba94033a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp @@ -7,6 +7,11 @@ #include #include +namespace DB::ErrorCodes +{ + extern const int ICEBERG_SPECIFICATION_VIOLATION; +} + using namespace DB::Iceberg; namespace @@ -185,6 +190,145 @@ TEST(IcebergSchemaProcessor, RebindingSchemaIdToDifferentTypeStillRejected) EXPECT_THROW(processor.addIcebergTableSchema(second), DB::Exception); } +/// The manifest header 'schema' key is only a copy of the table schema at write time; metadata.json +/// is authoritative. Broken writers (observed with AWS S3 Tables maintenance jobs) store degraded +/// copies in manifest headers under an already-used schema-id, e.g. `timestamp` instead of +/// `timestamptz`. With toleration enabled (the default of `iceberg_tolerate_conflicting_manifest_schemas`), +/// the divergent manifest copy must be ignored and the metadata.json schema kept. +TEST(IcebergSchemaProcessor, ConflictingManifestSchemaToleratedWhenEnabled) +{ + auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(from_metadata); + EXPECT_NO_THROW(processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + + /// The metadata.json copy must win: the field keeps the timestamptz type. + auto schema = processor.getClickHouseTableSchemaById(0); + ASSERT_EQ(schema->size(), 1u); + EXPECT_EQ(schema->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); +} + +/// With toleration disabled (`compatibility` below 26.9), the same conflict must still fail. +TEST(IcebergSchemaProcessor, ConflictingManifestSchemaRejectedWhenDisabled) +{ + auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(from_metadata); + EXPECT_THROW( + processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + DB::Exception); +} + +/// Toleration only applies to manifest header copies: two conflicting metadata.json definitions of +/// the same schema-id are genuine catalog corruption and must always be rejected. +TEST(IcebergSchemaProcessor, ConflictingMetadataSchemaAlwaysRejected) +{ + auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(first); + EXPECT_THROW( + processor.addIcebergTableSchema( + second, IcebergSchemaProcessor::SchemaSource::Metadata, /*tolerate_conflicting_manifest_schemas=*/true), + DB::Exception); +} + +/// A manifest header carrying a schema-id NOT registered from metadata.json (e.g. an expired schema +/// still referenced by an old manifest) must register normally regardless of the toleration flag. +TEST(IcebergSchemaProcessor, ManifestSchemaWithNewIdRegistersNormally) +{ + auto from_manifest = parseSchema(R"json({"schema-id":5,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + EXPECT_NO_THROW(processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + EXPECT_TRUE(processor.hasClickHouseTableSchemaById(5)); +} + +/// The maintenance entrypoints (`remove_orphan_files`, `expire_snapshots`, manifest compaction) can +/// register a schema that came from a manifest before metadata.json is read. Such a schema is never +/// authoritative: the metadata.json copy of the same schema-id must replace it, together with the +/// per-field lookups and the cached transformation DAGs derived from it. +TEST(IcebergSchemaProcessor, ManifestHeaderRegisteredFirstIsReplacedByMetadata) +{ + auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + auto other = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"},{"id":2,"name":"v","required":false,"type":"int"}]})json"); + auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + processor.addIcebergTableSchema(other); + + /// Populate everything derived from the schema that came from the manifest. + EXPECT_EQ(processor.getFieldCharacteristics(0, 1).type->getName(), "Nullable(DateTime64(6))"); + auto stale_dag = processor.getSchemaTransformationDagByIds(0, 1); + ASSERT_NE(stale_dag, nullptr); + + EXPECT_NO_THROW(processor.addIcebergTableSchema(from_metadata)); + + auto schema = processor.getClickHouseTableSchemaById(0); + ASSERT_EQ(schema->size(), 1u); + EXPECT_EQ(schema->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); + EXPECT_EQ(processor.getFieldCharacteristics(0, 1).type->getName(), "Nullable(DateTime64(6, 'UTC'))"); + EXPECT_NE(processor.getSchemaTransformationDagByIds(0, 1).get(), stale_dag.get()); + + /// Once metadata.json has settled the schema-id, a later divergent schema from a manifest is handled like + /// any header conflicting with a known metadata.json copy: ignored when tolerated, rejected otherwise. + EXPECT_NO_THROW(processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + EXPECT_THROW( + processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + DB::Exception); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); +} + +/// An identical metadata.json copy confirms a schema that came from a manifest and was registered before it; the schema-id then +/// behaves as if it had been registered from metadata.json in the first place. +TEST(IcebergSchemaProcessor, ManifestHeaderRegisteredFirstIsConfirmedByIdenticalMetadata) +{ + auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto degraded = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + EXPECT_NO_THROW(processor.addIcebergTableSchema(from_metadata)); + + EXPECT_NO_THROW(processor.addIcebergTableSchema( + degraded, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + EXPECT_THROW( + processor.addIcebergTableSchema( + degraded, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + DB::Exception); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); +} + +/// Two manifest headers that disagree on a schema-id metadata.json has not defined leave no +/// authoritative copy to decide between them, so this is a specification violation even when +/// conflicts with metadata.json are tolerated. Other schema-ids are unaffected. +TEST(IcebergSchemaProcessor, ConflictingManifestHeadersWithoutMetadataSchemaRejected) +{ + auto first = parseSchema(R"json({"schema-id":5,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + auto second = parseSchema(R"json({"schema-id":5,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto other = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(other); + processor.addIcebergTableSchema(first, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + try + { + processor.addIcebergTableSchema(second, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + FAIL() << "expected ICEBERG_SPECIFICATION_VIOLATION"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + } + EXPECT_TRUE(processor.hasClickHouseTableSchemaById(0)); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); +} + /// A renamed field bound to the same schema-id must still be rejected (issue #107316). TEST(IcebergSchemaProcessor, RebindingSchemaIdToRenamedFieldStillRejected) { diff --git a/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference new file mode 100644 index 0000000000000..a3a948fe51786 --- /dev/null +++ b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference @@ -0,0 +1,7 @@ +strict read +ICEBERG_SPECIFICATION_VIOLATION +tolerant compaction +3 +DateTime64(6, \'UTC\') 2024-01-01 00:00:00.000000 1 +DateTime64(6, \'UTC\') 2024-01-02 00:00:00.000000 2 +DateTime64(6, \'UTC\') 2024-01-03 00:00:00.000000 3 diff --git a/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh new file mode 100755 index 0000000000000..9a72f718081a2 --- /dev/null +++ b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) +# - no-parallel: uses DETACH/ATTACH which serializes per database + +# A manifest file header keeps a copy of the schema the data was written with, while metadata.json is +# the authoritative source. The conflict below is produced the way broken writers produce it: the +# manifest headers keep the `timestamp` type, while metadata.json binds the same schema-id to +# `timestamptz`. +# +# A read registers the metadata.json schemas before it walks the manifests, so a strict read meets a +# header conflicting with a known metadata.json copy and fails with `ICEBERG_SPECIFICATION_VIOLATION`. +# `OPTIMIZE TABLE ... MANIFEST` walks the current manifests first and registers the metadata.json +# schemas only afterwards, so on a freshly attached table the schemas that came from the manifests are registered before +# metadata.json: the metadata.json copy has to replace them and the compaction has to succeed. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The data is written as `timestamp` (parsed in the session time zone) and read back as `timestamptz` +# (rendered in UTC), so the session time zone has to be UTC for the values to round-trip unchanged. +CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT} --session_timezone UTC" + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap 'rm -rf "${TABLE_PATH}" 2>/dev/null' EXIT + +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${TABLE} (ts DateTime64(6), v Int32) + ENGINE = IcebergLocal('${TABLE_PATH}', 'Parquet') +" + +# One INSERT per manifest, so the table has enough manifests to compact. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --use_iceberg_metadata_files_cache=0 -m --query " + INSERT INTO ${TABLE} VALUES ('2024-01-01 00:00:00', 1); + INSERT INTO ${TABLE} VALUES ('2024-01-02 00:00:00', 2); + INSERT INTO ${TABLE} VALUES ('2024-01-03 00:00:00', 3); +" + +# Rebind the schema-id in metadata.json to a schema that differs from the schemas that came from the manifests. +LATEST_METADATA=$(ls "${TABLE_PATH}"metadata/v*.metadata.json | sed 's#.*/v##;s#\.metadata.json##' | sort -n | tail -1) +python3 - "${TABLE_PATH}metadata/v${LATEST_METADATA}.metadata.json" <<'PY' +import json, sys +path = sys.argv[1] +meta = json.load(open(path)) +for schema in meta["schemas"]: + for field in schema["fields"]: + if field["type"] == "timestamp": + field["type"] = "timestamptz" +json.dump(meta, open(path, "w")) +PY + +# Drop the in-memory metadata and the shared schema processor, so the read starts from an empty +# processor and registers the metadata.json schemas before it meets the schemas that came from the manifests. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query "DETACH TABLE ${TABLE}" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --send_logs_level=fatal --query "ATTACH TABLE ${TABLE}" + +echo "strict read" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --iceberg_tolerate_conflicting_manifest_schemas=0 \ + --query "SELECT count() FROM ${TABLE}" 2>&1 \ + | grep -oF 'ICEBERG_SPECIFICATION_VIOLATION' | head -n1 + +# Start from an empty processor again, so the compaction registers the schemas that came from the manifests first +# and the metadata.json schemas replace them. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query "DETACH TABLE ${TABLE}" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --send_logs_level=fatal --query "ATTACH TABLE ${TABLE}" + +echo "tolerant compaction" +${CLICKHOUSE_CLIENT} --allow_experimental_iceberg_compaction=1 --use_iceberg_metadata_files_cache=0 \ + --iceberg_tolerate_conflicting_manifest_schemas=1 --send_logs_level=error \ + --query "OPTIMIZE TABLE ${TABLE} MANIFEST SETTINGS iceberg_manifest_min_count_to_compact=2" + +# The compaction rewrote three data manifests into one, and the data is intact. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --iceberg_tolerate_conflicting_manifest_schemas=1 --query " + SELECT count() FROM ${TABLE}; + SELECT toTypeName(ts), ts, v FROM ${TABLE} ORDER BY v; +" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE} SYNC" From 4963403fe6e249b587212db9624ec2518d68bd0d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 09:35:45 +0000 Subject: [PATCH 095/296] Backport #120036 to 26.8: Fix silent truncation at `LimitReadBuffer` limits and the multipart form budget --- src/Core/ExternalTable.cpp | 13 +- src/Core/ExternalTable.h | 3 + src/Core/MySQL/IMySQLReadPacket.cpp | 11 +- src/Core/MySQL/IMySQLReadPacket.h | 3 + .../tests/gtest_mysql_limited_read_packet.cpp | 121 +++++++++++++ src/IO/LimitReadBuffer.cpp | 11 +- src/IO/LimitReadBuffer.h | 3 + src/IO/tests/gtest_limit_read_buffer.cpp | 162 ++++++++++++++++++ src/Server/HTTP/HTTPServerRequest.cpp | 4 +- src/Server/TCPHandler.cpp | 5 +- ...p_multipart_form_data_total_size.reference | 3 + ...213_http_multipart_form_data_total_size.sh | 32 ++++ 12 files changed, 359 insertions(+), 12 deletions(-) create mode 100644 src/Core/tests/gtest_mysql_limited_read_packet.cpp create mode 100644 src/IO/tests/gtest_limit_read_buffer.cpp create mode 100644 tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference create mode 100755 tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index 584b2b50f1fbd..a03f0109a3ae4 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -216,11 +216,12 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, const Settings & settings = getContext()->getSettingsRef(); - if (settings[Setting::http_max_multipart_form_data_size]) + const size_t form_data_size_limit = settings[Setting::http_max_multipart_form_data_size]; + if (form_data_size_limit) read_buffer = std::make_unique( stream, LimitReadBuffer::Settings{ - .read_no_more = settings[Setting::http_max_multipart_form_data_size], + .read_no_more = form_data_size_limit > form_data_bytes_read ? form_data_size_limit - form_data_bytes_read : 0, .expect_eof = true, .excetion_hint = "the maximum size of multipart/form-data. This limit can be tuned by 'http_max_multipart_form_data_size' setting", }); @@ -293,6 +294,14 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, CompletedPipelineExecutor executor(pipeline); executor.execute(); + + /// Whatever the format left unread still belongs to the part, and `HTMLForm` would skip it + /// outside the limiter. Read it out through the limiter so it counts against the budget, and so + /// that a part running past the budget trips `expect_eof` rather than slipping by. + if (form_data_size_limit) + read_buffer->ignoreAll(); + + form_data_bytes_read += read_buffer->count(); } } diff --git a/src/Core/ExternalTable.h b/src/Core/ExternalTable.h index e7c5918a209d8..c120fb22e2cd4 100644 --- a/src/Core/ExternalTable.h +++ b/src/Core/ExternalTable.h @@ -91,6 +91,9 @@ class ExternalTablesHandler : public HTMLForm::PartHandler, BaseExternalTable, W private: const Poco::Net::NameValueCollection & params; + /// `http_max_multipart_form_data_size` is one budget across the form's external-table parts, not a + /// fresh limit per part. Other fields have their own bound, `http_max_field_value_size`. + size_t form_data_bytes_read = 0; }; diff --git a/src/Core/MySQL/IMySQLReadPacket.cpp b/src/Core/MySQL/IMySQLReadPacket.cpp index e29fb0385a562..d40af18658bac 100644 --- a/src/Core/MySQL/IMySQLReadPacket.cpp +++ b/src/Core/MySQL/IMySQLReadPacket.cpp @@ -18,6 +18,11 @@ namespace MySQLProtocol void IMySQLReadPacket::readPayload(ReadBuffer & in, uint8_t & sequence_id) { MySQLPacketPayloadReadBuffer payload(in, sequence_id); + readPayloadFrom(payload); +} + +void IMySQLReadPacket::readPayloadFrom(ReadBuffer & payload) +{ payload.next(); readPayloadImpl(payload); if (!payload.eof()) @@ -35,8 +40,10 @@ void IMySQLReadPacket::readPayloadWithUnpacked(ReadBuffer & in) void LimitedReadPacket::readPayload(ReadBuffer &in, uint8_t &sequence_id) { - LimitReadBuffer limited(in, {.read_no_more = 10000, .expect_eof = true, .excetion_hint = "too long MySQL packet."}); - IMySQLReadPacket::readPayload(limited, sequence_id); + /// On the payload, not on `in`: `in` is the connection, which continues with the next packet. + MySQLPacketPayloadReadBuffer payload(in, sequence_id); + LimitReadBuffer limited(payload, {.read_no_more = 10000, .expect_eof = true, .excetion_hint = "too long MySQL packet."}); + readPayloadFrom(limited); } void LimitedReadPacket::readPayloadWithUnpacked(ReadBuffer & in) diff --git a/src/Core/MySQL/IMySQLReadPacket.h b/src/Core/MySQL/IMySQLReadPacket.h index b6c3d59f5eef6..1e7fb1b501b47 100644 --- a/src/Core/MySQL/IMySQLReadPacket.h +++ b/src/Core/MySQL/IMySQLReadPacket.h @@ -23,6 +23,9 @@ class IMySQLReadPacket protected: virtual void readPayloadImpl(ReadBuffer & buf) = 0; + + /// `payload` has to end where the packet ends. + void readPayloadFrom(ReadBuffer & payload); }; class LimitedReadPacket : public IMySQLReadPacket diff --git a/src/Core/tests/gtest_mysql_limited_read_packet.cpp b/src/Core/tests/gtest_mysql_limited_read_packet.cpp new file mode 100644 index 0000000000000..62fbdb893a44c --- /dev/null +++ b/src/Core/tests/gtest_mysql_limited_read_packet.cpp @@ -0,0 +1,121 @@ +#include + +#include +#include +#include +#include + +#include + +using namespace DB; +using namespace DB::MySQLProtocol; + +namespace DB::ErrorCodes +{ +extern const int LIMIT_EXCEEDED; +} + +namespace +{ + +/// The limit `LimitedReadPacket` applies, from IMySQLReadPacket.cpp. +constexpr size_t PACKET_PAYLOAD_LIMIT = 10000; + +/// Serves its data a few bytes at a time. The limiter only reaches its `expect_eof` check when it +/// has to refill, which a fully buffered source never makes it do. +class ChunkedSource : public ReadBuffer +{ +public: + ChunkedSource(String data_, size_t chunk_) : ReadBuffer(nullptr, 0), data(std::move(data_)), chunk(chunk_) {} + +private: + bool nextImpl() override + { + if (consumed >= data.size()) + return false; + + const size_t count = std::min(chunk, data.size() - consumed); + BufferBase::set(data.data() + consumed, count, 0); + consumed += count; + return true; + } + + String data; + size_t chunk; + size_t consumed = 0; +}; + +struct PayloadCollector : public LimitedReadPacket +{ + String payload; + + void readPayloadImpl(ReadBuffer & buf) override { readStringUntilEOF(payload, buf); } +}; + +String packet(const String & payload, uint8_t sequence_id) +{ + String result; + const size_t length = payload.size(); + result += static_cast(length & 0xFF); + result += static_cast((length >> 8) & 0xFF); + result += static_cast((length >> 16) & 0xFF); + result += static_cast(sequence_id); + result += payload; + return result; +} + +} + +/// The limit counts one packet's payload, not the connection. A payload exactly at the limit is +/// allowed even though the 4-byte header pushes the connection past it, and the next packet is not +/// mistaken for overflow. +TEST(MySQLLimitedReadPacket, PayloadAtTheLimitFollowedByAnotherPacket) +{ + const String first(PACKET_PAYLOAD_LIMIT, 'a'); + ChunkedSource in(packet(first, 0) + packet("second", 1), 64); + + uint8_t sequence_id = 0; + PayloadCollector collector; + collector.readPayload(in, sequence_id); + + EXPECT_EQ(collector.payload, first); + EXPECT_EQ(sequence_id, 1); + + PayloadCollector next; + next.readPayload(in, sequence_id); + EXPECT_EQ(next.payload, "second"); +} + +TEST(MySQLLimitedReadPacket, PayloadOverTheLimitIsRejected) +{ + ChunkedSource in(packet(String(PACKET_PAYLOAD_LIMIT + 1, 'a'), 0), 64); + + uint8_t sequence_id = 0; + PayloadCollector collector; + try + { + collector.readPayload(in, sequence_id); + FAIL() << "An oversized payload was accepted"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::LIMIT_EXCEEDED); + } +} + +/// The unpacked variant is handed a payload that already ends where the packet ends. +TEST(MySQLLimitedReadPacket, UnpackedPayloadOverTheLimitIsRejected) +{ + ChunkedSource in(String(PACKET_PAYLOAD_LIMIT + 1, 'a'), 64); + + PayloadCollector collector; + try + { + collector.readPayloadWithUnpacked(in); + FAIL() << "An oversized unpacked payload was accepted"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::LIMIT_EXCEEDED); + } +} diff --git a/src/IO/LimitReadBuffer.cpp b/src/IO/LimitReadBuffer.cpp index 6f580b64f9aa1..aa727c5b57cda 100644 --- a/src/IO/LimitReadBuffer.cpp +++ b/src/IO/LimitReadBuffer.cpp @@ -20,15 +20,14 @@ bool LimitReadBuffer::nextImpl() /// Let underlying buffer calculate read bytes in `next()` call. in->position() = position(); - if (bytes >= settings.read_no_less) + if (bytes >= settings.read_no_more) { - if (settings.expect_eof && bytes > settings.read_no_more) + /// A stream ending exactly at the limit is not an error, so the check waits until the limit is + /// reached. `eof` refills the nested buffer, it consumes nothing. + if (settings.expect_eof && !in->eof()) throw Exception(ErrorCodes::LIMIT_EXCEEDED, "Limit for LimitReadBuffer exceeded: {}", settings.excetion_hint); - if (bytes >= settings.read_no_more) - return false; - - //throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected data, got {} bytes, expected {}", bytes, settings.read_atmost); + return false; } if (!in->next()) diff --git a/src/IO/LimitReadBuffer.h b/src/IO/LimitReadBuffer.h index fbcb4570e83d1..af4626a11cc2a 100644 --- a/src/IO/LimitReadBuffer.h +++ b/src/IO/LimitReadBuffer.h @@ -19,6 +19,9 @@ class LimitReadBuffer : public ReadBuffer { size_t read_no_less = 0; size_t read_no_more = std::numeric_limits::max(); + /// Throw instead of reporting EOF when the nested buffer still has data at the limit. Do not set + /// it when the nested buffer carries an unrelated message after this one, such as the next + /// keep-alive request: those bytes are not part of what the limit counts. bool expect_eof = false; std::string excetion_hint = {}; }; diff --git a/src/IO/tests/gtest_limit_read_buffer.cpp b/src/IO/tests/gtest_limit_read_buffer.cpp new file mode 100644 index 0000000000000..04dde06885f12 --- /dev/null +++ b/src/IO/tests/gtest_limit_read_buffer.cpp @@ -0,0 +1,162 @@ +#include + +#include +#include +#include +#include +#include + +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ +extern const int LIMIT_EXCEEDED; +extern const int CANNOT_READ_ALL_DATA; +} + +namespace +{ + +String readAll(ReadBuffer & in) +{ + String result; + WriteBufferFromString out(result); + copyData(in, out); + out.finalize(); + return result; +} + +int codeOfThrown(ReadBuffer & in) +{ + try + { + readAll(in); + } + catch (const Exception & e) + { + return e.code(); + } + return 0; +} + +} + +TEST(LimitReadBuffer, StreamEndingAtTheLimitIsNotAnError) +{ + ReadBufferFromString nested(std::string_view("0123456789")); + LimitReadBuffer limited(nested, {.read_no_more = 10, .expect_eof = true, .excetion_hint = "hint"}); + EXPECT_EQ(readAll(limited), "0123456789"); +} + +TEST(LimitReadBuffer, ExpectEofRejectsDataPastTheLimit) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + LimitReadBuffer limited(nested, {.read_no_more = 10, .expect_eof = true, .excetion_hint = "hint"}); + EXPECT_EQ(codeOfThrown(limited), ErrorCodes::LIMIT_EXCEEDED); +} + +TEST(LimitReadBuffer, ZeroLimitRejectsANonEmptyStream) +{ + ReadBufferFromString nested(std::string_view("a")); + LimitReadBuffer limited(nested, {.read_no_more = 0, .expect_eof = true, .excetion_hint = "hint"}); + EXPECT_EQ(codeOfThrown(limited), ErrorCodes::LIMIT_EXCEEDED); +} + +TEST(LimitReadBuffer, WithoutExpectEofDataPastTheLimitIsCutOff) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), "0123456789"); +} + +/// The check runs in `nextImpl`, so a consumer that reads exactly the limit and stops never asks for +/// the byte that would reveal the overflow. `expect_eof` is therefore best-effort, not a guarantee. +TEST(LimitReadBuffer, ExpectEofIsNotCheckedWhenTheConsumerStopsAtTheLimit) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + LimitReadBuffer limited(nested, {.read_no_more = 10, .expect_eof = true, .excetion_hint = "hint"}); + char buf[10] = {}; + limited.readStrict(buf, sizeof(buf)); + EXPECT_EQ(String(buf, sizeof(buf)), "0123456789"); +} + +TEST(LimitReadBuffer, ExpectEofShortStreamIsNotAnError) +{ + ReadBufferFromString nested(std::string_view("012")); + LimitReadBuffer limited(nested, {.read_no_more = 10, .expect_eof = true, .excetion_hint = "hint"}); + EXPECT_EQ(readAll(limited), "012"); +} + +TEST(LimitReadBuffer, StreamShorterThanReadNoLessIsAnError) +{ + ReadBufferFromString nested(std::string_view("012")); + LimitReadBuffer limited(nested, {.read_no_less = 10, .read_no_more = 10}); + EXPECT_EQ(codeOfThrown(limited), ErrorCodes::CANNOT_READ_ALL_DATA); +} + +TEST(LimitReadBuffer, WithoutExpectEofStreamEndingAtTheLimitIsFine) +{ + ReadBufferFromString nested(std::string_view("0123456789")); + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), "0123456789"); +} + +TEST(LimitReadBuffer, WithoutExpectEofShortStreamIsFine) +{ + ReadBufferFromString nested(std::string_view("012")); + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), "012"); +} + +TEST(LimitReadBuffer, WithoutExpectEofEmptyStreamIsFine) +{ + ReadBufferFromString nested(std::string_view("")); + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), ""); +} + +TEST(LimitReadBuffer, WithoutExpectEofZeroLimitReadsNothing) +{ + ReadBufferFromString nested(std::string_view("abc")); + { + LimitReadBuffer limited(nested, {.read_no_more = 0}); + EXPECT_EQ(readAll(limited), ""); + } + EXPECT_EQ(readAll(nested), "abc"); +} + +TEST(LimitReadBuffer, WithoutExpectEofNestedBufferContinuesPastTheLimit) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + { + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), "0123456789"); + } + EXPECT_EQ(readAll(nested), "abc"); +} + +/// The `Content-Length` shape of `HTTPServerRequest`. With keep-alive the nested buffer holds the next +/// request, so the bytes past the limit have to stay readable. +TEST(LimitReadBuffer, ExactLengthLeavesTheRestForTheNextReader) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + { + LimitReadBuffer limited(nested, {.read_no_less = 10, .read_no_more = 10}); + EXPECT_EQ(readAll(limited), "0123456789"); + } + EXPECT_EQ(readAll(nested), "abc"); +} + +TEST(LimitReadBuffer, PartialReadLeavesNestedBufferAtTheConsumedOffset) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + { + LimitReadBuffer limited(nested, {.read_no_more = 10}); + char buf[4] = {}; + limited.readStrict(buf, sizeof(buf)); + EXPECT_EQ(String(buf, sizeof(buf)), "0123"); + } + EXPECT_EQ(readAll(nested), "456789abc"); +} diff --git a/src/Server/HTTP/HTTPServerRequest.cpp b/src/Server/HTTP/HTTPServerRequest.cpp index c580ee955f4b9..a0cbe4e8a2c07 100644 --- a/src/Server/HTTP/HTTPServerRequest.cpp +++ b/src/Server/HTTP/HTTPServerRequest.cpp @@ -93,7 +93,9 @@ HTTPServerRequest::HTTPServerRequest(HTTPContextPtr context, HTTPServerResponse else if (hasContentLength()) { size_t content_length = getContentLength(); - stream = std::make_shared(std::move(in), LimitReadBuffer::Settings{.read_no_less = content_length, .read_no_more = content_length, .expect_eof = true}); + /// No `expect_eof`: `Content-Length` already defines where the body ends, and with keep-alive + /// the socket legitimately holds the bytes of the next request. + stream = std::make_shared(std::move(in), LimitReadBuffer::Settings{.read_no_less = content_length, .read_no_more = content_length}); stream_is_bounded = true; } else if (getMethod() != HTTPRequest::HTTP_GET && getMethod() != HTTPRequest::HTTP_HEAD && getMethod() != HTTPRequest::HTTP_DELETE) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 18d9711bf0c59..c681f104ec357 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1947,7 +1947,10 @@ bool TCPHandler::receiveProxyHeader() /// Only PROXYv1 is supported. /// Validation of protocol is not fully performed. - LimitReadBuffer limit_in(*in, {.read_no_more=107, .expect_eof=true}); /// Maximum length from the specs. + /// No `expect_eof`: except for the `UNKNOWN` health check below, the client sends its handshake + /// right after the header, so the connection does not end at the limit. An over-long header is + /// rejected anyway, by carrying no `\r\n` within these 107 bytes. + LimitReadBuffer limit_in(*in, {.read_no_more=107}); /// Maximum length from the specs. assertString("PROXY ", limit_in); diff --git a/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference new file mode 100644 index 0000000000000..a9796c468d97d --- /dev/null +++ b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference @@ -0,0 +1,3 @@ +the maximum size of multipart/form-data +30 +50 diff --git a/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh new file mode 100755 index 0000000000000..4d497382264e5 --- /dev/null +++ b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `http_max_multipart_form_data_size` cannot be set through URL parameters, so it comes from a user. +user="u_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} -q "CREATE USER $user IDENTIFIED WITH plaintext_password BY 'pw' SETTINGS http_max_multipart_form_data_size = 100" +${CLICKHOUSE_CLIENT} -q "GRANT CREATE TEMPORARY TABLE, SELECT ON *.* TO $user" + +url="${CLICKHOUSE_URL}&user=${user}&password=pw" +part="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_part.tsv" +python3 -c 'import sys; sys.stdout.write("x\n" * 30)' > "$part" + +# Two parts of 60 bytes: each one fits the 100 byte limit on its own, the form as a whole does not. +${CLICKHOUSE_CURL} -sS -F "a=@$part" -F "b=@$part" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&b_structure=s+String&a_format=TSV&b_format=TSV" \ + | grep -o -m1 'the maximum size of multipart/form-data' + +# One part of the same size stays under the limit. +${CLICKHOUSE_CURL} -sS -F "a=@$part" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&a_format=TSV" + +# A part that ends exactly at the limit is not over it. +exact="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_exact.tsv" +python3 -c 'import sys; sys.stdout.write("x\n" * 50)' > "$exact" +${CLICKHOUSE_CURL} -sS -F "a=@$exact" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&a_format=TSV" + +${CLICKHOUSE_CLIENT} -q "DROP USER $user" +rm -f "$part" "$exact" From 04596a5465e15351f23350e6245c416604099f35 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 09:37:47 +0000 Subject: [PATCH 096/296] Backport #116692 to 26.8: Prevent stale OpenSSL errors from breaking asynchronous TLS connections --- .../include/Poco/Net/SecureSocketImpl.h | 9 +- .../include/Poco/Net/SecureStreamSocketImpl.h | 9 + .../NetSSL_OpenSSL/src/SecureSocketImpl.cpp | 172 ++++++-- src/Common/tests/gtest_ssl_error_queue.cpp | 392 ++++++++++++++++++ .../tests/gtest_ssl_send_pending_data.cpp | 177 ++++++++ src/IO/SilkFiberStreamSocketImpl.h | 5 - src/IO/SilkSecureFiberStreamSocketImpl.cpp | 150 ++----- src/IO/SilkSecureFiberStreamSocketImpl.h | 5 - src/IO/SocketPeerClosed.cpp | 99 +++-- src/IO/SocketPeerClosed.h | 4 +- .../tests/gtest_silk_fiber_stream_socket.cpp | 34 +- 11 files changed, 819 insertions(+), 237 deletions(-) create mode 100644 src/Common/tests/gtest_ssl_error_queue.cpp create mode 100644 src/Common/tests/gtest_ssl_send_pending_data.cpp diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h index 475226652f5a7..82b38890b7917 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h @@ -149,6 +149,10 @@ namespace Net /// The object is normally guarded by the socket's mutex; a caller that uses it /// directly must ensure the socket is not accessed concurrently. + void markFatalError(); + /// Records that an external operation on the underlying `SSL` object failed fatally. + /// An orderly SSL shutdown must not be attempted afterwards. + X509 * peerCertificate() const; /// Returns the peer's certificate. @@ -233,7 +237,7 @@ namespace Net /// Returns true iff the given host name is the local host /// (either "localhost" or "127.0.0.1"). - bool mustRetry(int rc, Poco::Timespan & remaining_time); + bool mustRetry(int rc, int sslError, int socketError, Poco::Timespan & remaining_time); /// Returns true if the last operation should be retried, /// otherwise false. /// @@ -247,7 +251,7 @@ namespace Net /// not become readable or writable within the sockets /// receive or send timeout. - int handleError(int rc); + int handleError(int rc, int sslError, int socketError, unsigned long errorCode); /// Handles an SSL error by throwing an appropriate exception. void reset(); @@ -281,6 +285,7 @@ namespace Net Poco::AutoPtr _pSocket; Context::Ptr _pContext; bool _needHandshake; + bool _fatalError; std::string _peerHostName; Session::Ptr _pSession; const BIO_METHOD * _bioMethod = nullptr; diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h index a222ee01914c7..c5420492ec663 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h @@ -183,6 +183,9 @@ namespace Net /// Returns the underlying OpenSSL SSL object, or null if the SSL handshake /// has not been performed yet. + void markFatalError(); + /// Records that an external operation on the underlying `SSL` object failed fatally. + void setLazyHandshake(bool flag = true); /// Enable lazy SSL handshake. If enabled, the SSL handshake /// will be performed the first time date is sent or @@ -301,6 +304,12 @@ namespace Net } + inline void SecureStreamSocketImpl::markFatalError() + { + _impl.markFatalError(); + } + + inline Session::Ptr SecureStreamSocketImpl::currentSession() { return _impl.currentSession(); diff --git a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp index 60a55fec88bb8..3636d4b5fdd36 100644 --- a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp @@ -25,6 +25,7 @@ #include "Poco/NumberFormatter.h" #include "Poco/NumberParser.h" #include "Poco/Format.h" +#include #include #include @@ -60,11 +61,52 @@ struct RemainingTimeCounter Poco::Timestamp start; }; +struct SSLOperationResult +{ + int rc = 0; + int sslError = SSL_ERROR_NONE; + int socketError = 0; + unsigned long errorCode = 0; +}; + + +template +SSLOperationResult performSSLOperation(SSL * ssl, Operation && operation, bool zeroIsError = true) +{ + /// The error queue contract for TLS I/O and `SSL_get_error`: + /// https://docs.openssl.org/3.5/man3/SSL_get_error/ + /// The queue is cleared with `ERR_clear_error`: + /// https://docs.openssl.org/3.5/man3/ERR_clear_error/ + ERR_clear_error(); + + SSLOperationResult result; + /// `errno` is only meaningful if it was set by this operation. In particular, a custom `BIO` + /// can report `SSL_ERROR_SYSCALL` without changing it, so do not inherit another syscall result. + errno = 0; + result.rc = operation(); + + if (result.rc < 0 || (zeroIsError && result.rc == 0)) + { + /// Save `errno` before calling another function. `SSL_get_error` must be the first + /// OpenSSL call after the operation and must observe the queue created by that operation. + result.socketError = errno; + result.sslError = SSL_get_error(ssl, result.rc); + result.errorCode = ERR_get_error(); + } + + /// `errorCode` above preserves the diagnostic used by `handleError`. Do not leave any + /// additional entries in the thread-local queue for another connection on this thread. + ERR_clear_error(); + return result; +} + + SecureSocketImpl::SecureSocketImpl(Poco::AutoPtr pSocketImpl, Context::Ptr pContext): _pSSL(nullptr), _pSocket(pSocketImpl), _pContext(pContext), - _needHandshake(false) + _needHandshake(false), + _fatalError(false) { poco_check_ptr (_pSocket); poco_check_ptr (_pContext); @@ -111,6 +153,13 @@ void SecureSocketImpl::setMutex(std::unique_ptr mutex) } +void SecureSocketImpl::markFatalError() +{ + ScopedLock lock(*_mutex); + _fatalError = true; +} + + const BIO_METHOD * SecureSocketImpl::getBioMethod() const { return _bioMethod ? _bioMethod : BIO_s_socket(); @@ -121,6 +170,7 @@ void SecureSocketImpl::acceptSSL() { ScopedLock lock(*_mutex); poco_assert (!_pSSL); + _fatalError = false; BIO* pBIO = BIO_new(getBioMethod()); if (!pBIO) throw SSLException("Cannot create BIO object"); @@ -190,6 +240,7 @@ void SecureSocketImpl::connectSSL(bool performHandshake) ScopedLock lock(*_mutex); poco_assert (!_pSSL); poco_assert (_pSocket->initialized()); + _fatalError = false; BIO* pBIO = BIO_new(getBioMethod()); if (!pBIO) throw SSLException("Cannot create SSL BIO object"); @@ -218,31 +269,39 @@ void SecureSocketImpl::connectSSL(bool performHandshake) SSL_set_session(_pSSL, _pSession->sslSession()); } + SSL_set_connect_state(_pSSL); + _needHandshake = true; + try { if (performHandshake && _pSocket->getBlocking()) { - int ret; + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - ret = SSL_connect(_pSSL); + result = performSSLOperation(_pSSL, [this] + { + return SSL_connect(_pSSL); + }); + } + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + if (result.rc <= 0) + { + if (handleError(result.rc, result.sslError, result.socketError, result.errorCode) < 0) + throw Poco::TimeoutException("SSL handshake timed out"); + throw SSLConnectionUnexpectedlyClosedException(); } - while (mustRetry(ret, remaining_time)); - handleError(ret); + _needHandshake = false; verifyPeerCertificate(); } - else - { - SSL_set_connect_state(_pSSL); - _needHandshake = true; - } } catch (...) { SSL_free(_pSSL); _pSSL = 0; + _fatalError = false; throw; } } @@ -271,6 +330,16 @@ void SecureSocketImpl::shutdown() ScopedLock lock(*_mutex); if (_pSSL) { + if (_fatalError) + { + /// OpenSSL forbids `SSL_shutdown` after a fatal TLS or syscall error. + /// https://docs.openssl.org/3.5/man3/SSL_shutdown/ + /// Close the underlying transport without attempting an orderly TLS shutdown. + if (_pSocket->getBlocking()) + _pSocket->shutdown(); + return; + } + // Don't shut down the socket more than once. int shutdownState = SSL_get_shutdown(_pSSL); bool shutdownSent = (shutdownState & SSL_SENT_SHUTDOWN) == SSL_SENT_SHUTDOWN; @@ -283,8 +352,21 @@ void SecureSocketImpl::shutdown() // most web browsers, so we just set the shutdown // flag by calling SSL_shutdown() once and be // done with it. - int rc = SSL_shutdown(_pSSL); - if (rc < 0) handleError(rc); + /// A zero result is not an error for `SSL_shutdown` and must not be passed to + /// `SSL_get_error`; it means that `close_notify` was sent but not received yet. + SSLOperationResult result; + Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); + do + { + RemainingTimeCounter counter(remaining_time); + result = performSSLOperation(_pSSL, [this] + { + return SSL_shutdown(_pSSL); + }, false); + } + while (result.rc < 0 && mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + if (result.rc < 0) + handleError(result.rc, result.sslError, result.socketError, result.errorCode); if (_pSocket->getBlocking()) { _pSocket->shutdown(); @@ -328,24 +410,21 @@ int SecureSocketImpl::sendBytes(const void* buffer, int length, int flags) return rc; } + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - rc = SSL_write(_pSSL, buffer, length); + result = performSSLOperation(_pSSL, [this, buffer, length] + { + return SSL_write(_pSSL, buffer, length); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - // At this stage we still can have last not yet received SSL message containing SSL error - // so make a read to force SSL to process possible SSL error - if (SSL_get_error(_pSSL, rc) == SSL_ERROR_SYSCALL && SocketImpl::lastError() == POCO_ECONNRESET) - { - char c = 0; - SSL_read(_pSSL, &c, 1); - } - - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc == 0) throw SSLConnectionUnexpectedlyClosedException(); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL_write timed out"); @@ -379,6 +458,7 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags) return rc; } + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { @@ -386,12 +466,16 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags) /// so thread can be blocked on recv/send and epoll_wait several times /// until SSL_read will return rc > 0. Let's use our own time counter. RemainingTimeCounter counter(remaining_time); - rc = SSL_read(_pSSL, buffer, length); + result = performSSLOperation(_pSSL, [this, buffer, length] + { + return SSL_read(_pSSL, buffer, length); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL_read timed out"); return rc; @@ -419,16 +503,21 @@ int SecureSocketImpl::completeHandshake() poco_check_ptr (_pSSL); int rc; + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - rc = SSL_do_handshake(_pSSL); + result = performSSLOperation(_pSSL, [this] + { + return SSL_do_handshake(_pSSL); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL handshake timed out"); return rc; @@ -536,15 +625,13 @@ Poco::Timespan SecureSocketImpl::getMaxTimeoutOrLimit() return remaining_time; } -bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time) +bool SecureSocketImpl::mustRetry(int rc, int sslError, int socketError, Poco::Timespan& remaining_time) { if (remaining_time == 0) return false; ScopedLock lock(*_mutex); if (rc <= 0) { - int sslError = SSL_get_error(_pSSL, rc); - int socketError = _pSocket->lastError(); switch (sslError) { case SSL_ERROR_WANT_READ: @@ -571,21 +658,20 @@ bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time) case SSL_ERROR_SYSCALL: return socketError == POCO_EAGAIN || socketError == POCO_EINTR; default: - return socketError == POCO_EINTR; + /// `errno` is only meaningful for `SSL_ERROR_SYSCALL`; other errors must not + /// be retried because of a leftover `EINTR`, especially `SSL_ERROR_SSL`. + return false; } } return false; } -int SecureSocketImpl::handleError(int rc) +int SecureSocketImpl::handleError(int rc, int sslError, int error, unsigned long errorCode) { ScopedLock lock(*_mutex); if (rc > 0) return rc; - int sslError = SSL_get_error(_pSSL, rc); - int error = SocketImpl::lastError(); - switch (sslError) { case SSL_ERROR_ZERO_RETURN: @@ -598,18 +684,21 @@ int SecureSocketImpl::handleError(int rc) case SSL_ERROR_WANT_ACCEPT: case SSL_ERROR_WANT_X509_LOOKUP: // these should not occur + _fatalError = true; poco_bugcheck(); return rc; case SSL_ERROR_SYSCALL: + _fatalError = true; if (error != 0) { SocketImpl::error(error); } - // fallthrough + [[fallthrough]]; + case SSL_ERROR_SSL: default: { - long lastError = ERR_get_error(); - if (lastError == 0) + _fatalError = true; + if (errorCode == 0) { if (rc == 0) { @@ -631,7 +720,7 @@ int SecureSocketImpl::handleError(int rc) else { char buffer[256]; - ERR_error_string_n(lastError, buffer, sizeof(buffer)); + ERR_error_string_n(errorCode, buffer, sizeof(buffer)); std::string msg(buffer); throw SSLException(msg); } @@ -658,6 +747,7 @@ void SecureSocketImpl::reset() SSL_free(_pSSL); _pSSL = nullptr; } + _fatalError = false; } diff --git a/src/Common/tests/gtest_ssl_error_queue.cpp b/src/Common/tests/gtest_ssl_error_queue.cpp new file mode 100644 index 0000000000000..53e2a560ecf5e --- /dev/null +++ b/src/Common/tests/gtest_ssl_error_queue.cpp @@ -0,0 +1,392 @@ +#include "config.h" + +#if USE_SSL + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace +{ + +void leaveShutdownWhileInInit(Poco::Net::Context::Ptr context) +{ + ERR_clear_error(); + SSL * ssl = SSL_new(context->sslContext()); + ASSERT_NE(ssl, nullptr); + SSL_set_connect_state(ssl); + ASSERT_LT(SSL_shutdown(ssl), 0); + SSL_free(ssl); + + const auto error = ERR_peek_last_error(); + /// Unwrap the error manually because OpenSSL declares `ERR_GET_LIB` and + /// `ERR_GET_REASON` with `ossl_unused`, which causes compiler warnings when used. + ASSERT_EQ((error >> ERR_LIB_OFFSET) & ERR_LIB_MASK, ERR_LIB_SSL); + ASSERT_EQ(error & ERR_REASON_MASK, SSL_R_SHUTDOWN_WHILE_IN_INIT); +} + +int failReadWithoutErrno(BIO *, char *, int) +{ + /// Return a fatal `BIO` error without setting either `errno` or the OpenSSL error queue. + return -1; +} + +long failReadCtrl(BIO *, int command, long, void *) // NOLINT(google-runtime-int) +{ + if (command == BIO_CTRL_FLUSH) + return 1; + return 0; +} + +int failReadCreate(BIO * bio) +{ + BIO_set_init(bio, 1); + BIO_set_data(bio, nullptr); + return 1; +} + +int failReadDestroy(BIO *) +{ + return 1; +} + +const BIO_METHOD * failReadBioMethod() +{ + static const BIO_METHOD * method = [] + { + BIO_METHOD * result = BIO_meth_new(BIO_get_new_index() | BIO_TYPE_SOURCE_SINK, "fail-read-without-errno"); + BIO_meth_set_read(result, failReadWithoutErrno); + BIO_meth_set_ctrl(result, failReadCtrl); + BIO_meth_set_create(result, failReadCreate); + BIO_meth_set_destroy(result, failReadDestroy); + return result; + }(); + return method; +} + +int retryRead(BIO * bio, char *, int) +{ + BIO_clear_retry_flags(bio); + BIO_set_retry_read(bio); + return -1; +} + +int retryWrite(BIO * bio, const char *, int) +{ + BIO_clear_retry_flags(bio); + BIO_set_retry_write(bio); + return -1; +} + +long retryCtrl(BIO *, int command, long, void *) // NOLINT(google-runtime-int) +{ + if (command == BIO_CTRL_FLUSH) + return 1; + return 0; +} + +int retryCreate(BIO * bio) +{ + BIO_set_init(bio, 1); + BIO_set_data(bio, nullptr); + return 1; +} + +int retryDestroy(BIO *) +{ + return 1; +} + +const BIO_METHOD * retryBioMethod() +{ + static const BIO_METHOD * method = [] + { + BIO_METHOD * result = BIO_meth_new(BIO_get_new_index() | BIO_TYPE_SOURCE_SINK, "always-retry"); + BIO_meth_set_read(result, retryRead); + BIO_meth_set_write(result, retryWrite); + BIO_meth_set_ctrl(result, retryCtrl); + BIO_meth_set_create(result, retryCreate); + BIO_meth_set_destroy(result, retryDestroy); + return result; + }(); + return method; +} + +class ExhaustTimeoutStreamSocketImpl final : public Poco::Net::StreamSocketImpl +{ +public: + bool pollImpl(Poco::Timespan & timeout, int) override + { + timeout = 0; + return true; + } +}; + +class LiveTLSPair +{ +public: + LiveTLSPair() + : server_context(cert.makeContext(Poco::Net::Context::SERVER_USE)) + , client_context(cert.makeContext(Poco::Net::Context::CLIENT_USE)) + , listener(Poco::Net::SocketAddress("127.0.0.1", 0), 1, server_context) + , server_thread([this] + { + runServer(); + }) + { + try + { + client = std::make_unique( + Poco::Net::SocketAddress("127.0.0.1", listener.address().port()), client_context); + if (client->sendBytes("x", 1) != 1) + throw std::runtime_error("TLS client could not send the handshake byte"); + server_ready.wait(); + if (server_exception) + std::rethrow_exception(server_exception); + } + catch (...) + { + stop(); + throw; + } + } + + ~LiveTLSPair() + { + stop(); + } + + void sendMalformedRecord() + { + send_malformed_record.store(true, std::memory_order_release); + action_requested.count_down(); + action_was_requested = true; + action_done.wait(); + if (server_exception) + std::rethrow_exception(server_exception); + } + + Poco::Net::SecureStreamSocket & getClient() + { + return *client; + } + + Poco::Net::Context::Ptr getClientContext() const + { + return client_context; + } + +private: + void runServer() + { + try + { + Poco::Net::SecureStreamSocket peer(listener.acceptConnection()); + char byte = 0; + if (peer.receiveBytes(&byte, 1) != 1) + throw std::runtime_error("TLS server did not receive the handshake byte"); + + server_ready.count_down(); + action_requested.wait(); + + if (send_malformed_record.load(std::memory_order_acquire)) + { + /// A syntactically TLS-looking application-data record with an invalid payload. + /// It bypasses the server's `SSL` object and causes a fatal record-layer error on + /// the client while leaving the underlying TCP connection open. + constexpr unsigned char malformed_record[] = {0x17, 0x03, 0x03, 0x00, 0x01, 0xff}; + const ssize_t sent = ::send( + peer.impl()->sockfd(), malformed_record, sizeof(malformed_record), MSG_NOSIGNAL); + if (sent != static_cast(sizeof(malformed_record))) + throw std::runtime_error("TLS server could not send the malformed record"); + } + + action_done.count_down(); + finish.wait(); + peer.abort(); + } + catch (...) + { + server_exception = std::current_exception(); + server_ready.count_down(); + action_done.count_down(); + } + } + + void stop() noexcept + { + if (stopped) + return; + stopped = true; + + if (client) + { + try + { + client->abort(); + } + catch (...) + { + /// Ok: report the failure but continue cleanup so that the server thread is always joined. + ADD_FAILURE() << "Failed to abort the TLS client during cleanup"; + } + } + + if (!action_was_requested) + action_requested.count_down(); + finish.count_down(); + try + { + listener.close(); + } + catch (...) + { + /// Ok: report the failure but continue cleanup so that the server thread is always joined. + ADD_FAILURE() << "Failed to close the TLS listener during cleanup"; + } + if (server_thread.joinable()) + server_thread.join(); + } + + EphemeralCert cert; + Poco::Net::Context::Ptr server_context; + Poco::Net::Context::Ptr client_context; + Poco::Net::SecureServerSocket listener; + std::latch server_ready{1}; + std::latch action_requested{1}; + std::latch action_done{1}; + std::latch finish{1}; + std::atomic send_malformed_record{false}; + std::exception_ptr server_exception; + std::thread server_thread; + std::unique_ptr client; + bool action_was_requested = false; + bool stopped = false; +}; + +} + + +TEST(SSLErrorQueue, StaleErrorDoesNotChangeNonBlockingReadRetry) +{ + LiveTLSPair pair; + auto & client = pair.getClient(); + client.setBlocking(false); + + char byte = 0; + leaveShutdownWhileInInit(pair.getClientContext()); + EXPECT_EQ(client.receiveBytes(&byte, 1), Poco::Net::SecureStreamSocket::ERR_SSL_WANT_READ); + EXPECT_EQ(ERR_peek_error(), 0UL); + + /// The external async socket path retries `receiveBytes` after polling. The queue must be + /// cleared for every attempt, not only for the first operation on the connection. + leaveShutdownWhileInInit(pair.getClientContext()); + EXPECT_EQ(client.receiveBytes(&byte, 1), Poco::Net::SecureStreamSocket::ERR_SSL_WANT_READ); + EXPECT_EQ(ERR_peek_error(), 0UL); +} + + +TEST(SSLErrorQueue, StaleErrnoDoesNotRetryFatalBioError) +{ + LiveTLSPair pair; + auto & client = pair.getClient(); + client.setSendTimeout(Poco::Timespan(0, 10'000)); + client.setReceiveTimeout(Poco::Timespan(0, 10'000)); + + auto * client_impl = static_cast(client.impl()); + SSL * ssl = client_impl->ssl(); + ASSERT_NE(ssl, nullptr); + + /// Replace only the read `BIO`; `SSL_set0_rbio` transfers its ownership to `SSL`. + BIO * failing_read_bio = BIO_new(failReadBioMethod()); + ASSERT_NE(failing_read_bio, nullptr); + SSL_set0_rbio(ssl, failing_read_bio); + + /// The failing `BIO` deliberately leaves `errno` unchanged. A stale retriable value must not + /// make `SecureSocketImpl::mustRetry` repeat the operation or report an unrelated socket error. + errno = EINTR; + char byte = 0; + EXPECT_THROW(client.receiveBytes(&byte, 1), Poco::Net::SSLConnectionUnexpectedlyClosedException); + EXPECT_EQ(ERR_peek_error(), 0UL); +} + + +TEST(SSLErrorQueue, BlockingConnectDoesNotAcceptTimedOutHandshake) +{ + Poco::Net::ServerSocket listener(Poco::Net::SocketAddress("127.0.0.1", 0)); + EphemeralCert cert; + auto client_context = cert.makeContext(Poco::Net::Context::CLIENT_USE); + + Poco::AutoPtr client_impl + = new Poco::Net::SecureStreamSocketImpl(new ExhaustTimeoutStreamSocketImpl, client_context); + client_impl->setBioMethod(retryBioMethod()); + + /// The custom `BIO` keeps returning `WANT_READ` or `WANT_WRITE`. The first poll reports + /// readiness but consumes the entire timeout, so a blocking handshake must not be accepted. + EXPECT_THROW(client_impl->connect(listener.address()), Poco::TimeoutException); +} + + +TEST(SSLErrorQueue, FatalReadSkipsTLSShutdown) +{ + LiveTLSPair pair; + pair.sendMalformedRecord(); + + char byte = 0; + EXPECT_THROW(pair.getClient().receiveBytes(&byte, 1), Poco::Net::SSLException); + + /// `shutdown` must close the transport directly. Calling `SSL_shutdown` after the fatal read + /// would raise another SSL exception and could contaminate the thread's error queue. + EXPECT_NO_THROW(pair.getClient().shutdown()); + EXPECT_EQ(ERR_peek_error(), 0UL); +} + + +TEST(SSLErrorQueue, FatalPeekSkipsTLSShutdown) +{ + LiveTLSPair pair; + pair.sendMalformedRecord(); + + auto state = DB::SocketState::Idle; + /// `poll` can first wake for a TLS 1.3 post-handshake message. Repeat while `SSL_peek` + /// consumes only such control records, with a total timeout of five seconds. + for (int attempt = 0; attempt < 50 && state == DB::SocketState::Idle; ++attempt) + { + if (pair.getClient().poll(Poco::Timespan(0, 100'000), Poco::Net::Socket::SELECT_READ)) + state = DB::getSocketState(pair.getClient()); + } + EXPECT_EQ(state, DB::SocketState::Closed); + EXPECT_EQ(ERR_peek_error(), 0UL); + + /// The diagnostic `SSL_peek` saw a fatal TLS error, so `shutdown` must close the transport + /// directly instead of attempting the forbidden `SSL_shutdown` operation. + EXPECT_NO_THROW(pair.getClient().shutdown()); + EXPECT_EQ(ERR_peek_error(), 0UL); +} + +#endif diff --git a/src/Common/tests/gtest_ssl_send_pending_data.cpp b/src/Common/tests/gtest_ssl_send_pending_data.cpp new file mode 100644 index 0000000000000..20a26adc9613e --- /dev/null +++ b/src/Common/tests/gtest_ssl_send_pending_data.cpp @@ -0,0 +1,177 @@ +#include "config.h" + +#if USE_SSL + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace +{ + +/// This custom `BIO` makes every TLS write fail as if the underlying socket was reset. +/// It is installed as the write `BIO` only, so the real read `BIO` and buffered application data remain intact. +int resetWrite(BIO * bio, const char *, int) +{ + BIO_clear_retry_flags(bio); + errno = ECONNRESET; + return -1; +} + +long resetCtrl(BIO *, int command, long, void *) // NOLINT(google-runtime-int) +{ + if (command == BIO_CTRL_FLUSH) + return 1; + return 0; +} + +int resetCreate(BIO * bio) +{ + BIO_set_init(bio, 1); + BIO_set_data(bio, nullptr); + return 1; +} + +int resetDestroy(BIO *) +{ + return 1; +} + +const BIO_METHOD * resetWriteBioMethod() +{ + /// `SSL_set0_wbio` transfers ownership of the `BIO` to `SSL`, so the method must support its full lifetime. + /// A successful flush also lets OpenSSL perform routine `BIO` housekeeping without obscuring the write failure. + static const BIO_METHOD * method = [] + { + BIO_METHOD * result = BIO_meth_new(BIO_get_new_index() | BIO_TYPE_SOURCE_SINK, "reset-write"); + BIO_meth_set_write(result, resetWrite); + BIO_meth_set_ctrl(result, resetCtrl); + BIO_meth_set_create(result, resetCreate); + BIO_meth_set_destroy(result, resetDestroy); + return result; + }(); + return method; +} + +} + + +TEST(SSLSocketError, SendFailureDoesNotConsumePendingApplicationData) +{ + EphemeralCert cert; + auto server_context = cert.makeContext(Poco::Net::Context::SERVER_USE); + auto client_context = cert.makeContext(Poco::Net::Context::CLIENT_USE); + + Poco::Net::SecureServerSocket server_socket(Poco::Net::SocketAddress("127.0.0.1", 0), 1, server_context); + const Poco::Net::SocketAddress server_address("127.0.0.1", server_socket.address().port()); + const Poco::Timespan timeout(5, 0); + + /// Establish TCP before starting the server thread. The lazy TLS handshake is completed below, + /// concurrently with the first server-side write. + Poco::Net::SecureStreamSocket client(client_context); + client.setLazyHandshake(true); + client.connect(server_address); + client.setSendTimeout(timeout); + client.setReceiveTimeout(timeout); + + constexpr std::array payload{'p', 'e', 'n', 'd', 'i', 'n', 'g', '!'}; + std::exception_ptr server_exception; + std::jthread server_thread([&] + { + try + { + auto accepted = server_socket.acceptConnection(); + accepted.setSendTimeout(timeout); + accepted.setReceiveTimeout(timeout); + if (accepted.sendBytes(payload.data(), static_cast(payload.size())) != static_cast(payload.size())) + throw Poco::Net::NetException("Could not send the complete TLS test payload"); + } + catch (...) + { + server_exception = std::current_exception(); + } + }); + + auto * client_impl = static_cast(client.impl()); + ASSERT_EQ(client_impl->completeHandshake(), 1); + + SSL * ssl = client_impl->ssl(); + ASSERT_NE(ssl, nullptr); + + /// The failing write `BIO` installed below would also reject a TLS shutdown write. Mark shutdown as complete + /// during scope cleanup, while still letting `SSL` own and destroy the failing `BIO`. + SCOPE_EXIT(SSL_set_shutdown(ssl, SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN)); + + /// `SSL_peek` processes the incoming TLS record without consuming its first application-data byte. + /// Because the server sends the payload in one call, the complete payload must then be visible in `SSL_pending`. + char first_byte = 0; + const int peek_result = SSL_peek(ssl, &first_byte, 1); + + /// Wait until the server has completed its single write and propagate any server-side exception. + server_thread.join(); + if (server_exception) + std::rethrow_exception(server_exception); + + ASSERT_EQ(peek_result, 1); + EXPECT_EQ(first_byte, payload.front()); + const int pending_before_send = SSL_pending(ssl); + ASSERT_EQ(pending_before_send, static_cast(payload.size())); + + /// Replace only the write `BIO`. The read `BIO` and application data already buffered by `SSL` are preserved, + /// and `SSL_set0_wbio` transfers ownership of the failing `BIO` to `SSL`. + BIO * reset_write_bio = BIO_new(resetWriteBioMethod()); + ASSERT_NE(reset_write_bio, nullptr); + SSL_set0_wbio(ssl, reset_write_bio); + + /// OpenSSL requires the current thread's error queue to be empty before TLS I/O so that `SSL_get_error` + /// classifies this `SSL_write` result rather than an earlier error. Check the exact exception type because + /// the removed workaround also changed a connection reset into an `SSLException`. + ERR_clear_error(); + errno = 0; + const char byte_to_send = 'x'; + try + { + client.sendBytes(&byte_to_send, 1); + ADD_FAILURE() << "Expected Poco::Net::ConnectionResetException"; + } + catch (const Poco::Net::ConnectionResetException & exception) + { + EXPECT_EQ(typeid(exception), typeid(Poco::Net::ConnectionResetException)); + } + catch (const Poco::Exception & exception) + { + ADD_FAILURE() << "Expected Poco::Net::ConnectionResetException, got " << exception.className(); + } + catch (...) + { + /// Ok: this catch turns a non-Poco exception into a test failure. + ADD_FAILURE() << "Expected Poco::Net::ConnectionResetException, got a non-Poco exception"; + } + + /// Previously `SecureSocketImpl::sendBytes` called `SSL_read` after this write failure, silently consuming + /// one pending application-data byte. Handling the write error must leave all incoming data untouched. + EXPECT_EQ(SSL_pending(ssl), pending_before_send); +} + + +#endif diff --git a/src/IO/SilkFiberStreamSocketImpl.h b/src/IO/SilkFiberStreamSocketImpl.h index efbeaf042de2c..b6662336e4efc 100644 --- a/src/IO/SilkFiberStreamSocketImpl.h +++ b/src/IO/SilkFiberStreamSocketImpl.h @@ -24,11 +24,6 @@ class FiberStreamSocketImpl final : public Poco::Net::StreamSocketImpl void setBlocking(bool flag) override; bool supportsExternalPolling() const override { return false; } - bool getDontWait() const { return dont_wait; } - void setDontWait(bool flag) { dont_wait = flag; } - -private: - bool dont_wait = false; }; } diff --git a/src/IO/SilkSecureFiberStreamSocketImpl.cpp b/src/IO/SilkSecureFiberStreamSocketImpl.cpp index 871baaefc896b..78416e37310a0 100644 --- a/src/IO/SilkSecureFiberStreamSocketImpl.cpp +++ b/src/IO/SilkSecureFiberStreamSocketImpl.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -16,12 +15,10 @@ #include #include #include -#include #include #include #include -#include namespace Silk @@ -30,11 +27,6 @@ namespace Silk namespace { -uint64_t timeoutNs(const Poco::Timespan & timeout) -{ - return static_cast(timeout.totalMicroseconds()) * 1000ULL; -} - FiberStreamSocketImpl * getUnderlyingSocket(BIO * bio) { return static_cast(static_cast(BIO_get_data(bio))); @@ -45,68 +37,30 @@ int silkBioRead(BIO * bio, char * buf, int len) auto * socket_impl = getUnderlyingSocket(bio); const int fd = socket_impl->sockfd(); - if (socket_impl->getDontWait()) - { - /// Honor non-blocking mode instead of parking the caller on an io_uring read: a - /// non-blocking caller (e.g. `SSL_peek` from the pool's staleness probe) expects an - /// immediate `EAGAIN`/`SSL_ERROR_WANT_READ`, not a wait for `getReceiveTimeout()`. - BIO_clear_retry_flags(bio); - const ssize_t n = ::recv(fd, buf, len, MSG_DONTWAIT); - if (n < 0) - { - /// `recv` left the error in `errno`; the flag call below cannot touch it - /// (`BIO_set_retry_read` is a pure in-struct flag set), so it stays valid for the - /// caller through `return`. Read it into a local for the fatality test per the - /// errno rule in silk's docs/tls.md. - const int err = errno; - if (BIO_sock_non_fatal_error(err)) - BIO_set_retry_read(bio); - return -1; - } - - /// TODO(mstetsyuk): should be done at Silk level. - __msan_unpoison(buf, static_cast(n)); - - if (n == 0) - BIO_set_flags(bio, BIO_FLAGS_IN_EOF); - return static_cast(n); - } - - const uint64_t timeout_ns = timeoutNs(socket_impl->getReceiveTimeout()); - - uint64_t bytes_read = 0; - silk::FiberScheduler::IoFuture future; - iovec iov{buf, static_cast(len)}; - silk::FiberScheduler::read(fd, &iov, 1, 0, &bytes_read, &future); - - int r = timeout_ns > 0 - ? silk::FiberFuture::waitWithTimeout(&future, timeout_ns) - : future.wait(); - - if (r == ETIMEDOUT) - { - future.cancel(); - r = future.wait(); - if (r == ECANCELED) - r = ETIMEDOUT; - } - + /// Do not schedule a fiber-aware read or wait for its future here. This callback runs + /// inside an OpenSSL operation, and suspending the fiber could resume it on another + /// OS thread before `SSL_get_error` reads that thread's error queue. Use a non-blocking + /// syscall and report retry through the `BIO` flags instead. `SecureSocketImpl` saves + /// the OpenSSL result first, then `SecureSocketImpl::mustRetry` performs the timed, + /// fiber-aware wait through `pollImpl`. + /// https://docs.openssl.org/3.5/man3/SSL_get_error/ BIO_clear_retry_flags(bio); - - if (r == 0) + const ssize_t n = ::recv(fd, buf, len, MSG_DONTWAIT); + if (n < 0) { - /// TODO(mstetsyuk): should be done at Silk level. - __msan_unpoison(buf, bytes_read); - - if (bytes_read == 0) - BIO_set_flags(bio, BIO_FLAGS_IN_EOF); - return static_cast(bytes_read); + /// Capture `errno` immediately; fiber migration makes a later read invalid. + const int err = errno; + if (BIO_sock_non_fatal_error(err)) + BIO_set_retry_read(bio); + return -1; } - errno = r; - if (BIO_sock_non_fatal_error(r) || r == ETIMEDOUT) - BIO_set_retry_read(bio); - return -1; + /// TODO(mstetsyuk): should be done at Silk level. + __msan_unpoison(buf, static_cast(n)); + + if (n == 0) + BIO_set_flags(bio, BIO_FLAGS_IN_EOF); + return static_cast(n); } int silkBioWrite(BIO * bio, const char * buf, int len) @@ -114,52 +68,17 @@ int silkBioWrite(BIO * bio, const char * buf, int len) auto * socket_impl = getUnderlyingSocket(bio); const int fd = socket_impl->sockfd(); - if (socket_impl->getDontWait()) - { - /// See the matching branch in silkBioRead: honor non-blocking mode rather than parking - /// the caller in a fiber wait for getSendTimeout(). - BIO_clear_retry_flags(bio); - const ssize_t n = ::send(fd, buf, len, MSG_DONTWAIT | MSG_NOSIGNAL); - if (n < 0) - { - /// See the matching note in silkBioRead: `send` left the error in `errno`, the flag - /// call cannot clobber it, so it survives to the caller. - const int err = errno; - if (BIO_sock_non_fatal_error(err)) - BIO_set_retry_write(bio); - return -1; - } - return static_cast(n); - } - - const uint64_t timeout_ns = timeoutNs(socket_impl->getSendTimeout()); - - uint64_t bytes_written = 0; - silk::FiberScheduler::IoFuture future; - iovec iov{const_cast(buf), static_cast(len)}; - silk::FiberScheduler::write(fd, &iov, 1, 0, &bytes_written, &future); - - int r = timeout_ns > 0 - ? silk::FiberFuture::waitWithTimeout(&future, timeout_ns) - : future.wait(); - - if (r == ETIMEDOUT) + /// See `silkBioRead`: never suspend from inside an OpenSSL operation. + BIO_clear_retry_flags(bio); + const ssize_t n = ::send(fd, buf, len, MSG_DONTWAIT | MSG_NOSIGNAL); + if (n < 0) { - future.cancel(); - r = future.wait(); - if (r == ECANCELED) - r = ETIMEDOUT; + const int err = errno; + if (BIO_sock_non_fatal_error(err)) + BIO_set_retry_write(bio); + return -1; } - - BIO_clear_retry_flags(bio); - - if (r == 0) - return static_cast(bytes_written); - - errno = r; - if (BIO_sock_non_fatal_error(r) || r == ETIMEDOUT) - BIO_set_retry_write(bio); - return -1; + return static_cast(n); } long silkBioCtrl(BIO * bio, int cmd, [[maybe_unused]] long larg, void * parg) // NOLINT(google-runtime-int) @@ -259,22 +178,11 @@ SecureFiberStreamSocketImpl::SecureFiberStreamSocketImpl(Poco::Net::Context::Ptr SecureFiberStreamSocketImpl::SecureFiberStreamSocketImpl(FiberStreamSocketImpl * underlying_, Poco::Net::Context::Ptr context) : Poco::Net::SecureStreamSocketImpl(underlying_, context) - , underlying(underlying_) { setBioMethod(silkBioMethod()); setMutex(std::make_unique()); } -bool SecureFiberStreamSocketImpl::getDontWait() const -{ - return underlying->getDontWait(); -} - -void SecureFiberStreamSocketImpl::setDontWait(bool flag) -{ - underlying->setDontWait(flag); -} - bool SecureFiberStreamSocketImpl::pollImpl(Poco::Timespan & timeout, int mode) { uint32_t events = 0; diff --git a/src/IO/SilkSecureFiberStreamSocketImpl.h b/src/IO/SilkSecureFiberStreamSocketImpl.h index ccf8383413a08..1afd28d4a5746 100644 --- a/src/IO/SilkSecureFiberStreamSocketImpl.h +++ b/src/IO/SilkSecureFiberStreamSocketImpl.h @@ -17,16 +17,11 @@ class SecureFiberStreamSocketImpl final : public Poco::Net::SecureStreamSocketIm public: explicit SecureFiberStreamSocketImpl(Poco::Net::Context::Ptr context); - bool getDontWait() const; - void setDontWait(bool flag); - bool pollImpl(Poco::Timespan & timeout, int mode) override; bool supportsExternalPolling() const override { return false; } private: SecureFiberStreamSocketImpl(FiberStreamSocketImpl * underlying_, Poco::Net::Context::Ptr context); - - FiberStreamSocketImpl * underlying; }; } diff --git a/src/IO/SocketPeerClosed.cpp b/src/IO/SocketPeerClosed.cpp index c82b138b7ede2..af0017fa7494f 100644 --- a/src/IO/SocketPeerClosed.cpp +++ b/src/IO/SocketPeerClosed.cpp @@ -44,7 +44,16 @@ SocketState getSocketState(int fd) #if USE_SSL -SocketState getSSLSocketState(ssl_st * ssl) +namespace +{ + +struct SSLSocketStateResult +{ + SocketState state; + bool fatal_error; +}; + +SSLSocketStateResult getSSLSocketStateImpl(ssl_st * ssl) { /// `SSL_peek` decrypts just enough of the pending records to tell real application data and /// harmless post-handshake messages (session tickets, `KeyUpdate`) apart from a `close_notify`. @@ -54,32 +63,47 @@ SocketState getSSLSocketState(ssl_st * ssl) ERR_clear_error(); char c = 0; int res = SSL_peek(ssl, &c, 1); + SSLSocketStateResult result{SocketState::Closed, false}; if (res > 0) - return SocketState::DataPending; /// Application data is waiting to be read; the peer is alive. - - switch (SSL_get_error(ssl, res)) { - case SSL_ERROR_WANT_READ: [[fallthrough]]; - case SSL_ERROR_WANT_WRITE: - /// `SSL_peek` found no complete application-data record, but that alone does not prove - /// the connection is idle: the bytes of a record that has only partially arrived (e.g. - /// the first fragment of a queued response) are buffered inside the SSL object too, and - /// look identical from here - both end in `SSL_ERROR_WANT_READ`. `SSL_has_pending` - /// reports on that internal buffer regardless of whether the record is complete, so a - /// session ticket / `KeyUpdate` that was fully consumed reads as idle (nothing left - /// buffered), while a partial record correctly reads as pending. - return SSL_has_pending(ssl) ? SocketState::DataPending : SocketState::Idle; - case SSL_ERROR_ZERO_RETURN: - return SocketState::Closed; /// The peer sent `close_notify`: an orderly TLS shutdown. - default: - /// A FIN without `close_notify` (`SSL_ERROR_SYSCALL`), a protocol error (`SSL_ERROR_SSL`), - /// or anything else: treat as closed/broken. - return SocketState::Closed; + result = {SocketState::DataPending, false}; /// Application data is waiting to be read; the peer is alive. + } + else + { + switch (SSL_get_error(ssl, res)) + { + case SSL_ERROR_WANT_READ: [[fallthrough]]; + case SSL_ERROR_WANT_WRITE: + /// `SSL_peek` found no complete application-data record, but that alone does not prove + /// the connection is idle: the bytes of a record that has only partially arrived (e.g. + /// the first fragment of a queued response) are buffered inside the SSL object too, and + /// look identical from here - both end in `SSL_ERROR_WANT_READ`. `SSL_has_pending` + /// reports on that internal buffer regardless of whether the record is complete, so a + /// session ticket / `KeyUpdate` that was fully consumed reads as idle (nothing left + /// buffered), while a partial record correctly reads as pending. + result = {SSL_has_pending(ssl) ? SocketState::DataPending : SocketState::Idle, false}; + break; + case SSL_ERROR_ZERO_RETURN: + result = {SocketState::Closed, false}; /// The peer sent `close_notify`: an orderly TLS shutdown. + break; + case SSL_ERROR_SYSCALL: [[fallthrough]]; + case SSL_ERROR_SSL: + /// A FIN without `close_notify` or a protocol error is fatal. OpenSSL forbids + /// `SSL_shutdown` afterwards. + result = {SocketState::Closed, true}; + break; + default: + /// Any other unexpected result is treated as closed/broken, but only + /// `SSL_ERROR_SYSCALL` and `SSL_ERROR_SSL` make the connection fatal. + result = {SocketState::Closed, false}; + break; + } } -} -namespace -{ + /// Do not leak errors from this diagnostic probe into subsequent operations on this thread. + ERR_clear_error(); + return result; +} /// Force the socket into non-blocking mode for the duration of a call, restoring the original /// mode afterwards, so that `SSL_peek` on an idle pooled connection can never block. @@ -90,13 +114,10 @@ class ScopedNonBlocking : socket_impl(socket_impl_) { #if USE_SILK - if (auto * fiber_socket_impl = dynamic_cast(&socket_impl)) - { - was_blocking = !fiber_socket_impl->getDontWait(); - if (was_blocking) - fiber_socket_impl->setDontWait(true); + /// The Silk TLS BIO is always non-blocking so that an OpenSSL operation cannot + /// suspend and migrate between the operation and `SSL_get_error`. + if (dynamic_cast(&socket_impl)) return; - } #endif was_blocking = socket_impl.getBlocking(); if (was_blocking) @@ -110,13 +131,6 @@ class ScopedNonBlocking try { -#if USE_SILK - if (auto * fiber_socket_impl = dynamic_cast(&socket_impl)) - { - fiber_socket_impl->setDontWait(false); - return; - } -#endif socket_impl.setBlocking(true); } catch (...) @@ -130,13 +144,17 @@ class ScopedNonBlocking private: Poco::Net::SocketImpl & socket_impl; - /// For regular (non-silk) socket: whether it was blocking before. - /// For silk socket: whether it was dont-wait before. + /// Whether a regular (non-Silk) socket was blocking before the probe. bool was_blocking = false; }; } +SocketState getSSLSocketState(ssl_st * ssl) +{ + return getSSLSocketStateImpl(ssl).state; +} + #endif SocketState getSocketState(const Poco::Net::StreamSocket & socket) @@ -149,7 +167,10 @@ SocketState getSocketState(const Poco::Net::StreamSocket & socket) if (auto * ssl = secure->ssl()) { ScopedNonBlocking non_blocking(*secure); - return getSSLSocketState(ssl); + auto result = getSSLSocketStateImpl(ssl); + if (result.fatal_error) + secure->markFatalError(); + return result.state; } } #endif diff --git a/src/IO/SocketPeerClosed.h b/src/IO/SocketPeerClosed.h index 0d50cc64e3073..fb7ad792164f1 100644 --- a/src/IO/SocketPeerClosed.h +++ b/src/IO/SocketPeerClosed.h @@ -52,8 +52,8 @@ SocketState getSocketState(const Poco::Net::StreamSocket & socket); #if USE_SSL /// TLS-aware core of the check, exposed for testing. The socket underlying `ssl` MUST be in -/// non-blocking mode - don't-wait mode for a fiber socket - (the caller guarantees this) so that -/// `SSL_peek` cannot block. +/// non-blocking mode (the caller guarantees this) so that `SSL_peek` cannot block. The Silk TLS +/// BIO is always non-blocking. SocketState getSSLSocketState(ssl_st * ssl); #endif diff --git a/src/IO/tests/gtest_silk_fiber_stream_socket.cpp b/src/IO/tests/gtest_silk_fiber_stream_socket.cpp index 3afc0f233ca43..79fc9dafca47b 100644 --- a/src/IO/tests/gtest_silk_fiber_stream_socket.cpp +++ b/src/IO/tests/gtest_silk_fiber_stream_socket.cpp @@ -310,8 +310,7 @@ TYPED_TEST(SilkFiberSocketTest, ThrottlerLimitEnforced) } -/// Secure-only: the bug is TLS-specific (a blocking-only `silkBioRead`/`silkBioWrite` surfaces -/// through `SSL_peek`, not through a raw, BIO-less socket read). Reuses the +/// Secure-only tests for the TLS BIO and direct OpenSSL operations. Reuses the /// `SecurePolicy policy` member from the typed fixture rather than redeclaring it. using SilkFiberSecureSocketTest = SilkFiberSocketTest; @@ -345,15 +344,12 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingPeekDoesNotBlockOnIdleConnection) char pong[1] = {}; EXPECT_EQ(socket.receiveBytes(pong, sizeof(pong)), 1); - /// A long receive timeout. Pre-fix, `silkBioRead` has no non-blocking mode and always - /// parks the caller in a fiber wait up to this timeout, so a slow probe below proves - /// the bug; a fast one proves the fix. + /// A long receive timeout. The probe must remain non-blocking regardless of it. socket.setReceiveTimeout(Poco::Timespan(5, 0)); /// The actual production sequence (`DB::getSocketState(StreamSocket)`, the core of the - /// connection pool's staleness check in `HTTPConnectionPool.cpp`): it puts the silk - /// socket into don't-wait mode with `setDontWait` - `Socket::setBlocking(false)` is rejected - /// by silk sockets - and calls `SSL_peek`, which reaches OpenSSL's socket BIO, i.e. `silkBioRead`. + /// connection pool's staleness check in `HTTPConnectionPool.cpp`) calls `SSL_peek`, + /// which reaches the always non-blocking Silk TLS BIO. Stopwatch watch; *p->state = DB::getSocketState(socket); *p->elapsed_us = watch.elapsedMicroseconds(); @@ -378,17 +374,13 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingPeekDoesNotBlockOnIdleConnection) EXPECT_EQ(state, DB::SocketState::Idle); EXPECT_LT(elapsed_us, 500'000U) << "getSocketState() took " << elapsed_us - << "us: silkBioRead ignored non-blocking mode and blocked on the receive timeout instead " - "of returning EAGAIN immediately"; + << "us: the Silk TLS BIO blocked on the receive timeout instead of returning EAGAIN immediately"; } -/// The same bug at the raw level, without any ClickHouse helper: a plain `SSL_peek` on a -/// non-blocking TLS connection with no data pending must return `SSL_ERROR_WANT_READ` -/// immediately. This is exactly how a non-blocking consumer uses the socket - and the only way, -/// since silk sockets reject `Socket::setBlocking(false)`: the socket is put into don't-wait -/// mode with `setDontWait` and only `silkBioRead` ever observes it. Pre-fix, the BIO has no -/// non-blocking mode and parks the caller for the full receive timeout. +/// At the raw level, without a ClickHouse helper, a plain `SSL_peek` on an idle TLS connection +/// must return `SSL_ERROR_WANT_READ` immediately. Silk sockets reject `Socket::setBlocking(false)`, +/// so the BIO itself has to be non-blocking. TEST_F(SilkFiberSecureSocketTest, NonBlockingSslPeekReturnsWantReadImmediately) { auto listener = policy.makeListener(); @@ -419,11 +411,10 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingSslPeekReturnsWantReadImmediately) socket.setReceiveTimeout(Poco::Timespan(5, 0)); - /// Put the socket into don't-wait mode - what any non-blocking user must do here, - /// since `Socket::setBlocking(false)` throws on a silk socket. + /// The Silk TLS BIO is always non-blocking. OpenSSL operations return WANT_READ or + /// WANT_WRITE and `SecureSocketImpl` performs the fiber-aware wait outside OpenSSL. auto * secure = dynamic_cast(socket.impl()); SSL * ssl = secure->ssl(); - secure->setDontWait(true); char c = 0; ERR_clear_error(); @@ -432,7 +423,6 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingSslPeekReturnsWantReadImmediately) *p->ssl_error = SSL_get_error(ssl, res); *p->elapsed_us = watch.elapsedMicroseconds(); - secure->setDontWait(false); socket.close(); return 0; }, @@ -452,8 +442,8 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingSslPeekReturnsWantReadImmediately) EXPECT_EQ(ssl_error, SSL_ERROR_WANT_READ); EXPECT_LT(elapsed_us, 500'000U) << "SSL_peek() took " << elapsed_us - << "us on an idle non-blocking connection: silkBioRead ignored non-blocking mode and " - "blocked on the receive timeout instead of returning EAGAIN immediately"; + << "us on an idle connection: the Silk TLS BIO blocked on the receive timeout instead " + "of returning EAGAIN immediately"; } From 931083f201549c9a79eaa19446e1e0e7a86387ec Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 10:57:59 +0000 Subject: [PATCH 097/296] Backport #117429 to 26.8: Mask secrets in SHOW CREATE DATABASE of a Backup database --- src/Backups/BackupInfo.cpp | 6 +- src/Backups/BackupMetadataFinder.cpp | 8 +- src/Backups/registerBackupEngineS3.cpp | 15 +- src/Databases/DatabaseBackup.cpp | 17 +- src/Parsers/FunctionSecretArgumentsFinder.cpp | 264 +++++++++++++++++- src/Parsers/FunctionSecretArgumentsFinder.h | 21 ++ .../test_mask_sensitive_info/test.py | 56 ++++ ...3_explicit_url_named_secret_mask.reference | 56 +++- ...4510_s3_explicit_url_named_secret_mask.sql | 207 +++++++++++++- ...abase_backup_locator_secret_mask.reference | 14 + ...233_database_backup_locator_secret_mask.sh | 58 ++++ ...duplicate_definition_secret_mask.reference | 34 +++ ...estore_duplicate_definition_secret_mask.sh | 164 +++++++++++ 13 files changed, 889 insertions(+), 31 deletions(-) create mode 100644 tests/queries/0_stateless/05233_database_backup_locator_secret_mask.reference create mode 100755 tests/queries/0_stateless/05233_database_backup_locator_secret_mask.sh create mode 100644 tests/queries/0_stateless/05234_restore_duplicate_definition_secret_mask.reference create mode 100755 tests/queries/0_stateless/05234_restore_duplicate_definition_secret_mask.sh diff --git a/src/Backups/BackupInfo.cpp b/src/Backups/BackupInfo.cpp index 0357ab7859816..755027d190753 100644 --- a/src/Backups/BackupInfo.cpp +++ b/src/Backups/BackupInfo.cpp @@ -286,7 +286,7 @@ BackupInfo BackupInfo::fromAST(const IAST & ast) { const auto * func = ast.as(); if (!func) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected function, got {}", ast.formatForErrorMessage()); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected a function as the backup locator"); BackupInfo res; res.backup_engine_name = func->name; @@ -295,7 +295,7 @@ BackupInfo BackupInfo::fromAST(const IAST & ast) { const auto * list = func->arguments->as(); if (!list) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected list, got {}", func->arguments->formatForErrorMessage()); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected an argument list in the backup locator"); size_t index = 0; if (!list->children.empty()) @@ -329,7 +329,7 @@ BackupInfo BackupInfo::fromAST(const IAST & ast) res.function_arg = elem; break; } - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected literal, got {}", elem->formatForErrorMessage()); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected a literal argument in the backup locator"); } res.args.push_back(lit->value); } diff --git a/src/Backups/BackupMetadataFinder.cpp b/src/Backups/BackupMetadataFinder.cpp index c9b6fee3f362f..cfc3c257784de 100644 --- a/src/Backups/BackupMetadataFinder.cpp +++ b/src/Backups/BackupMetadataFinder.cpp @@ -304,8 +304,8 @@ void BackupMetadataFinder::findTableInBackupImpl( ErrorCodes::CANNOT_RESTORE_TABLE, "Extracted two different create queries for the same {}: {} and {}", tableNameWithTypeToString(table_name.database, table_name.table, false), - table_info.create_table_query_str, - create_table_query_str); + table_info.create_table_query->formatForErrorMessage(), + create_table_query->formatForErrorMessage()); } } @@ -395,8 +395,8 @@ void BackupMetadataFinder::findDatabaseInBackupImpl( ErrorCodes::CANNOT_RESTORE_DATABASE, "Extracted two different create queries for the same database {}: {} and {}", backQuoteIfNeed(database_name), - database_info.create_database_query_str, - create_database_query_str); + database_info.create_database_query->formatForErrorMessage(), + create_database_query->formatForErrorMessage()); } database_info.create_database_query = create_database_query; diff --git a/src/Backups/registerBackupEngineS3.cpp b/src/Backups/registerBackupEngineS3.cpp index 4f3f9b075dee7..feddeddd78879 100644 --- a/src/Backups/registerBackupEngineS3.cpp +++ b/src/Backups/registerBackupEngineS3.cpp @@ -304,8 +304,19 @@ void registerBackupEngineS3(BackupFactory & factory) /// copy: `BackupImpl::writeBackupMetadata` decides whether the base backup may take this /// backup's credentials by comparing the two locators as text, and a locator resolved on /// one side alone no longer matches the other written the same way. - if (!StorageS3Configuration::collectCredentials(params.backup_info.function_arg->clone(), auth_settings, params.context)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid argument: {}", params.backup_info.function_arg->formatForErrorMessage()); + /// The rejected argument is never echoed, so both failure modes report the same way: + /// a nested map like `headers(..)` is masked only by its enclosing formatter, so + /// formatting one alone prints its values, and evaluating one can throw quoting it. + try + { + if (!StorageS3Configuration::collectCredentials( + params.backup_info.function_arg->clone(), auth_settings, params.context)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 extra credentials"); + } + catch (const Exception &) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 extra credentials"); + } role_arn = std::move(auth_settings[S3AuthSetting::role_arn]); role_session_name = std::move(auth_settings[S3AuthSetting::role_session_name]); diff --git a/src/Databases/DatabaseBackup.cpp b/src/Databases/DatabaseBackup.cpp index c46a7a663b1f6..97d063e5fa30a 100644 --- a/src/Databases/DatabaseBackup.cpp +++ b/src/Databases/DatabaseBackup.cpp @@ -506,7 +506,16 @@ DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr, bool DatabaseBackup::Configuration result; - result.database_name = checkAndGetLiteralArgument(engine_args[0], "database_name"); + /// `checkAndGetLiteralArgument` formats the argument it rejects, and a locator written in this + /// position would format its credentials in plaintext. + try + { + result.database_name = checkAndGetLiteralArgument(engine_args[0], "database_name"); + } + catch (const Exception &) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument 'database_name' must be a string literal"); + } /** A locator held in a string literal (`Backup('db', 'File(\'backup.zip\')')`) is the form that * metadata rewritten by an older server carries, so it has to keep loading - a server that cannot @@ -525,9 +534,9 @@ DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr, bool } } - /// `BackupInfo::fromAST` puts the offending argument into its message, and that message reaches the - /// error log and the `exception` column of `query_log`. A locator held in a string literal is exactly - /// the text that can carry credentials, so refuse it here without echoing it. + /// A locator held in a string literal is exactly the text that can carry credentials, and a rejection + /// message reaches the error log and the `exception` column of `query_log`, so name the accepted form + /// here rather than the text given. if (!engine_args[1]->as()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected function as the backup destination of a `Backup` database. It must be spelled as the " diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 8a6138d23552e..95bfbc06b57e5 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -26,6 +26,89 @@ namespace changed |= maskPresignedURLParameters(url); return changed; } + + /// How an Azure destination reads a connection value, and whether the masking here can show it. + enum class AzureConnectionValue + { + /// An http(s) scheme and a host, with no userinfo, query or fragment: each of those carries a + /// credential of its own (`http://user:key@host`, a SAS `?sig=`). + PlainStorageAccountURL, + /// A connection string, whose secret keys `maskAzureConnectionString` masks in place. + ConnectionString, + /// A value that can carry a credential no rule here masks. + Unmaskable, + }; + + AzureConnectionValue classifyAzureConnectionValue(const String & value) + { + static constexpr std::string_view SEPARATOR = "://"; + const size_t separator = value.find(SEPARATOR); + const std::string_view scheme = std::string_view(value).substr(0, std::min(separator, value.length())); + /// The scheme grammar `maskURIUserinfo` reads. A connection string does not match it, even when + /// one of its values embeds an endpoint URL. + const bool is_url = separator != String::npos && !scheme.empty() && isAlphaASCII(scheme.front()) + && std::all_of( + scheme.begin(), scheme.end(), [](char c) { return isAlphaNumericASCII(c) || c == '+' || c == '.' || c == '-'; }); + /// `maskAzureConnectionString` masks nothing in a value starting with `http`, so one that is no + /// URL either would be left as written. + if (!is_url) + return value.starts_with("http") ? AzureConnectionValue::Unmaskable : AzureConnectionValue::ConnectionString; + + if ((!equalsCaseInsensitive(scheme, "http") && !equalsCaseInsensitive(scheme, "https")) + || value.find_first_of("?#") != String::npos) + return AzureConnectionValue::Unmaskable; + + const size_t authority_begin = separator + SEPARATOR.length(); + const size_t authority_end = std::min(value.find('/', authority_begin), value.length()); + if (authority_end == authority_begin || value.find('@', authority_begin) < authority_end) + return AzureConnectionValue::Unmaskable; + return AzureConnectionValue::PlainStorageAccountURL; + } + + /// The backup engines whose locator names a destination with no credential in it, each with the + /// argument count it accepts. `BackupFactory` registers exactly these plus `S3` and `AzureBlobStorage`. + std::optional credentialFreeBackupEngineArity(const String & engine_name) + { + if (engine_name == "File" || engine_name == "Memory") + return 1; + if (engine_name == "Disk") + return 2; + if (engine_name == "Null") + return 0; + return {}; + } +} + +void FunctionSecretArgumentsFinder::maskEveryArgument() +{ + for (size_t i = 0, size = function->arguments->size(); i < size; ++i) + markSecretArgument(i); +} + +bool FunctionSecretArgumentsFinder::hasOnlyLiteralArguments(const AbstractFunction & function) +{ + if (!function.hasArguments()) + return true; + for (size_t i = 0, size = function.arguments->size(); i < size; ++i) + if (!function.arguments->at(i)->tryGetLiteralText(nullptr)) + return false; + return true; +} + +bool FunctionSecretArgumentsFinder::isCredentialFreeBackupLocator(const AbstractFunction & function) +{ + auto arity = credentialFreeBackupEngineArity(function.name()); + if (!arity) + return false; + const size_t count = function.hasArguments() ? function.arguments->size() : 0; + if (count != *arity) + return false; + /// Each of these reads every argument of its own as a string, so another shape - an array among + /// them - is read by none of them and can carry a string of its own. + for (size_t i = 0; i < count; ++i) + if (!tryGetStringFromArgument(*function.arguments->at(i), nullptr, /* allow_identifier= */ false)) + return false; + return true; } void FunctionSecretArgumentsFinder::markSecretArgument(size_t index, bool argument_is_named) @@ -599,6 +682,54 @@ bool FunctionSecretArgumentsFinder::maskAzureConnectionString(ssize_t url_arg_id return false; } +bool FunctionSecretArgumentsFinder::azureCollectionArgumentsAreShowable(size_t start, size_t positional_limit) +{ + size_t positionals = 0; + for (size_t i = start, size = function->arguments->size(); i < size; ++i) + { + const auto argument_function = function->arguments->at(i)->getFunction(); + if (argument_function && argument_function->name() == "equals") + { + /// A key this rule cannot read hides which credential the override carries; a value that is + /// no plain literal or identifier can nest one (`headers('Authorization' = '...')`). + if (argument_function->arguments && argument_function->arguments->size() == 2 + && tryGetStringFromArgument(*argument_function->arguments->at(0), nullptr) + && (tryGetStringFromArgument(*argument_function->arguments->at(1), nullptr) + || argument_function->arguments->at(1)->tryGetLiteralText(nullptr))) + continue; + return false; + } + if (++positionals > positional_limit || !function->arguments->at(i)->tryGetLiteralText(nullptr)) + return false; + } + + /// A destination reads at most one of the two mutually exclusive connection keys, and rejects a + /// second one only after the statement has been formatted, so a surplus one stays as written. + size_t connection_overrides = 0; + for (const auto & key : {"connection_string", "storage_account_url"}) + for (ssize_t i = findNamedArgument(nullptr, key, start); i >= 0; + i = findNamedArgument(nullptr, key, static_cast(i) + 1)) + ++connection_overrides; + + if (connection_overrides > 1) + return false; + + for (const auto & key : {"connection_string", "storage_account_url"}) + { + String value; + if (findNamedArgument(&value, key, start) < 0) + continue; + /// Hiding a connection string replaces its whole argument, which cannot be combined with + /// hiding `account_key`. + const auto shape = classifyAzureConnectionValue(value); + if (value.empty() || shape == AzureConnectionValue::Unmaskable + || (shape == AzureConnectionValue::ConnectionString + && findNamedArgument(nullptr, "account_key", start) >= 0)) + return false; + } + return true; +} + void FunctionSecretArgumentsFinder::findURLSecretArguments(size_t url_offset) { /// `headers(...)` can appear at any position in every url form (function, cluster function, engine, @@ -966,31 +1097,52 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen if (isNamedCollectionName(url_arg_idx)) { /// AzureBlobStorage(named_collection, ..., account_key = 'account_key', ...) + if (!azureCollectionArgumentsAreShowable(url_arg_idx + 1, /* positional_limit= */ 0)) + { + maskEveryArgument(); + return; + } if (maskAzureConnectionString(-1, true, 1)) return; findSecretNamedArgument("account_key", 1); return; } - if (maskAzureConnectionString(url_arg_idx)) - return; - /// We should check other arguments first because we don't need to do any replacement in case of /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument size_t count = function->arguments->size(); + bool fourth_argument_is_format = false; if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) { String fourth_arg; if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) - { - if (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg)) - return; - } + fourth_argument_is_format = fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg); + } + /// Which argument holds a credential: the two-argument shape takes a shared access signature beside + /// the url (`endpoint.sas_auth`), the longer ones an `account_key` - unless the fourth names a format. + std::optional credential_arg_idx; + if (count == url_arg_idx + 2) + credential_arg_idx = url_arg_idx + 1; + else if (!fourth_argument_is_format && (url_arg_idx + 4 < count)) + credential_arg_idx = url_arg_idx + 4; + + /// The engine reads this argument as a connection string or as a plain account url; a value of + /// another shape is read by neither rule below, and a hidden connection string replaces it whole. + String connection_value; + const auto shape = tryGetStringFromArgument(url_arg_idx, &connection_value) + ? classifyAzureConnectionValue(connection_value) + : AzureConnectionValue::Unmaskable; + if (shape == AzureConnectionValue::Unmaskable || (shape == AzureConnectionValue::ConnectionString && credential_arg_idx)) + { + maskEveryArgument(); + return; } - /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature - if (url_arg_idx + 4 < count) - markSecretArgument(url_arg_idx + 4); + if (maskAzureConnectionString(url_arg_idx)) + return; + + if (credential_arg_idx) + markSecretArgument(*credential_arg_idx); } void FunctionSecretArgumentsFinder::findRedisFunctionSecretArguments() @@ -1175,8 +1327,13 @@ void FunctionSecretArgumentsFinder::findDataLakeCatalogSecretArguments() void FunctionSecretArgumentsFinder::findBackupDatabaseSecretArguments() { - if (function->arguments->size() < 2) + /// `Backup(database_name, locator)` is the only valid shape, a locator carrying credentials can be + /// written in either position, and the query is formatted for logging before validation rejects it. + if (function->arguments->size() != 2 || !function->arguments->at(0)->tryGetLiteralText(nullptr)) + { + maskEveryArgument(); return; + } auto storage_arg = function->arguments->at(1); auto storage_function = storage_arg->getFunction(); @@ -1200,7 +1357,26 @@ void FunctionSecretArgumentsFinder::findBackupDatabaseSecretArguments() /// Backup('', S3('url', 'access_key_id', 'secret_access_key' [, ...])) /// Backup('', S3(named_collection, ..., secret_access_key = '...', session_token = '...', ...)) /// by reconstructing the nested `S3(...)` with the secret arguments replaced by `[HIDDEN]`. - if (!storage_function || storage_function->name() != "S3" || !storage_function->hasArguments()) + if (storage_function->name() != "S3") + { + if (isCredentialFreeBackupLocator(*storage_function)) + return; + + /// Any other locator holds a credential no rule below reconstructs (`AzureBlobStorage` holds + /// `account_key` and connection-string material); its engine name and arity are not secrets. + std::string replacement = storage_function->name() + "("; + for (size_t i = 0, size = storage_function->hasArguments() ? storage_function->arguments->size() : 0; i < size; ++i) + replacement += i > 0 ? ", '[HIDDEN]'" : "'[HIDDEN]'"; + replacement += ")"; + + result.start = 1; + result.count = 1; + result.replacement = std::move(replacement); + result.quote_replacement = false; + return; + } + + if (!storage_function->hasArguments()) return; const auto & nested_args = *storage_function->arguments; @@ -1383,10 +1559,70 @@ void FunctionSecretArgumentsFinder::findBackupNameSecretArguments() maskS3UrlArgument(positional, 0); maskS3PositionalsFrom(positional, positional.size() == 3 ? 2 : 1); } - else if (engine_name == "AzureBlobStorage" || engine_name == "AzureQueue") + else if (engine_name == "AzureBlobStorage") { - findAzureBlobStorageTableEngineSecretArguments(); + findAzureBlobStorageBackupSecretArguments(); + } + else if (!isCredentialFreeBackupLocator(*function)) + { + /// Everything else either is an engine no rule here reconstructs, or has arguments the named + /// engine does not read (an override, a nested map, a surplus slot), which can carry a credential. + /// `AzureQueue` reaches this branch: it is a table engine, not a registered backup engine. + maskEveryArgument(); + } +} + +void FunctionSecretArgumentsFinder::findAzureBlobStorageBackupSecretArguments() +{ + /// The destination reads AzureBlobStorage(named_collection [, 'filename'] [, key = value, ...]), + /// ('connection_string|storage_account_url', 'container', 'path'), or those three followed by + /// ('account_name', 'account_key'). An argument no shape reads holds whatever was written in it. + const size_t count = function->arguments->size(); + + if (isNamedCollectionName(0)) + { + if (!azureCollectionArgumentsAreShowable(1, /* positional_limit= */ 1)) + { + maskEveryArgument(); + return; + } + if (maskAzureConnectionString(-1, /* argument_is_named= */ true, 1)) + return; + findSecretNamedArgument("account_key", 1); + return; + } + + if ((count != 3 && count != 5) || !hasOnlyLiteralArguments(*function)) + { + maskEveryArgument(); + return; } + + if (count == 3) + { + /// Only this shape accepts a connection string, which can embed `AccountKey`. A value that is no + /// string is read by neither the classification below nor the destination. + String connection_value; + if (!tryGetStringFromArgument(0, &connection_value) + || classifyAzureConnectionValue(connection_value) == AzureConnectionValue::Unmaskable) + { + maskEveryArgument(); + return; + } + maskAzureConnectionString(0); + return; + } + + String storage_account_url; + if (!tryGetStringFromArgument(0, &storage_account_url) + || classifyAzureConnectionValue(storage_account_url) != AzureConnectionValue::PlainStorageAccountURL) + { + /// This shape requires a plain account URL. A connection string here can only be hidden whole, + /// which cannot be combined with hiding `account_key`. + maskEveryArgument(); + return; + } + markSecretArgument(4); } bool FunctionSecretArgumentsFinder::isNamedCollectionName(size_t arg_idx) const diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 10342f18cc6b8..ee6b4922923b5 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -171,6 +171,11 @@ class FunctionSecretArgumentsFinder void findS3FunctionSecretArguments(bool is_cluster_function); void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function); bool maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named = false, size_t start = 0); + /// Whether the arguments an `AzureBlobStorage(named_collection, ...)` destination or table takes + /// from `start` can be shown: only an argument written here can carry a credential, and each has + /// to be readable enough to tell that it does not. `positional_limit` bounds the plain literals + /// read beside the overrides: one filename for a backup locator, none for a table engine. + bool azureCollectionArgumentsAreShowable(size_t start, size_t positional_limit); /// Masks the secrets of every URL form (`url`/`urlCluster` table functions, the `URL` table /// engine, and their named-collection variants): the userinfo password of the url positional or a /// named `url = ...` override, and the `headers(...)` values at any position. `url` is at @@ -180,6 +185,18 @@ class FunctionSecretArgumentsFinder bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const; static bool tryGetStringFromArgument(const AbstractFunction::Argument & argument, String * res, bool allow_identifier = true); + /// `BackupInfo` keeps named overrides and a trailing map for every backup engine, including the + /// ones that read neither, so an argument that is not a plain literal can carry a credential. + static bool hasOnlyLiteralArguments(const AbstractFunction & function); + + /// Whether a backup locator names its destination with exactly the literal arguments its engine + /// accepts, and therefore holds no credential. An engine that takes fewer rejects the rest only + /// after the statement has been formatted for logging, so the count has to be checked here too. + static bool isCredentialFreeBackupLocator(const AbstractFunction & function); + + /// Hides every argument, for a shape whose valid slots cannot be established. + void maskEveryArgument(); + void findRemoteFunctionSecretArguments(); /// Tries to get either a database name or a qualified table name from an argument. @@ -207,6 +224,10 @@ class FunctionSecretArgumentsFinder void findBackupDatabaseSecretArguments(); void findBackupNameSecretArguments(); + /// A backup destination reads a different signature than the table engine of the same name, so the + /// table-engine rule leaves an argument it does not model visible. + void findAzureBlobStorageBackupSecretArguments(); + /// Whether a specified argument can be the name of a named collection? bool isNamedCollectionName(size_t arg_idx) const; diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index d16d8f5623f74..4ea3878add595 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -1058,6 +1058,62 @@ def test_database_backup_engine_s3(): node.query("DROP DATABASE IF EXISTS backup_db_s3_2") +def test_database_backup_engine_azure_display_surfaces(): + """A live `Backup` database over an `AzureBlobStorage` locator: `account_key` must not reach + `SHOW CREATE DATABASE` or `system.databases.engine_full`. Only `S3` locators are reconstructed + argument by argument, so an Azure one keeps its engine name and arity and hides every argument.""" + azure_storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] + azure_account_name = "devstoreaccount1" + azure_account_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + + # A backup destination is write-once and the container is emptied only when the cluster starts, + # so give every run its own blob path: a repeat inside one session hits BACKUP_ALREADY_EXISTS. + blob = "db_backup_azure_surfaces_" + "".join( + random.choice(string.ascii_lowercase) for _ in range(8) + ) + locator = ( + f"AzureBlobStorage('{azure_storage_account_url}', '{cluster.azurite_container}', " + f"'{blob}', '{azure_account_name}', '{azure_account_key}')" + ) + masked_locator = ( + "AzureBlobStorage('[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]')" + ) + surfaces = [ + "SHOW CREATE DATABASE backup_db_azure_view", + "SELECT engine_full FROM system.databases WHERE name = 'backup_db_azure_view'", + ] + + node.query("DROP DATABASE IF EXISTS backup_db_azure_src SYNC") + node.query("DROP DATABASE IF EXISTS backup_db_azure_view SYNC") + node.query("CREATE DATABASE backup_db_azure_src") + node.query( + "CREATE TABLE backup_db_azure_src.t (x int) ENGINE = MergeTree ORDER BY x" + ) + node.query("INSERT INTO backup_db_azure_src.t SELECT * FROM numbers(10)") + node.query(f"BACKUP DATABASE backup_db_azure_src TO {locator} FORMAT Null") + node.query( + f"CREATE DATABASE backup_db_azure_view ENGINE = Backup('backup_db_azure_src', {locator})" + ) + + # TSVRaw so the locator is compared as written rather than through TSV escaping. + for surface in surfaces: + shown = node.query(f"{surface} FORMAT TSVRaw") + assert masked_locator in shown, shown + assert azure_account_key not in shown, shown + + # The key is in the locator, so its absence above is masking rather than an argument that the + # regenerated definition never carried. + for surface in surfaces: + shown = node.query(f"{surface} {show_secrets}=1 FORMAT TSVRaw") + assert azure_account_key in shown, shown + + # The masked surfaces belong to a working database, not to one that failed to open. + assert node.query("SELECT count() FROM backup_db_azure_view.t") == "10\n" + + node.query("DROP DATABASE IF EXISTS backup_db_azure_view SYNC") + node.query("DROP DATABASE IF EXISTS backup_db_azure_src SYNC") + + def test_backup_table_azure_named_collection(): """Test that secrets in Azure named collection backups are masked in system.backups and logs.""" azure_storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference index 030f5c5e60319..0b581d50ec4a4 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference @@ -197,6 +197,46 @@ BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_pos\', \'[HIDDEN]\', \'[HIDDEN]\' BACKUP TABLE nonexistent_04510 TO S3(nc_bkp_missing, \'visible_bkp_dir\', \'[HIDDEN]\') BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, secret_access_key = \'[HIDDEN]\', \'visible_bkp_dir2\') BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_mixed\', equals(access_key_id, \'ak\'), \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO File(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Disk(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Memory(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO File(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO `Null`(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Foo(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Null(); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureQueue(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', \'cont\', \'visible_04510_dir/b.zip\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'http://localhost:11111/acct\', \'visible_04510_cont\', \'visible_04510_dir/b.zip\'); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, connection_string = \'DefaultEndpointsProtocol=https;AccountName=visible_04510_acct;AccountKey=[HIDDEN];\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, account_key = \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, equals(storage_account_url, \'http://localhost:11111/visible_04510_url\'), account_key = \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, container = \'visible_04510_c1\', container = \'visible_04510_c2\'); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'http://localhost:11111/visible_04510_acct5\', \'visible_04510_cont5\', \'visible_04510_blob5\', \'visible_04510_acctname5\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte1 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte2 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte3 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte4 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte5 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte6 (`x` UInt8) ENGINE = AzureBlobStorage(\'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', \'visible_04510_teco\', \'visible_04510_teblob\') +CREATE TABLE t_04510_azte7 (`x` UInt8) ENGINE = AzureBlobStorage(\'http://localhost:11111/visible_04510_teacct\', \'visible_04510_tec5\', \'visible_04510_teb5\', \'visible_04510_teacctname\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte8 (`x` UInt8) ENGINE = AzureQueue(\'http://localhost:11111/visible_04510_teq/cont/*\', \'[HIDDEN]\') SETTINGS mode = \'unordered\' CREATE DATABASE db_04510_ec ENGINE = Backup(\'\', S3(\'url_dbec\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) CREATE DATABASE db_04510_postok ENGINE = Backup(\'\', S3(\'url_dbpostok\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_ncpos ENGINE = Backup(\'\', S3(nc_dbnc_missing, \'visible_dbnc_dir\', \'[HIDDEN]\')) @@ -208,6 +248,20 @@ CREATE DATABASE db_04510_ncurl ENGINE = Backup(\'\', S3(nc_dburl_missing, url = CREATE DATABASE db_04510_hdr ENGINE = Backup(\'\', S3(\'url_dbhdr\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_expr ENGINE = Backup(\'\', S3(\'url_dbexpr\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_quoted ENGINE = Backup(\'\', \'[HIDDEN]\') +CREATE DATABASE db_04510_tail ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_tmap ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_lone ENGINE = Backup(\'[HIDDEN]\') +CREATE DATABASE db_04510_swap ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_ident ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_ftail ENGINE = Backup(\'src_04510\', File(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_fover ENGINE = Backup(\'src_04510\', File(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_dover ENGINE = Backup(\'src_04510\', Disk(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_mover ENGINE = Backup(\'src_04510\', Memory(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_fvalid ENGINE = Backup(\'src_04510\', File(\'nonexistent_04510\')); +CREATE DATABASE db_04510_dvalid ENGINE = Backup(\'src_04510\', Disk(\'backups\', \'nonexistent_04510\')); +CREATE DATABASE db_04510_eval ENGINE = Backup(\'\', S3(\'http://localhost:11111/test/04510eval\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) +CREATE DATABASE db_04510_nonlit ENGINE = Backup(\'\', S3(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_azure ENGINE = Backup(\'\', AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_s3pos ENGINE = S3(\'url_dbs3pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\') DROP DATABASE IF EXISTS default_1 CREATE DATABASE default_1 ENGINE = S3(\'url_dbenv\', \'ak\', \'[HIDDEN]\', use_environment_credentials = 1) @@ -219,4 +273,4 @@ EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', NOSIGN, \'TSV\', \'x UInt8\', headers(\'Authorization\' = \'[HIDDEN]\')) EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(nc_04510_missing, url = \'https://[HIDDEN]@localhost:11111/test/04510qt?X-Amz-Signature=[HIDDEN]\', structure = \'x UInt8\') EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') UNION ALL SELECT 1 -1 0 +1 0 0 diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql index 569086cf03b2c..f3258d0805dfb 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql @@ -234,6 +234,138 @@ BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, BACKUP TABLE nonexistent_04510 TO S3('url_bkp_mixed', access_key_id = 'ak', 'SEKRIT_BKPMIX'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +-- The credential-free engines (Disk, File, Memory, Null) are credential-free only in the shape they +-- read: File and Memory take one argument, Disk two, Null none. A surplus argument is rejected after +-- the statement is logged, so it reaches the log holding whatever was written in it, an argument that is +-- no string is read by none of these engines and can carry a string of its own, and an engine name that +-- is not registered at all has no known shape to trust; all three must be hidden. The last statement is +-- the control: the shape Null does read carries nothing to hide, so it stays visible verbatim and gets +-- as far as resolving the table. +BACKUP TABLE nonexistent_04510 TO File('nonexistent_04510', + 'SEKRIT_TOFILEOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Disk('backups', 'nonexistent_04510', + 'SEKRIT_TODISKOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Memory('nonexistent_04510', + 'SEKRIT_TOMEMOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO File(['SEKRIT_TOFILEARR']); -- { serverError BAD_GET } +BACKUP TABLE nonexistent_04510 TO Null('SEKRIT_TONULLOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Foo('SEKRIT_TOUNKNOWN'); -- { serverError BACKUP_ENGINE_NOT_FOUND } +BACKUP TABLE nonexistent_04510 TO Null(); -- { serverError UNKNOWN_TABLE } + +-- The AzureBlobStorage backup destination reads a different signature than the table engine of the same +-- name: a named collection with an optional filename, three arguments (connection string or account url, +-- container, path), or five (adding account_name and account_key). An argument outside those shapes is +-- rejected only after the statement is logged, and AzureQueue has no backup engine at all. The last +-- two statements are the controls: a connection string hides its AccountKey, and the three-argument +-- shape has nothing to hide, so it stays visible verbatim. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct', 'cont', 'blob', + 'SEKRIT_AZTO4'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, 'dir', + 'SEKRIT_AZTONCPOS'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureQueue('http://localhost:11111/acct', 'cont', 'blob', + 'SEKRIT_AZQTO'); -- { serverError BACKUP_ENGINE_NOT_FOUND } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTOCSKEY==;', + 'cont', 'blob', 'acct', 'SEKRIT_AZTOCS5'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=c2VrcmV0Cg==;', + 'cont', 'visible_04510_dir/b.zip'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct', 'visible_04510_cont', 'visible_04510_dir/b.zip'); -- { serverError BAD_ARGUMENTS } + +-- A named collection can be overridden per statement, and the destination evaluates those overrides as +-- constant expressions. An override this rule cannot read may hold either credential, and hiding a +-- connection string replaces the whole argument, which cannot be combined with hiding account_key, so +-- both shapes hide the locator whole. The last three statements are the controls: a connection string +-- alone still hides only its AccountKey, account_key alone is hidden by itself, and an account url +-- override does not take the replacement path, so it stays visible next to a hidden account_key. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = concat('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=', + 'SEKRIT_AZNCEXPR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=c2VrcmV0Cg==;', + account_key = 'SEKRIT_AZNCBOTH'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = concat('https://a.blob.core.windows.net/c?sig=', + 'SEKRIT_AZNCURLEXPR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'DefaultEndpointsProtocol=https;AccountName=visible_04510_acct;AccountKey=SEKRIT_AZNCCS==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + account_key = 'SEKRIT_AZNCKEY'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'http://localhost:11111/visible_04510_url', + account_key = 'SEKRIT_AZNCURLKEY'); -- { serverError BAD_ARGUMENTS } + +-- An override key can be an expression too, and one this rule cannot read hides which credential the +-- value is: both the computed key and a malformed override hide the locator whole, as the S3 form above +-- already does for a key it cannot read. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + concat('account_', 'key') = 'SEKRIT_AZNCKEYEXPR'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + equals('account_key', 'SEKRIT_AZNCEQ3', 'surplus')); -- { serverError BAD_ARGUMENTS } + +-- A readable key does not make the override readable: the destination evaluates its value as a constant +-- expression, so a value that is no plain literal or identifier can nest a credential of its own and is +-- formatted verbatim before that evaluation rejects it, as the S3 form above already is for such a value. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + container = headers('Authorization' = 'SEKRIT_AZNCHDR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + blob_path = concat('SEKRIT_AZNCPATHEXPR', '/b.zip')); -- { serverError BAD_ARGUMENTS } + +-- connection_string and storage_account_url are mutually exclusive, and the destination reads at most +-- one of them, so an override it never reads holds whatever was written there. The last statement is +-- the control: a key that carries no credential stays visible however often it is repeated. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'http://localhost:11111/acct', + storage_account_url = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZNCPAIR==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'http://localhost:11111/acct', + connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZNCDUP==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, container = 'visible_04510_c1', container = 'visible_04510_c2'); -- { serverError BAD_ARGUMENTS } + +-- An account url is shown only when it is a plain storage account URL, which is what the destination +-- requires beside explicit credentials: userinfo, a query string (a SAS is a credential) and a fragment +-- each carry a credential of their own, in any shape and under any scheme spelling. A connection value +-- that is no string is read by neither this rule nor the destination, so it is hidden as well. The last +-- statement is the control: a plain url keeps the account url, container, path and account name visible +-- while account_key is hidden, and reaches the account_key decoding that rejects it. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'http://user:SEKRIT_AZNCUSERINFO@localhost:11111/acct'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'HTTPS://localhost:11111/acct?sig=SEKRIT_AZNCSAS'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://user:SEKRIT_AZ3USERINFO@localhost:11111/acct', + 'cont', 'blob'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct#f', 'cont', 'blob', + 'acct', 'SEKRIT_AZTO5KEY'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(['SEKRIT_AZ3ARR'], 'cont', 'blob'); -- { serverError BAD_GET } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/visible_04510_acct5', + 'visible_04510_cont5', 'visible_04510_blob5', 'visible_04510_acctname5', + 'SEKRIT_AZ5PLAINKEY'); -- { serverError STD_EXCEPTION } + +-- The AzureBlobStorage table engine reads its arguments the same way: an override whose value or key +-- this rule cannot read may hold a credential the engine does read (the parser evaluates both as +-- constant expressions), an account url is shown only when it is a plain storage account url, and a +-- connection value that is no string is read by neither. A connection string is hidden by replacing +-- its whole argument, which cannot be combined with hiding a following account_key. The last two +-- statements are the controls: the connection string alone still hides only its AccountKey, and a +-- plain account url keeps url, container, path and account name visible beside a hidden account_key. +CREATE TABLE t_04510_azte1 (x UInt8) ENGINE = AzureBlobStorage(nc_04510_missing, + connection_string = headers('Authorization' = 'SEKRIT_AZTEHDR')); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +CREATE TABLE t_04510_azte2 (x UInt8) ENGINE = AzureBlobStorage(nc_04510_missing, + upper('account_key') = 'SEKRIT_AZTEKEY'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +CREATE TABLE t_04510_azte3 (x UInt8) ENGINE = AzureBlobStorage(['SEKRIT_AZTEARR'], 'cont', 'blob'); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_04510_azte4 (x UInt8) ENGINE = AzureBlobStorage('http://localhost:11111/acct?sig=SEKRIT_AZTESAS', + 'cont', 'blob', 'acct', 'SEKRIT_AZTE5KEY'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte5 (x UInt8) ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTECS==;', + 'cont', 'blob', 'acct', 'SEKRIT_AZTEMIXKEY'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte6 (x UInt8) ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTECTLCS==;', + 'visible_04510_teco', 'visible_04510_teblob'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte7 (x UInt8) ENGINE = AzureBlobStorage('http://localhost:11111/visible_04510_teacct', + 'visible_04510_tec5', 'visible_04510_teb5', 'visible_04510_teacctname', + 'SEKRIT_AZTECTLKEY'); -- { serverError STD_EXCEPTION } + +-- The two-argument signature takes container and path from the url and a shared access signature +-- beside it, so its second argument is a credential wherever the same rule serves the engine. +CREATE TABLE t_04510_azte8 (x UInt8) ENGINE = AzureQueue('http://localhost:11111/visible_04510_teq/cont/*', + 'SEKRIT_AZTESAS2') SETTINGS mode = 'unordered'; -- { serverError UNKNOWN_FORMAT } + -- Backup database engine reconstructs the nested S3 destination; extra_credentials must be masked. CREATE DATABASE db_04510_ec ENGINE = Backup('', S3('url_dbec', 'ak', 'SEKRIT_SAK', extra_credentials(external_id = 'SEKRIT_EID'))); -- { serverError BAD_ARGUMENTS } @@ -272,9 +404,11 @@ CREATE DATABASE db_04510_mixed ENGINE = Backup('', S3('url_dbmixed', CREATE DATABASE db_04510_ncurl ENGINE = Backup('', S3(nc_dburl_missing, url = concat('https://user:SEKRIT_PW@', 'localhost/x?X-Amz-Signature=SEKRIT_SIG'))); -- { serverError BAD_ARGUMENTS } --- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim. +-- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim, and the +-- rejection message must not echo it either: a nested map's values are hidden only by its parent's +-- formatter, so such a node formatted on its own carries them in plaintext. CREATE DATABASE db_04510_hdr ENGINE = Backup('', S3('url_dbhdr', 'ak', 'SEKRIT_SAK', - headers('X-Auth' = 'SEKRIT_HDR'))); -- { serverError BAD_ARGUMENTS } + headers('X-Auth' = 'SEKRIT_DBHDR'))); -- { serverError BAD_ARGUMENTS } -- The reconstructor must also fail closed on a constant-expression extra_credentials key. CREATE DATABASE db_04510_expr ENGINE = Backup('', S3('url_dbexpr', 'ak', 'SEKRIT_SAK', @@ -287,6 +421,65 @@ CREATE DATABASE db_04510_expr ENGINE = Backup('', S3('url_dbexpr', 'ak', 'SEKRIT CREATE DATABASE db_04510_quoted ENGINE = Backup('', 'S3(\'https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG\', \'ak\', \'SEKRIT_QUOTED\')'); -- { serverError BAD_ARGUMENTS } +-- `Backup(database_name, locator)` is the only valid shape. The statement is logged before the arity +-- is validated, so a credential parked in a surplus argument must be hidden as well. +CREATE DATABASE db_04510_tail ENGINE = Backup('', S3('url_dbtail', 'ak', 'SEKRIT_SAK'), + 'SEKRIT_DBTAIL'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_tmap ENGINE = Backup('', S3('url_dbtmap', 'ak', 'SEKRIT_SAK'), + extra_credentials(external_id = 'SEKRIT_DBTMAP')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_lone ENGINE = Backup(S3('url_dblone', 'ak', + 'SEKRIT_DBLONE')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- A locator is also a credential carrier in the database-name position, where the argument is neither +-- masked by the S3 rule below nor safe to echo: the message that rejects it formats it standalone. +CREATE DATABASE db_04510_swap ENGINE = Backup(S3('url_dbswap', 'ak', 'SEKRIT_DBSWAP'), + ''); -- { serverError BAD_ARGUMENTS } + +-- An identifier is not a literal either, and the parsers evaluate one as a literal, so the +-- database-name position must hold a literal before the rest of the shape is trusted. +CREATE DATABASE db_04510_ident ENGINE = Backup(SEKRIT_DBIDENT, + S3('url_dbident', 'ak', 'SEKRIT_SAK')); -- { serverError BAD_ARGUMENTS } + +-- A credential-free locator (Disk, File, Memory, Null) names its destination with literals and holds +-- no credential, but it keeps a named override or nested map that it never reads, and that can carry +-- one. Both the logged text and the not-found message identify the locator, so both must hide it. +CREATE DATABASE db_04510_ftail ENGINE = Backup('src_04510', + File('nonexistent_04510', extra_credentials(external_id = 'SEKRIT_DBFTAIL'))); -- { serverError BACKUP_NOT_FOUND } + +-- A surplus literal argument reaches the same locator through the shape check instead of the tail, and +-- it is rejected only after the statement is logged, so it has to be hidden as well. The two controls +-- that follow hold the argument counts these engines do read (File one, Disk two), where the locator +-- names a destination only: nothing is masked, so both the logged text and the not-found message keep +-- it visible verbatim, and the transcript records the statement as sent rather than a re-formatted AST. +CREATE DATABASE db_04510_fover ENGINE = Backup('src_04510', + File('nonexistent_04510', 'SEKRIT_DBFILEOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_dover ENGINE = Backup('src_04510', + Disk('backups', 'nonexistent_04510', 'SEKRIT_DBDISKOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_mover ENGINE = Backup('src_04510', + Memory('nonexistent_04510', 'SEKRIT_DBMEMOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_fvalid ENGINE = Backup('src_04510', File('nonexistent_04510')); -- { serverError BACKUP_NOT_FOUND } +CREATE DATABASE db_04510_dvalid ENGINE = Backup('src_04510', Disk('backups', 'nonexistent_04510')); -- { serverError BACKUP_NOT_FOUND } + +-- A credential value given as an expression is hidden in the logged text, but evaluating it can fail +-- with a message that quotes its input, so the rejection must not carry that message either. The url +-- has to parse for the locator to reach credential evaluation at all. +CREATE DATABASE db_04510_eval ENGINE = Backup('', S3('http://localhost:11111/test/04510eval', 'ak', 'SEKRIT_SAK', + extra_credentials(external_id = toUInt64('SEKRIT_DBEVAL')))); -- { serverError BAD_ARGUMENTS } + +-- A non-tail argument that is neither a literal nor `key = value` cannot be reconstructed either. It +-- is rejected by a different message than the quoted locator above, and that message used to echo the +-- offending argument verbatim, so it needs its own tag. +CREATE DATABASE db_04510_nonlit ENGINE = Backup('', S3(concat('SEKRIT_NONLIT', 'x'), + 'url_dbnonlit')); -- { serverError BAD_ARGUMENTS } + +-- S3 is not the only backup engine whose locator carries credentials: AzureBlobStorage takes an +-- account_key and accepts connection strings and named-collection overrides that carry one too. +-- Only S3 is reconstructed above, so an Azure locator keeps its engine name and argument count +-- (neither is a secret) and every argument is hidden. The url carries a query string, which the +-- engine rejects before it reaches the network. +CREATE DATABASE db_04510_azure ENGINE = Backup('', AzureBlobStorage('http://localhost:11111/acct?sig=x', + 'cont', 'blob', 'account', 'SEKRIT_AZUREKEY')); -- { serverError BAD_ARGUMENTS } + -- The S3 database engine accepts no positional beyond secret_access_key; an extra positional must -- be masked in the logged query text. CREATE DATABASE db_04510_s3pos ENGINE = S3('url_dbs3pos', 'ak', 'SEKRIT_SAK', @@ -345,7 +538,15 @@ ORDER BY event_time_microseconds; -- logs each DDL from the replay worker, which re-masks a rewritten AST independently, so assert -- the masking property over every row this test produced, replay rows included. count() > 0 keeps -- an empty row set from passing vacuously. -SELECT count() > 0, countIf(query LIKE '%SEKRIT%') +-- The third column covers the recorded exception messages of the seven unreconstructible-locator +-- statements above, whose rejections used to echo the locator verbatim - one tag per message, since +-- they are thrown at different sites. It is scoped to those tags because widening it to every +-- deliberately-failing statement here would report unrelated pre-existing echoes. +SELECT count() > 0, countIf(query LIKE '%SEKRIT%'), + countIf(exception LIKE '%SEKRIT_QUOTED%' OR exception LIKE '%SEKRIT_NONLIT%' + OR exception LIKE '%SEKRIT_DBHDR%' OR exception LIKE '%SEKRIT_DBSWAP%' + OR exception LIKE '%SEKRIT_DBEVAL%' OR exception LIKE '%SEKRIT_DBFTAIL%' + OR exception LIKE '%SEKRIT_DBIDENT%') FROM system.query_log WHERE current_database = currentDatabase() AND type != 'QueryStart' diff --git a/tests/queries/0_stateless/05233_database_backup_locator_secret_mask.reference b/tests/queries/0_stateless/05233_database_backup_locator_secret_mask.reference new file mode 100644 index 0000000000000..38dae75858778 --- /dev/null +++ b/tests/queries/0_stateless/05233_database_backup_locator_secret_mask.reference @@ -0,0 +1,14 @@ +-- SHOW CREATE DATABASE +CREATE DATABASE default_view\nENGINE = Backup(\'default_src\', S3(\'http://localhost:11111/test/backups/default/locator_mask\', \'test\', \'[HIDDEN]\')) +-- system.databases.engine_full +Backup(\'default_src\', S3(\'http://localhost:11111/test/backups/default/locator_mask\', \'test\', \'[HIDDEN]\')) +-- secret occurrences in SHOW CREATE (must be 0) +0 +-- [HIDDEN] present in SHOW CREATE (must be 1) +1 +-- secret occurrences in system.databases.engine_full (must be 0) +0 +-- [HIDDEN] present in system.databases.engine_full (must be 1) +1 +-- rows visible through the Backup database +10 diff --git a/tests/queries/0_stateless/05233_database_backup_locator_secret_mask.sh b/tests/queries/0_stateless/05233_database_backup_locator_secret_mask.sh new file mode 100755 index 0000000000000..cb236e68b03c9 --- /dev/null +++ b/tests/queries/0_stateless/05233_database_backup_locator_secret_mask.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-encrypted-storage +# Tag no-fasttest: requires S3 +# Tag no-encrypted-storage: a backup from an encrypted disk restores only to an encrypted disk, so the Backup database gets no parts. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +client_opts=( + --allow_repeated_settings + --send_logs_level 'error' +) + +db=${CLICKHOUSE_DATABASE}_src +view=${CLICKHOUSE_DATABASE}_view +# The credentials the stateless suite uses for S3 backups: access key id 'test', secret 'testtest'. +# They are two distinct strings, so the assertions below tell the id apart from the secret. +dest="S3('http://localhost:11111/test/backups/${CLICKHOUSE_DATABASE}/locator_mask', 'test', 'testtest')" + +# BACKUP TO S3 emits a bandwidth warning on stderr under some configurations, which would break +# the reference in a bare run, so keep the log level at error throughout (as 02843 and 04327 do). +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP DATABASE IF EXISTS ${db}; +DROP DATABASE IF EXISTS ${view}; +CREATE DATABASE ${db}; +CREATE TABLE ${db}.t (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO ${db}.t SELECT number FROM numbers(10); +BACKUP DATABASE ${db} TO ${dest} FORMAT Null; +CREATE DATABASE ${view} ENGINE = Backup('${db}', ${dest}); +" + +# Both display surfaces of a real, successfully created Backup database. The locator must stay a +# nested S3 function so that only its secret_access_key is replaced by [HIDDEN]: the url and the +# access key id must remain visible, and the two surfaces must agree. +echo '-- SHOW CREATE DATABASE' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SHOW CREATE DATABASE ${view}" +echo '-- system.databases.engine_full' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SELECT engine_full FROM system.databases WHERE name = '${view}'" + +# Explicit counters, so a regression is visible on its own line rather than buried in a long one. +echo '-- secret occurrences in SHOW CREATE (must be 0)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SHOW CREATE DATABASE ${view}" | grep -c testtest +echo '-- [HIDDEN] present in SHOW CREATE (must be 1)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SHOW CREATE DATABASE ${view}" | grep -c -m1 '\[HIDDEN\]' +echo '-- secret occurrences in system.databases.engine_full (must be 0)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SELECT engine_full FROM system.databases WHERE name = '${view}'" | grep -c testtest +echo '-- [HIDDEN] present in system.databases.engine_full (must be 1)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SELECT engine_full FROM system.databases WHERE name = '${view}'" | grep -c -m1 '\[HIDDEN\]' + +# The Backup database is readable, so the masked surfaces above are not the output of a broken database. +echo '-- rows visible through the Backup database' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SELECT count() FROM ${view}.t" + +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP DATABASE ${view}; +DROP DATABASE ${db}; +" diff --git a/tests/queries/0_stateless/05234_restore_duplicate_definition_secret_mask.reference b/tests/queries/0_stateless/05234_restore_duplicate_definition_secret_mask.reference new file mode 100644 index 0000000000000..2d33a25de9b30 --- /dev/null +++ b/tests/queries/0_stateless/05234_restore_duplicate_definition_secret_mask.reference @@ -0,0 +1,34 @@ +-- database-level duplicate definition reached (must be 1) +1 +-- CANNOT_RESTORE_DATABASE (must be 1) +1 +-- secret occurrences in the error (must be 0) +0 +-- [HIDDEN] present in the error (must be 1) +1 +-- archived locator still identifiable in the error (must be 1) +1 +-- restored Backup database reads its source table (must be 3) +3 +-- the restored locator is hidden on display (must be 1) +1 +-- table-level duplicate definition reached (must be 1) +1 +-- CANNOT_RESTORE_TABLE (must be 1) +1 +-- secret occurrences in the error (must be 0) +0 +-- [HIDDEN] present in the error (must be 1) +1 +-- archived table definition still identifiable in the error (must be 1) +1 +-- Azure table-level duplicate definition reached (must be 1) +1 +-- CANNOT_RESTORE_TABLE for the Azure definitions (must be 1) +1 +-- signature occurrences in the error (must be 0) +0 +-- [HIDDEN] present in the error (must be 1) +1 +-- restore target still named in the error (must be 1) +1 diff --git a/tests/queries/0_stateless/05234_restore_duplicate_definition_secret_mask.sh b/tests/queries/0_stateless/05234_restore_duplicate_definition_secret_mask.sh new file mode 100755 index 0000000000000..8ca86313e9b80 --- /dev/null +++ b/tests/queries/0_stateless/05234_restore_duplicate_definition_secret_mask.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-encrypted-storage +# Tag no-fasttest: requires the S3 endpoint +# Tag no-encrypted-storage: a backup from an encrypted disk restores only to an encrypted disk, so the restored Backup database gets no parts. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +client_opts=( + --allow_repeated_settings + --send_logs_level 'error' +) + +s1=${CLICKHOUSE_DATABASE}_s1 +s2=${CLICKHOUSE_DATABASE}_s2 +v1=${CLICKHOUSE_DATABASE}_v1 +v2=${CLICKHOUSE_DATABASE}_v2 +merged=${CLICKHOUSE_DATABASE}_merged +restored=${CLICKHOUSE_DATABASE}_restored + +# Access key id 'test', secret 'testtest': the credentials the stateless suite uses for S3. They are +# two distinct strings, so the assertions below tell the id apart from the secret. +inner1="S3('http://localhost:11111/test/backups/${CLICKHOUSE_DATABASE}/dupdef1', 'test', 'testtest')" +inner2="S3('http://localhost:11111/test/backups/${CLICKHOUSE_DATABASE}/dupdef2', 'test', 'testtest')" +# The outer locator carries no credential, so a secret found in the restore error can only have come +# from one of the two archived definitions and not from the statement being executed. +outer="Disk('backups', '${CLICKHOUSE_DATABASE}_dupdef_outer')" + +# The two tables have different names on purpose: two tables renamed into one target would reach the +# table-level duplicate-definition check first, and the database-level one is what is asserted here. +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP DATABASE IF EXISTS ${s1}; +DROP DATABASE IF EXISTS ${s2}; +DROP DATABASE IF EXISTS ${v1}; +DROP DATABASE IF EXISTS ${v2}; +DROP DATABASE IF EXISTS ${merged}; +DROP DATABASE IF EXISTS ${restored}; +CREATE DATABASE ${s1}; +CREATE TABLE ${s1}.t1 (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO ${s1}.t1 SELECT number FROM numbers(3); +CREATE DATABASE ${s2}; +CREATE TABLE ${s2}.t2 (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO ${s2}.t2 SELECT number FROM numbers(3); +BACKUP DATABASE ${s1} TO ${inner1} FORMAT Null; +BACKUP DATABASE ${s2} TO ${inner2} FORMAT Null; +CREATE DATABASE ${v1} ENGINE = Backup('${s1}', ${inner1}); +CREATE DATABASE ${v2} ENGINE = Backup('${s2}', ${inner2}); +BACKUP DATABASE ${v1}, DATABASE ${v2} TO ${outer} FORMAT Null; +" + +# Both elements rename to one target, so the definition read second reaches the duplicate-definition +# check, which reports both archived definitions. +err=$(${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q \ + "RESTORE DATABASE ${v1} AS ${merged}, DATABASE ${v2} AS ${merged} FROM ${outer}" 2>&1) + +# The first two counters pin the failure to that check rather than to an earlier one, so the +# remaining three cannot be satisfied by an unrelated error or by an empty result. +echo '-- database-level duplicate definition reached (must be 1)' +echo "$err" | grep -c -m1 'Extracted two different create queries for the same database' +echo '-- CANNOT_RESTORE_DATABASE (must be 1)' +echo "$err" | grep -c -m1 CANNOT_RESTORE_DATABASE +echo '-- secret occurrences in the error (must be 0)' +echo "$err" | grep -c testtest +echo '-- [HIDDEN] present in the error (must be 1)' +echo "$err" | grep -c -m1 '\[HIDDEN\]' +echo '-- archived locator still identifiable in the error (must be 1)' +echo "$err" | grep -c -m1 dupdef1 + +# The archived definition has to keep the credential as written: attaching a Backup database checks the +# backup with the credentials the locator carries, so a definition archived with '[HIDDEN]' would fail +# the restore below with ACCESS_DENIED instead of reading the source table through it. +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "RESTORE DATABASE ${v1} AS ${restored} FROM ${outer}" > /dev/null +echo '-- restored Backup database reads its source table (must be 3)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SELECT count() FROM ${restored}.t1" +echo '-- the restored locator is hidden on display (must be 1)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q \ + "SHOW CREATE DATABASE ${restored} SETTINGS format_display_secrets_in_show_and_select = 0" | grep -c -m1 '\[HIDDEN\]' + +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP DATABASE IF EXISTS ${restored}; +DROP DATABASE IF EXISTS ${merged}; +DROP DATABASE ${v1}; +DROP DATABASE ${v2}; +DROP DATABASE ${s1}; +DROP DATABASE ${s2}; +" + +# The same function checks table definitions the same way, and a table engine keeps its credential in +# the same argument position a database locator does. +ta=${CLICKHOUSE_DATABASE}.dup_a +tb=${CLICKHOUSE_DATABASE}.dup_b +tmerged=${CLICKHOUSE_DATABASE}.dup_merged +table_outer="Disk('backups', '${CLICKHOUSE_DATABASE}_tabledef_outer')" + +# Two tables renamed into one target: the definition read second is compared against one that is +# already recorded for that target and differs from it, which is what the check requires. +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP TABLE IF EXISTS ${ta}; +DROP TABLE IF EXISTS ${tb}; +DROP TABLE IF EXISTS ${tmerged}; +CREATE TABLE ${ta} (id UInt64) ENGINE = S3('http://localhost:11111/test/${CLICKHOUSE_DATABASE}/dup_a.csv', 'test', 'testtest', 'CSV'); +CREATE TABLE ${tb} (id UInt64) ENGINE = S3('http://localhost:11111/test/${CLICKHOUSE_DATABASE}/dup_b.csv', 'test', 'testtest', 'CSV'); +BACKUP TABLE ${ta}, TABLE ${tb} TO ${table_outer} FORMAT Null; +" + +err=$(${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q \ + "RESTORE TABLE ${ta} AS ${tmerged}, TABLE ${tb} AS ${tmerged} FROM ${table_outer}" 2>&1) + +echo '-- table-level duplicate definition reached (must be 1)' +echo "$err" | grep -c -m1 'Extracted two different create queries for the same table' +echo '-- CANNOT_RESTORE_TABLE (must be 1)' +echo "$err" | grep -c -m1 CANNOT_RESTORE_TABLE +echo '-- secret occurrences in the error (must be 0)' +echo "$err" | grep -c testtest +echo '-- [HIDDEN] present in the error (must be 1)' +echo "$err" | grep -c -m1 '\[HIDDEN\]' +echo '-- archived table definition still identifiable in the error (must be 1)' +echo "$err" | grep -c -m1 dup_a + +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP TABLE ${ta}; +DROP TABLE ${tb}; +" + +# An Azure table definition reaches the same check, and an Azure locator can hold its credential inside +# the account url rather than in an argument of its own. +aza=${CLICKHOUSE_DATABASE}.az_a +azb=${CLICKHOUSE_DATABASE}.az_b +azmerged=${CLICKHOUSE_DATABASE}.az_merged +azure_outer="Disk('backups', '${CLICKHOUSE_DATABASE}_azuredef_outer')" +# A shared access signature is a credential, and the account url is plain apart from it, so a secret +# found in the restore error can only have come from one of the two archived definitions. Neither +# creating nor backing up the tables reads the endpoint, so no Azure service is needed here. +azure_a="AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=SEKRITAZURESAS', 'cont', 'az_a.csv', 'CSV')" +azure_b="AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=SEKRITAZURESAS', 'cont', 'az_b.csv', 'CSV')" + +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP TABLE IF EXISTS ${aza}; +DROP TABLE IF EXISTS ${azb}; +DROP TABLE IF EXISTS ${azmerged}; +CREATE TABLE ${aza} (id UInt64) ENGINE = ${azure_a}; +CREATE TABLE ${azb} (id UInt64) ENGINE = ${azure_b}; +BACKUP TABLE ${aza}, TABLE ${azb} TO ${azure_outer} FORMAT Null; +" + +err=$(${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q \ + "RESTORE TABLE ${aza} AS ${azmerged}, TABLE ${azb} AS ${azmerged} FROM ${azure_outer}" 2>&1) + +echo '-- Azure table-level duplicate definition reached (must be 1)' +echo "$err" | grep -c -m1 'Extracted two different create queries for the same table' +echo '-- CANNOT_RESTORE_TABLE for the Azure definitions (must be 1)' +echo "$err" | grep -c -m1 CANNOT_RESTORE_TABLE +echo '-- signature occurrences in the error (must be 0)' +echo "$err" | grep -c SEKRITAZURESAS +echo '-- [HIDDEN] present in the error (must be 1)' +echo "$err" | grep -c -m1 '\[HIDDEN\]' +echo '-- restore target still named in the error (must be 1)' +echo "$err" | grep -c -m1 az_merged + +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP TABLE ${aza}; +DROP TABLE ${azb}; +" From 317cd284cb24361af84aa3a2cc1f3ad6e3865d2e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 10:58:36 +0000 Subject: [PATCH 098/296] Backport #116378 to 26.8: Do not put a row into a dictionary source's pipeline header --- src/Dictionaries/CassandraSource.cpp | 2 +- src/Dictionaries/DictionarySourceFactory.cpp | 2 ++ src/Dictionaries/DirectDictionary.cpp | 4 ++-- src/Dictionaries/NullDictionarySource.cpp | 2 +- src/Dictionaries/RedisSource.cpp | 2 +- src/Processors/Sources/MongoDBSource.cpp | 2 +- src/Processors/Sources/ShellCommandSource.cpp | 2 +- src/Processors/Sources/YTsaurusSource.cpp | 4 ++-- .../05043_direct_dictionary_in_merge.reference | 1 + .../05043_direct_dictionary_in_merge.sh | 15 +++++++++++++++ 10 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference create mode 100755 tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh diff --git a/src/Dictionaries/CassandraSource.cpp b/src/Dictionaries/CassandraSource.cpp index 83e7869ddeb6c..8030b120b82c9 100644 --- a/src/Dictionaries/CassandraSource.cpp +++ b/src/Dictionaries/CassandraSource.cpp @@ -25,7 +25,7 @@ CassandraSource::CassandraSource( const String & query_str, SharedHeader & sample_block, size_t max_block_size_) - : ISource(sample_block) + : ISource(std::make_shared(sample_block->cloneEmpty())) , session(session_) , statement(query_str.c_str(), /*parameters count*/ 0) , max_block_size(max_block_size_) diff --git a/src/Dictionaries/DictionarySourceFactory.cpp b/src/Dictionaries/DictionarySourceFactory.cpp index 248efbe0e5ac6..c96a071d945dc 100644 --- a/src/Dictionaries/DictionarySourceFactory.cpp +++ b/src/Dictionaries/DictionarySourceFactory.cpp @@ -20,6 +20,8 @@ namespace ErrorCodes namespace { + /// Holds one row per column, an attribute's row being its `null_value` default. A port header + /// must have no rows, so a source publishing this block as its header has to strip them first. Block createSampleBlock(const DictionaryStructure & dict_struct) { Block block; diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 684bd77f4c83c..05d98fc6c70a9 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -346,14 +346,14 @@ class SourceFromQueryPipeline : public ISource { public: explicit SourceFromQueryPipeline(QueryPipeline & pipeline_) - : ISource(pipeline_.getSharedHeader()) + : ISource(std::make_shared(pipeline_.getSharedHeader()->cloneEmpty())) , executor(pipeline_) { pipeline_.setConcurrencyControl(false); } explicit SourceFromQueryPipeline(BlockIO io) - : ISource(io.pipeline.getSharedHeader()) + : ISource(std::make_shared(io.pipeline.getSharedHeader()->cloneEmpty())) , io_holder(std::move(io)) , executor(io_holder->pipeline) { diff --git a/src/Dictionaries/NullDictionarySource.cpp b/src/Dictionaries/NullDictionarySource.cpp index c940c28e5ef70..beb30e8b8f8d7 100644 --- a/src/Dictionaries/NullDictionarySource.cpp +++ b/src/Dictionaries/NullDictionarySource.cpp @@ -22,7 +22,7 @@ BlockIO NullDictionarySource::loadAll() { LOG_TRACE(getLogger("NullDictionarySource"), "loadAll {}", toString()); BlockIO io; - io.pipeline = QueryPipeline(std::make_shared(sample_block)); + io.pipeline = QueryPipeline(std::make_shared(std::make_shared(sample_block->cloneEmpty()))); return io; } diff --git a/src/Dictionaries/RedisSource.cpp b/src/Dictionaries/RedisSource.cpp index a606d9cc69cb8..c2c38623dce34 100644 --- a/src/Dictionaries/RedisSource.cpp +++ b/src/Dictionaries/RedisSource.cpp @@ -26,7 +26,7 @@ namespace DB const RedisStorageType & storage_type_, SharedHeader sample_block, size_t max_block_size_) - : ISource(sample_block) + : ISource(std::make_shared(sample_block->cloneEmpty())) , connection(std::move(connection_)) , keys(keys_) , storage_type(storage_type_) diff --git a/src/Processors/Sources/MongoDBSource.cpp b/src/Processors/Sources/MongoDBSource.cpp index 4dca8b9372656..e5993dde6597c 100644 --- a/src/Processors/Sources/MongoDBSource.cpp +++ b/src/Processors/Sources/MongoDBSource.cpp @@ -160,7 +160,7 @@ MongoDBSource::MongoDBSource( const mongocxx::options::find & options, SharedHeader sample_block_, const UInt64 & max_block_size_) - : ISource{sample_block_} + : ISource{std::make_shared(sample_block_->cloneEmpty())} , client{uri} , database{client.database(uri.database())} , collection{database.collection(collection_name)} diff --git a/src/Processors/Sources/ShellCommandSource.cpp b/src/Processors/Sources/ShellCommandSource.cpp index b39668905a7f7..01a0eca5d00e6 100644 --- a/src/Processors/Sources/ShellCommandSource.cpp +++ b/src/Processors/Sources/ShellCommandSource.cpp @@ -503,7 +503,7 @@ namespace const ShellCommandSourceConfiguration & configuration_ = {}, std::unique_ptr && command_holder_ = nullptr, std::shared_ptr process_pool_ = nullptr) - : ISource(sample_block_) + : ISource(std::make_shared(sample_block_->cloneEmpty())) , context(context_) , format(format_) , sample_block(sample_block_) diff --git a/src/Processors/Sources/YTsaurusSource.cpp b/src/Processors/Sources/YTsaurusSource.cpp index 64e1b12506261..48e8faa9a12dd 100644 --- a/src/Processors/Sources/YTsaurusSource.cpp +++ b/src/Processors/Sources/YTsaurusSource.cpp @@ -31,7 +31,7 @@ namespace YTsaurusSetting YTsaurusTableSourceStaticTable::YTsaurusTableSourceStaticTable( YTsaurusClientPtr client_, const String & cypress_path_, std::pair rows_range_, const YTsaurusTableSourceOptions & source_options_, const SharedHeader & sample_block_, const UInt64 & max_block_size_) - : ISource(sample_block_) + : ISource(std::make_shared(sample_block_->cloneEmpty())) , client(std::move(client_)) , cypress_path(cypress_path_) , rows_range(std::move(rows_range_)) @@ -62,7 +62,7 @@ Chunk YTsaurusTableSourceStaticTable::generate() YTsaurusTableSourceDynamicTable::YTsaurusTableSourceDynamicTable( YTsaurusClientPtr client_, const String & cypress_path, const YTsaurusTableSourceOptions & source_options_, const SharedHeader & sample_block_, const UInt64 & max_block_size_) - : ISource(sample_block_) + : ISource(std::make_shared(sample_block_->cloneEmpty())) , client(std::move(client_)) , sample_block(sample_block_) , max_block_size(max_block_size_) diff --git a/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference new file mode 100644 index 0000000000000..72c3262a309d8 --- /dev/null +++ b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference @@ -0,0 +1 @@ +Hello 1 diff --git a/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh new file mode 100755 index 0000000000000..8385d6f97658c --- /dev/null +++ b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# A `direct` dictionary reads through the source's own pipeline, and `merge` wraps every child in +# `materialize`, which rejects a header that carries rows. +$CLICKHOUSE_CLIENT -q " + CREATE DICTIONARY dict (word String, counter UInt32) + PRIMARY KEY word + SOURCE(HTTP(url '${CLICKHOUSE_URL}&query=SELECT+%27Hello%27,1+FORMAT+CSV' format 'CSV')) + LAYOUT(DIRECT())" + +$CLICKHOUSE_CLIENT -q "SELECT * FROM merge(currentDatabase(), '^dict\$')" From 8fd1e7f2d12a44613508a1d0b1b1b3cc88c0aa84 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 11:32:33 +0000 Subject: [PATCH 099/296] Backport #120453 to 26.8: Speed up Map key lookup for LowCardinality keys --- src/Columns/LowCardinalityValueIndex.h | 70 ++++++ src/DataTypes/DataTypeMapHelpers.cpp | 164 ++++++++++++-- src/Functions/array/arrayElement.cpp | 57 +++-- .../map_lowcardinality_key_lookup.xml | 43 ++++ ...18_map_lowcardinality_key_lookup.reference | 102 +++++++++ .../05218_map_lowcardinality_key_lookup.sql | 203 ++++++++++++++++++ .../05234_map_key_lookup_nan.reference | 22 ++ .../0_stateless/05234_map_key_lookup_nan.sql | 61 ++++++ 8 files changed, 671 insertions(+), 51 deletions(-) create mode 100644 src/Columns/LowCardinalityValueIndex.h create mode 100644 tests/performance/map_lowcardinality_key_lookup.xml create mode 100644 tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference create mode 100644 tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql create mode 100644 tests/queries/0_stateless/05234_map_key_lookup_nan.reference create mode 100644 tests/queries/0_stateless/05234_map_key_lookup_nan.sql diff --git a/src/Columns/LowCardinalityValueIndex.h b/src/Columns/LowCardinalityValueIndex.h new file mode 100644 index 0000000000000..fb6fbecdad7d7 --- /dev/null +++ b/src/Columns/LowCardinalityValueIndex.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ + +enum class LowCardinalityValueLookupResult +{ + /// The dictionary does not hold strings, so the caller has to compare the values itself. + Unsupported, + /// No row of the column can hold the value. + NotFound, + Found, +}; + +/// Resolves `value` to its position in the dictionary of `column` and calls +/// `callback(const IndexType * indexes, IndexType value_index)`, so that comparing a row of `column` +/// with `value` becomes an integer comparison of dictionary positions. +template +LowCardinalityValueLookupResult callWithLowCardinalityValueIndex( + const ColumnLowCardinality & column, std::string_view value, Callback && callback) +{ + const auto & dictionary = column.getDictionary(); + const auto & dictionary_values = *dictionary.getNestedNotNullableColumn(); + if (!typeid_cast(&dictionary_values) && !typeid_cast(&dictionary_values)) + return LowCardinalityValueLookupResult::Unsupported; + + auto value_index = dictionary.getOrFindValueIndex(value); + if (!value_index) + return LowCardinalityValueLookupResult::NotFound; + + const IColumn & indexes = column.getIndexes(); + + auto call_for_index_type = [&](IndexType) -> LowCardinalityValueLookupResult + { + /// A shared dictionary also holds values of other columns, so a dictionary position is not + /// necessarily representable in the index type of this column. No row here references it then. + if constexpr (!std::is_same_v) + { + if (*value_index > std::numeric_limits::max()) + return LowCardinalityValueLookupResult::NotFound; + } + + callback( + assert_cast &>(indexes).getData().data(), + static_cast(*value_index)); + + return LowCardinalityValueLookupResult::Found; + }; + + switch (column.getSizeOfIndexType()) + { + case sizeof(UInt8): return call_for_index_type(UInt8{}); + case sizeof(UInt16): return call_for_index_type(UInt16{}); + case sizeof(UInt32): return call_for_index_type(UInt32{}); + case sizeof(UInt64): return call_for_index_type(UInt64{}); + default: throwUnexpectedLowCardinalityIndexType(column.getSizeOfIndexType()); + } +} + +} diff --git a/src/DataTypes/DataTypeMapHelpers.cpp b/src/DataTypes/DataTypeMapHelpers.cpp index 63aa451627f6c..5443542a3be62 100644 --- a/src/DataTypes/DataTypeMapHelpers.cpp +++ b/src/DataTypes/DataTypeMapHelpers.cpp @@ -1,11 +1,15 @@ #include #include +#include #include +#include #include #include #include #include +#include +#include #include #include @@ -35,21 +39,30 @@ struct KeyMatcherGeneric bool match(size_t keys_row) const { - return keys_column.compareAt(keys_row, 0, key, 0) == 0; + /// The direction hint must not be zero. When exactly one side is a `NaN`, or a NULL of a + /// `Nullable` nested in the key, `compareAt` answers with the hint itself, so a zero hint + /// reports them as equal to every other value: a row holding such a key would match any + /// requested key, and requesting such a key would match any key. + return keys_column.compareAt(keys_row, 0, key, 1) == 0; } }; -/// Specialized key matcher for ColumnVector. Compares values directly -/// without virtual dispatch. -template +/// Specialized key matcher for columns holding a flat array of values (ColumnVector, +/// ColumnDecimal). Compares values directly without virtual dispatch. +template struct KeyMatcherVector { - const typename ColumnVector::Container & data; + using T = typename ColumnType::ValueType; + + const typename ColumnType::Container & data; T key_value; bool match(size_t keys_row) const { - return data[keys_row] == key_value; + /// `CompareHelper` is a plain `==` for every type but the floating point ones, where it + /// keeps the matcher in agreement with `KeyMatcherGeneric`: a `NaN` key is found by a + /// requested `NaN` and by nothing else. + return CompareHelper::equals(data[keys_row], key_value, 1); } }; @@ -86,6 +99,20 @@ struct KeyMatcherFixedString } }; +/// Specialized key matcher for ColumnLowCardinality. The requested key is resolved to its dictionary +/// position once, so a key is matched without comparing the key values. +template +struct KeyMatcherLowCardinality +{ + const IndexType * indexes; + IndexType key_index; + + bool match(size_t keys_row) const + { + return indexes[keys_row] == key_index; + } +}; + /// The core position-finding loop, parametrized by Matcher type. /// For each row in [start, end), finds the flat index of the matching key. /// @@ -126,6 +153,59 @@ void findKeyPositions( } } +/// The value of the single-row key column, if the key is a String or a FixedString, +/// possibly wrapped in LowCardinality. +std::optional tryGetStringKey(const IColumn & key) +{ + const IColumn * key_values = &key; + if (const auto * key_low_cardinality = typeid_cast(&key)) + key_values = key_low_cardinality->getDictionary().getNestedNotNullableColumn().get(); + + if (!typeid_cast(key_values) && !typeid_cast(key_values)) + return {}; + + return key.getDataAt(0); +} + +/// Finds the key positions by comparing dictionary positions when the keys are LowCardinality. +/// Returns false if the key column types are not supported, leaving it to the generic matcher. +bool tryFindKeyPositionsLowCardinality( + const IColumn & keys_column, + const ColumnArray::Offsets & offsets, + const IColumn & key, + size_t start, + size_t end, + PaddedPODArray & matched_positions) +{ + const auto * keys_low_cardinality = typeid_cast(&keys_column); + if (!keys_low_cardinality) + return false; + + auto key_value = tryGetStringKey(key); + if (!key_value) + return false; + + auto lookup_result = callWithLowCardinalityValueIndex( + *keys_low_cardinality, + *key_value, + [&](const auto * indexes, auto key_index) + { + KeyMatcherLowCardinality matcher{indexes, key_index}; + findKeyPositions(offsets, matcher, start, end, matched_positions); + }); + + if (lookup_result == LowCardinalityValueLookupResult::Unsupported) + return false; + + if (lookup_result == LowCardinalityValueLookupResult::NotFound) + { + matched_positions.clear(); + matched_positions.resize_fill(end - start, KEY_NOT_FOUND); + } + + return true; +} + /// Dispatches to the appropriate specialized matcher based on the key column type, /// then calls findKeyPositions with that matcher. void findKeyPositionsDispatch( @@ -138,31 +218,47 @@ void findKeyPositionsDispatch( { TypeIndex type_id = keys_column.getDataType(); - /// Try ColumnVector specializations. switch (type_id) { -#define DISPATCH_VECTOR(T) \ +#define DISPATCH_COLUMN(T, ColType) \ case TypeIndex::T: \ { \ - using ColType = ColumnVector; \ const auto & typed_col = assert_cast(keys_column); \ const auto & key_col = assert_cast(key); \ - KeyMatcherVector matcher{typed_col.getData(), key_col.getData()[0]}; \ + KeyMatcherVector matcher{typed_col.getData(), key_col.getData()[0]}; \ findKeyPositions(offsets, matcher, start, end, matched_positions); \ return; \ } +#define DISPATCH_VECTOR(T) DISPATCH_COLUMN(T, ColumnVector) +#define DISPATCH_DECIMAL(T) DISPATCH_COLUMN(T, ColumnDecimal) DISPATCH_VECTOR(UInt8) DISPATCH_VECTOR(UInt16) DISPATCH_VECTOR(UInt32) DISPATCH_VECTOR(UInt64) + DISPATCH_VECTOR(UInt128) + DISPATCH_VECTOR(UInt256) DISPATCH_VECTOR(Int8) DISPATCH_VECTOR(Int16) DISPATCH_VECTOR(Int32) DISPATCH_VECTOR(Int64) + DISPATCH_VECTOR(Int128) + DISPATCH_VECTOR(Int256) + DISPATCH_VECTOR(BFloat16) DISPATCH_VECTOR(Float32) DISPATCH_VECTOR(Float64) + DISPATCH_VECTOR(UUID) + DISPATCH_VECTOR(IPv4) + DISPATCH_VECTOR(IPv6) + DISPATCH_DECIMAL(Decimal32) + DISPATCH_DECIMAL(Decimal64) + DISPATCH_DECIMAL(Decimal128) + DISPATCH_DECIMAL(Decimal256) + DISPATCH_DECIMAL(DateTime64) + DISPATCH_DECIMAL(Time64) +#undef DISPATCH_DECIMAL #undef DISPATCH_VECTOR +#undef DISPATCH_COLUMN case TypeIndex::String: { @@ -181,13 +277,19 @@ void findKeyPositionsDispatch( findKeyPositions(offsets, matcher, start, end, matched_positions); return; } - default: + case TypeIndex::LowCardinality: { - /// Fallback: generic matcher using virtual compareAt. - KeyMatcherGeneric matcher{keys_column, key}; - findKeyPositions(offsets, matcher, start, end, matched_positions); + if (tryFindKeyPositionsLowCardinality(keys_column, offsets, key, start, end, matched_positions)) + return; + break; } + default: + break; } + + /// Fallback: generic matcher using virtual compareAt. + KeyMatcherGeneric matcher{keys_column, key}; + findKeyPositions(offsets, matcher, start, end, matched_positions); } /// --------------------------------------------------------------------------- @@ -213,17 +315,20 @@ void extractValuesGeneric( } } -/// Specialized value extractor for ColumnVector, with optional Nullable support. +/// Specialized value extractor for columns holding a flat array of values (ColumnVector, +/// ColumnDecimal), with optional Nullable support. /// If src_null_map / dst_null_map are non-null, propagates null flags. /// For missing keys, inserts a default value and sets the null flag to 1. -template +template void extractValuesVector( - const ColumnVector & values_column, - ColumnVector & result, + const ColumnType & values_column, + ColumnType & result, const PaddedPODArray & matched_positions, const NullMap * src_null_map = nullptr, NullMap * dst_null_map = nullptr) { + using T = typename ColumnType::ValueType; + const auto & src_data = values_column.getData(); auto & dst_data = result.getData(); size_t old_size = dst_data.size(); @@ -374,28 +479,45 @@ void extractValuesDispatch( switch (type_id) { -#define DISPATCH_VECTOR(T) \ +#define DISPATCH_COLUMN(T, ColType) \ case TypeIndex::T: \ { \ - using ColType = ColumnVector; \ - extractValuesVector( \ + extractValuesVector( \ assert_cast(*data_column), \ assert_cast(*result_data_column), \ matched_positions, src_null_map, dst_null_map); \ return; \ } +#define DISPATCH_VECTOR(T) DISPATCH_COLUMN(T, ColumnVector) +#define DISPATCH_DECIMAL(T) DISPATCH_COLUMN(T, ColumnDecimal) DISPATCH_VECTOR(UInt8) DISPATCH_VECTOR(UInt16) DISPATCH_VECTOR(UInt32) DISPATCH_VECTOR(UInt64) + DISPATCH_VECTOR(UInt128) + DISPATCH_VECTOR(UInt256) DISPATCH_VECTOR(Int8) DISPATCH_VECTOR(Int16) DISPATCH_VECTOR(Int32) DISPATCH_VECTOR(Int64) + DISPATCH_VECTOR(Int128) + DISPATCH_VECTOR(Int256) + DISPATCH_VECTOR(BFloat16) DISPATCH_VECTOR(Float32) DISPATCH_VECTOR(Float64) + DISPATCH_VECTOR(UUID) + DISPATCH_VECTOR(IPv4) + DISPATCH_VECTOR(IPv6) + DISPATCH_DECIMAL(Decimal32) + DISPATCH_DECIMAL(Decimal64) + DISPATCH_DECIMAL(Decimal128) + DISPATCH_DECIMAL(Decimal256) + DISPATCH_DECIMAL(DateTime64) + DISPATCH_DECIMAL(Time64) +#undef DISPATCH_DECIMAL #undef DISPATCH_VECTOR +#undef DISPATCH_COLUMN case TypeIndex::String: { diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index 663b58c53340c..a041de61cb250 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1855,6 +1856,17 @@ struct MatcherNumberConst bool match(size_t row_data, size_t /* row_index */) const { return data[row_data] == index; } }; +/// Matcher for keys of a ColumnLowCardinality. The requested key is resolved to its dictionary +/// position once, so a key is matched without comparing the key values. +template +struct MatcherLowCardinalityConst +{ + const IndexType * indexes; + IndexType key_index; + + bool match(size_t row_data, size_t /* row_index */) const { return indexes[row_data] == key_index; } +}; + } template @@ -1921,11 +1933,6 @@ bool castColumnString(const IColumn * column, F && f) return castTypeToEither(column, std::forward(f)); } -bool isStringOrFixedStringColumn(const IColumn & column) -{ - return typeid_cast(&column) || typeid_cast(&column); -} - template bool FunctionArrayElement::matchKeyToIndexStringConst( const IColumn & data, const Offsets & offsets, const Field & index, PaddedPODArray & matched_idxs) @@ -1933,37 +1940,27 @@ bool FunctionArrayElement::matchKeyToIndexStringConst( if (index.getType() != Field::Types::String) return false; - /// The dictionary lookup below is defined only for String and FixedString keys. For other - /// LowCardinality key types, fall through so that the regular dispatch reports the type error - /// instead of silently finding no match. - const auto * low_cardinality_data = typeid_cast(&data); - if (low_cardinality_data - && isStringOrFixedStringColumn(*low_cardinality_data->getDictionary().getNestedNotNullableColumn())) + if (const auto * low_cardinality_data = typeid_cast(&data)) { - const auto & requested_key = index.safeGet(); - auto dictionary_index = low_cardinality_data->getDictionary().getOrFindValueIndex(requested_key); matched_idxs.reserve(offsets.size()); - if (!dictionary_index) + auto lookup_result = callWithLowCardinalityValueIndex( + *low_cardinality_data, + index.safeGet(), + [&](const auto * indexes, auto key_index) + { + MatcherLowCardinalityConst matcher{indexes, key_index}; + executeMatchKeyToIndex(offsets, matched_idxs, matcher); + }); + + /// For LowCardinality key types without a dictionary lookup, fall through so that the regular + /// dispatch reports the type error instead of silently finding no match. + if (lookup_result != LowCardinalityValueLookupResult::Unsupported) { - matched_idxs.resize_fill(offsets.size()); + if (lookup_result == LowCardinalityValueLookupResult::NotFound) + matched_idxs.resize_fill(offsets.size()); return true; } - - struct MatcherLowCardinalityStringConst - { - const ColumnLowCardinality & data; - UInt64 dictionary_index; - - bool match(size_t row_data, size_t /* row_index */) const - { - return data.getIndexAt(row_data) == dictionary_index; - } - }; - - MatcherLowCardinalityStringConst matcher{*low_cardinality_data, *dictionary_index}; - executeMatchKeyToIndex(offsets, matched_idxs, matcher); - return true; } return castColumnString( diff --git a/tests/performance/map_lowcardinality_key_lookup.xml b/tests/performance/map_lowcardinality_key_lookup.xml new file mode 100644 index 0000000000000..ee61624b8bf31 --- /dev/null +++ b/tests/performance/map_lowcardinality_key_lookup.xml @@ -0,0 +1,43 @@ + + + + + CREATE TABLE map_lc_keys (id UInt64, m Map(LowCardinality(String), String)) + ENGINE = MergeTree ORDER BY id + + + + INSERT INTO map_lc_keys + SELECT number, mapFromArrays( + arrayMap(x -> concat('k', leftPad(toString(x), 2, '0')), range(20)), + arrayMap(x -> concat('v', toString(cityHash64(number, x) % 1000)), range(20))) + FROM numbers(1500000) + + + SELECT sum(ignore(m['k00'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['k09'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['k19'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['absent'])) FROM map_lc_keys SETTINGS max_threads = 1 + + DROP TABLE map_lc_keys + + + + + CREATE TABLE map_uuid_keys (id UInt64, m Map(UUID, DateTime64(3))) + ENGINE = MergeTree ORDER BY id + + + + INSERT INTO map_uuid_keys + SELECT number, mapFromArrays( + arrayMap(x -> reinterpretAsUUID(toFixedString(leftPad(toString(x), 16, '0'), 16)), range(20)), + arrayMap(x -> toDateTime64(number + x, 3), range(20))) + FROM numbers(1000000) + + + SELECT sum(ignore(m[reinterpretAsUUID(toFixedString('0000000000000019', 16))])) FROM map_uuid_keys SETTINGS max_threads = 1 + + DROP TABLE map_uuid_keys + diff --git a/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference new file mode 100644 index 0000000000000..06f2d273f643e --- /dev/null +++ b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference @@ -0,0 +1,102 @@ +present key, subcolumns=0 1 b1 +present key, subcolumns=0 2 +present key, subcolumns=0 3 +present key, subcolumns=0 4 FIRST +present key, subcolumns=1 1 b1 +present key, subcolumns=1 2 +present key, subcolumns=1 3 +present key, subcolumns=1 4 FIRST +absent key, subcolumns=0 1 +absent key, subcolumns=0 2 +absent key, subcolumns=0 3 +absent key, subcolumns=0 4 +absent key, subcolumns=1 1 +absent key, subcolumns=1 2 +absent key, subcolumns=1 3 +absent key, subcolumns=1 4 +empty key, subcolumns=0 1 +empty key, subcolumns=0 2 +empty key, subcolumns=0 3 +empty key, subcolumns=0 4 empty +empty key, subcolumns=1 1 +empty key, subcolumns=1 2 +empty key, subcolumns=1 3 +empty key, subcolumns=1 4 empty +direct subcolumn 1 b1 +direct subcolumn 2 +direct subcolumn 3 +direct subcolumn 4 FIRST +compact, subcolumns=0 1 b1 +compact, subcolumns=0 2 +compact, subcolumns=0 3 +compact, subcolumns=0 4 FIRST +compact, subcolumns=1 1 b1 +compact, subcolumns=1 2 +compact, subcolumns=1 3 +compact, subcolumns=1 4 FIRST +fixed string key, subcolumns=0 1 b1 +fixed string key, subcolumns=0 2 +fixed string key, subcolumns=1 1 b1 +fixed string key, subcolumns=1 2 +wide map, subcolumns=0 17633783096591347703 10736917531793588130 +wide map, subcolumns=1 17633783096591347703 10736917531793588130 +high dictionary position, subcolumns=0 100000 100 +high dictionary position, subcolumns=1 100000 100 +random maps, subcolumns=0 9043450906938593318 +random maps, subcolumns=1 9043450906938593318 +UUID key, subcolumns=0 1 hit +UUID key, subcolumns=0 2 +UUID key, subcolumns=1 1 hit +UUID key, subcolumns=1 2 +IPv4 key, subcolumns=0 1 hit +IPv4 key, subcolumns=0 2 +IPv4 key, subcolumns=1 1 hit +IPv4 key, subcolumns=1 2 +IPv6 key, subcolumns=0 1 hit +IPv6 key, subcolumns=0 2 +IPv6 key, subcolumns=1 1 hit +IPv6 key, subcolumns=1 2 +Int128 key, subcolumns=0 1 hit +Int128 key, subcolumns=0 2 +Int128 key, subcolumns=1 1 hit +Int128 key, subcolumns=1 2 +UInt256 key, subcolumns=0 1 hit +UInt256 key, subcolumns=0 2 +UInt256 key, subcolumns=1 1 hit +UInt256 key, subcolumns=1 2 +Decimal64 key subcolumn 1 hit +Decimal64 key subcolumn 2 +DateTime64 key subcolumn 1 hit +DateTime64 key subcolumn 2 +UUID value, subcolumns=0 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +UUID value, subcolumns=0 2 00000000-0000-0000-0000-000000000000 +UUID value, subcolumns=1 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +UUID value, subcolumns=1 2 00000000-0000-0000-0000-000000000000 +IPv6 value, subcolumns=0 1 ::1 +IPv6 value, subcolumns=0 2 :: +IPv6 value, subcolumns=1 1 ::1 +IPv6 value, subcolumns=1 2 :: +Int128 value, subcolumns=0 1 -123456789012345678901234567890 +Int128 value, subcolumns=0 2 0 +Int128 value, subcolumns=1 1 -123456789012345678901234567890 +Int128 value, subcolumns=1 2 0 +Decimal64 value, subcolumns=0 1 1.25 +Decimal64 value, subcolumns=0 2 0 +Decimal64 value, subcolumns=1 1 1.25 +Decimal64 value, subcolumns=1 2 0 +DateTime64 value, subcolumns=0 1 2020-01-01 00:00:00.500 +DateTime64 value, subcolumns=0 2 1970-01-01 00:00:00.000 +DateTime64 value, subcolumns=1 1 2020-01-01 00:00:00.500 +DateTime64 value, subcolumns=1 2 1970-01-01 00:00:00.000 +Nullable(UUID) value, subcolumns=0 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +Nullable(UUID) value, subcolumns=0 2 \N +Nullable(UUID) value, subcolumns=0 3 \N +Nullable(UUID) value, subcolumns=1 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +Nullable(UUID) value, subcolumns=1 2 \N +Nullable(UUID) value, subcolumns=1 3 \N +Nullable(Decimal64) value, subcolumns=0 1 1.25 +Nullable(Decimal64) value, subcolumns=0 2 \N +Nullable(Decimal64) value, subcolumns=0 3 \N +Nullable(Decimal64) value, subcolumns=1 1 1.25 +Nullable(Decimal64) value, subcolumns=1 2 \N +Nullable(Decimal64) value, subcolumns=1 3 \N diff --git a/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql new file mode 100644 index 0000000000000..925533b788e1d --- /dev/null +++ b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql @@ -0,0 +1,203 @@ +-- m[key] on a Map with LowCardinality keys is resolved through dictionary positions instead of +-- comparing the key values. The result must be the same as with the generic comparison, and the +-- same for both the arrayElement path (optimize_functions_to_subcolumns = 0) and the subcolumn +-- path (optimize_functions_to_subcolumns = 1). + +DROP TABLE IF EXISTS t_map_lc; + +CREATE TABLE t_map_lc (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +-- Row 2 has no 'b' at all, row 3 is an empty map, row 4 has 'b' twice and '' as a key. +INSERT INTO t_map_lc VALUES (1, map('a', 'a1', 'b', 'b1')), (2, map('a', 'a2')), (3, map()), (4, map('', 'empty', 'b', 'FIRST', 'b', 'SECOND')); + +SELECT 'present key, subcolumns=0', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'present key, subcolumns=1', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- A key that is in no row is absent from the dictionary as well, and is resolved without +-- looking at the rows. +SELECT 'absent key, subcolumns=0', id, m['zz'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'absent key, subcolumns=1', id, m['zz'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- The empty string is kept in the reserved default position of the dictionary, not in its index. +SELECT 'empty key, subcolumns=0', id, m[''] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'empty key, subcolumns=1', id, m[''] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- Reading the subcolumn directly goes through the same key lookup. +SELECT 'direct subcolumn', id, m.key_b FROM t_map_lc ORDER BY id; + +DROP TABLE t_map_lc; + +-- Compact parts exercise the other reader path. +CREATE TABLE t_map_lc (id UInt64, m Map(LowCardinality(String), String)) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_map_lc VALUES (1, map('a', 'a1', 'b', 'b1')), (2, map('a', 'a2')), (3, map()), (4, map('', 'empty', 'b', 'FIRST', 'b', 'SECOND')); + +SELECT 'compact, subcolumns=0', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'compact, subcolumns=1', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc; + +-- LowCardinality(FixedString) keys use the same dictionary lookup. +CREATE TABLE t_map_lc_fixed (id UInt64, m Map(LowCardinality(FixedString(3)), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_lc_fixed VALUES (1, map('aaa', 'a1', 'bbb', 'b1')), (2, map('aaa', 'a2')); + +SELECT 'fixed string key, subcolumns=0', id, m['bbb'], m['zzz'] FROM t_map_lc_fixed ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'fixed string key, subcolumns=1', id, m['bbb'], m['zzz'] FROM t_map_lc_fixed ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_fixed; + +-- A wide map, where the looked up key is the last one in every row: the shape the dictionary +-- lookup is meant to make cheap. +DROP TABLE IF EXISTS t_map_lc_wide; + +CREATE TABLE t_map_lc_wide (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_wide +SELECT number, mapFromArrays(arrayMap(x -> concat('k', leftPad(toString(x), 2, '0')), range(20)), arrayMap(x -> concat('v', toString(number), '_', toString(x)), range(20))) +FROM numbers(1000); + +SELECT 'wide map, subcolumns=0', sum(cityHash64(m['k19'])), sum(cityHash64(m['k00'])) FROM t_map_lc_wide SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'wide map, subcolumns=1', sum(cityHash64(m['k19'])), sum(cityHash64(m['k00'])) FROM t_map_lc_wide SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_wide; + +-- A key at a high dictionary position looked up over rows that do not contain it. The dictionary of +-- a part is shared by all of its blocks, so a dictionary position can be out of the range of the +-- index type of an individual block, and must not be truncated into a false match. +DROP TABLE IF EXISTS t_map_lc_many_keys; + +CREATE TABLE t_map_lc_many_keys (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_many_keys +SELECT number, map(concat('k', leftPad(toString(number % 1000), 4, '0')), concat('v', toString(number))) +FROM numbers(100000); + +SELECT 'high dictionary position, subcolumns=0', count(), countIf(m['k0999'] != '') FROM t_map_lc_many_keys SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'high dictionary position, subcolumns=1', count(), countIf(m['k0999'] != '') FROM t_map_lc_many_keys SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_many_keys; + +-- Pseudo-random wide maps with duplicate keys: both paths must agree, which is the property that +-- the first-match fix of issue #111203 established. +DROP TABLE IF EXISTS t_map_lc_random; + +CREATE TABLE t_map_lc_random (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_random +SELECT + number, + mapFromArrays( + arrayMap(x -> concat('k', toString(cityHash64(number, x) % 12)), range(1 + (number % 15))), + arrayMap(x -> concat('v', toString(cityHash64(number, x, 'value'))), range(1 + (number % 15)))) +FROM numbers(20000); + +SELECT 'random maps, subcolumns=0', sum(cityHash64(m['k0'], m['k5'], m['k11'], m['k99'])) FROM t_map_lc_random SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'random maps, subcolumns=1', sum(cityHash64(m['k0'], m['k5'], m['k11'], m['k99'])) FROM t_map_lc_random SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_random; + +-- Key types that used to be compared through virtual compareAt. + +DROP TABLE IF EXISTS t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(UUID, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 'hit')), (2, map('00000000-0000-0000-0000-000000000001', 'other')); +SELECT 'UUID key, subcolumns=0', id, m[toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UUID key, subcolumns=1', id, m[toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(IPv4, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('1.2.3.4', 'hit')), (2, map('5.6.7.8', 'other')); +SELECT 'IPv4 key, subcolumns=0', id, m[toIPv4('1.2.3.4')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv4 key, subcolumns=1', id, m[toIPv4('1.2.3.4')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(IPv6, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('::1', 'hit')), (2, map('::2', 'other')); +SELECT 'IPv6 key, subcolumns=0', id, m[toIPv6('::1')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv6 key, subcolumns=1', id, m[toIPv6('::1')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +-- The wide integers are built from strings, because a literal that wide goes through Float64 and +-- loses precision. +CREATE TABLE t_map_key_types (id UInt64, m Map(Int128, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toInt128('-123456789012345678901234567890'), 'hit'); +INSERT INTO t_map_key_types SELECT 2, map(toInt128(1), 'other'); +SELECT 'Int128 key, subcolumns=0', id, m[toInt128('-123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Int128 key, subcolumns=1', id, m[toInt128('-123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(UInt256, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toUInt256('12345678901234567890123456789012345678901234567890'), 'hit'); +INSERT INTO t_map_key_types SELECT 2, map(toUInt256(1), 'other'); +SELECT 'UInt256 key, subcolumns=0', id, m[toUInt256('12345678901234567890123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UInt256 key, subcolumns=1', id, m[toUInt256('12345678901234567890123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +-- Decimal and DateTime64 keys are rejected by m[key], but reachable through the subcolumn name. + +CREATE TABLE t_map_key_types (id UInt64, m Map(Decimal64(2), String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toDecimal64(1.5, 2), 'hit', toDecimal64(2.5, 2), 'other'); +INSERT INTO t_map_key_types SELECT 2, map(toDecimal64(2.5, 2), 'other'); +SELECT 'Decimal64 key subcolumn', id, `m.key_1.50` FROM t_map_key_types ORDER BY id; +DROP TABLE t_map_key_types; + +-- The timezone is pinned, because the key is rendered into the subcolumn name. +CREATE TABLE t_map_key_types (id UInt64, m Map(DateTime64(3, 'UTC'), String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toDateTime64('2020-01-01 00:00:00.500', 3, 'UTC'), 'hit', toDateTime64('2021-01-01 00:00:00.500', 3, 'UTC'), 'other'); +INSERT INTO t_map_key_types SELECT 2, map(toDateTime64('2021-01-01 00:00:00.500', 3, 'UTC'), 'other'); +SELECT 'DateTime64 key subcolumn', id, `m.key_2020-01-01 00:00:00.500` FROM t_map_key_types ORDER BY id; +DROP TABLE t_map_key_types; + +-- Value types that used to be copied through virtual insertFrom. The rows not holding the key +-- cover the default-insert branch, the Nullable variants the null map. + +DROP TABLE IF EXISTS t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, UUID)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '61f0c404-5cb3-11e7-907b-a6006ad3dba0')), (2, map('b', '00000000-0000-0000-0000-000000000001')); +SELECT 'UUID value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UUID value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, IPv6)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '::1')), (2, map('b', '::2')); +SELECT 'IPv6 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv6 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Int128)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types SELECT 1, map('a', toInt128('-123456789012345678901234567890')); +INSERT INTO t_map_value_types SELECT 2, map('b', toInt128(1)); +SELECT 'Int128 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Int128 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Decimal64(3))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', 1.25)), (2, map('b', 2.5)); +SELECT 'Decimal64 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Decimal64 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +-- The timezone is pinned, because the missing key renders the epoch as the default value. +CREATE TABLE t_map_value_types (id UInt64, m Map(String, DateTime64(3, 'UTC'))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '2020-01-01 00:00:00.500')), (2, map('b', '2021-01-01 00:00:00.500')); +SELECT 'DateTime64 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'DateTime64 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Nullable(UUID))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '61f0c404-5cb3-11e7-907b-a6006ad3dba0')), (2, map('a', NULL)), (3, map('b', NULL)); +SELECT 'Nullable(UUID) value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Nullable(UUID) value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Nullable(Decimal64(3)))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', 1.25)), (2, map('a', NULL)), (3, map('b', NULL)); +SELECT 'Nullable(Decimal64) value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Nullable(Decimal64) value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; diff --git a/tests/queries/0_stateless/05234_map_key_lookup_nan.reference b/tests/queries/0_stateless/05234_map_key_lookup_nan.reference new file mode 100644 index 0000000000000..62d2b83565c4d --- /dev/null +++ b/tests/queries/0_stateless/05234_map_key_lookup_nan.reference @@ -0,0 +1,22 @@ +Float64 1 nan-hit one-five +Float64 2 other +Float64 3 plain +Float32 1 nan-hit one-five +Float32 2 other +Float32 3 plain +BFloat16 1 nan-hit one-five +BFloat16 2 other +BFloat16 3 plain +LowCardinality(Float64) 1 nan-hit one-five +LowCardinality(Float64) 2 other +LowCardinality(Float64) 3 plain +Tuple(Float64, UInt8) 1 nan-hit one-five +Tuple(Float64, UInt8) 2 other +Tuple(Float64, UInt8) 3 plain +Tuple(Nullable(UInt8), UInt8) 1 null-hit five +Tuple(Nullable(UInt8), UInt8) 2 other +Tuple(Nullable(UInt8), UInt8) 3 plain +NaN value 1 1 1.5 +NaN value 2 0 2.5 +NaN value 1 1 1.5 +NaN value 2 0 2.5 diff --git a/tests/queries/0_stateless/05234_map_key_lookup_nan.sql b/tests/queries/0_stateless/05234_map_key_lookup_nan.sql new file mode 100644 index 0000000000000..bfcb9deffeee4 --- /dev/null +++ b/tests/queries/0_stateless/05234_map_key_lookup_nan.sql @@ -0,0 +1,61 @@ +-- A `NaN` key of a Map is found by a requested `NaN` and by nothing else, and a requested `NaN` +-- finds nothing but a `NaN` key. The subcolumn extractor used to ask `compareAt` with a direction +-- hint of zero, which answers with the hint itself when one side is a `NaN`, so a row holding a +-- `NaN` key matched every requested key and a requested `NaN` matched every key. + +SET allow_suspicious_low_cardinality_types = 1; + +DROP TABLE IF EXISTS t_map_nan_key; + +-- The specialized matchers: BFloat16, Float32 and Float64 keys are compared value by value. + +CREATE TABLE t_map_nan_key (id UInt64, m Map(Float64, String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_nan_key VALUES (1, map(nan, 'nan-hit', 1.5, 'one-five')), (2, map(2.5, 'other')), (3, map(1.5, 'plain')); +SELECT 'Float64', id, `m.key_nan`, `m.key_1.5`, `m.key_2.5` FROM t_map_nan_key ORDER BY id; +DROP TABLE t_map_nan_key; + +CREATE TABLE t_map_nan_key (id UInt64, m Map(Float32, String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_nan_key VALUES (1, map(nan, 'nan-hit', 1.5, 'one-five')), (2, map(2.5, 'other')), (3, map(1.5, 'plain')); +SELECT 'Float32', id, `m.key_nan`, `m.key_1.5`, `m.key_2.5` FROM t_map_nan_key ORDER BY id; +DROP TABLE t_map_nan_key; + +CREATE TABLE t_map_nan_key (id UInt64, m Map(BFloat16, String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_nan_key VALUES (1, map(nan, 'nan-hit', 1.5, 'one-five')), (2, map(2.5, 'other')), (3, map(1.5, 'plain')); +SELECT 'BFloat16', id, `m.key_nan`, `m.key_1.5`, `m.key_2.5` FROM t_map_nan_key ORDER BY id; +DROP TABLE t_map_nan_key; + +-- The generic matcher: a LowCardinality dictionary that does not hold strings, and a Tuple, are +-- compared through the virtual `compareAt`. + +CREATE TABLE t_map_nan_key (id UInt64, m Map(LowCardinality(Float64), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_nan_key VALUES (1, map(nan, 'nan-hit', 1.5, 'one-five')), (2, map(2.5, 'other')), (3, map(1.5, 'plain')); +SELECT 'LowCardinality(Float64)', id, `m.key_nan`, `m.key_1.5`, `m.key_2.5` FROM t_map_nan_key ORDER BY id; +DROP TABLE t_map_nan_key; + +CREATE TABLE t_map_nan_key (id UInt64, m Map(Tuple(Float64, UInt8), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_nan_key VALUES (1, map((nan, 1), 'nan-hit', (1.5, 1), 'one-five')), (2, map((2.5, 1), 'other')), (3, map((1.5, 1), 'plain')); +SELECT 'Tuple(Float64, UInt8)', id, `m.key_(nan,1)`, `m.key_(1.5,1)`, `m.key_(2.5,1)` FROM t_map_nan_key ORDER BY id; +DROP TABLE t_map_nan_key; + +-- The same hint decides where a NULL goes, so a NULL inside a composite key used to be equal to +-- every other value as well. + +CREATE TABLE t_map_nan_key (id UInt64, m Map(Tuple(Nullable(UInt8), UInt8), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_nan_key VALUES (1, map((NULL, 1), 'null-hit', (5, 1), 'five')), (2, map((7, 1), 'other')), (3, map((5, 1), 'plain')); +SELECT 'Tuple(Nullable(UInt8), UInt8)', id, `m.key_(NULL,1)`, `m.key_(5,1)`, `m.key_(7,1)` FROM t_map_nan_key ORDER BY id; +DROP TABLE t_map_nan_key; + +-- A `NaN` in the value is copied as it is, whichever key it is reached by. + +CREATE TABLE t_map_nan_key (id UInt64, m Map(String, Float64)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_nan_key VALUES (1, map('a', nan, 'b', 1.5)), (2, map('b', 2.5)); +SELECT 'NaN value', id, isNaN(m['a']), m['b'] FROM t_map_nan_key ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'NaN value', id, isNaN(m['a']), m['b'] FROM t_map_nan_key ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_nan_key; From fec0ce2bc4e25b3a7255d95552c088435a757c03 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 12:20:51 +0000 Subject: [PATCH 100/296] Backport #121375 to 26.8: Use text and token skip indexes for IN with transform_null_in = 1 (Human alternative for #112785) --- .../MergeTreeIndexBloomFilterText.cpp | 22 +-- .../MergeTree/MergeTreeIndexConditionText.cpp | 47 +++++- ...xt_token_index_transform_null_in.reference | 35 +++++ ...346_text_token_index_transform_null_in.sql | 136 ++++++++++++++++++ 4 files changed, 223 insertions(+), 17 deletions(-) create mode 100644 tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference create mode 100644 tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp index c36427c44164b..d9d478b11dfbb 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp @@ -387,16 +387,11 @@ bool MergeTreeConditionBloomFilterText::extractAtomFromTree(const RPNBuilderTree { if (tryPrepareSetBloomFilter(left_argument, right_argument, out)) { - if (function_name == "notIn") - { - out.function = RPNElement::FUNCTION_NOT_IN; - return true; - } - if (function_name == "in") - { - out.function = RPNElement::FUNCTION_IN; - return true; - } + /// `transform_null_in = 1` renames the family; a NULL element is refused above. + const bool negated = function_name == "notIn" || function_name == "globalNotIn" + || function_name == "notNullIn" || function_name == "globalNotNullIn"; + out.function = negated ? RPNElement::FUNCTION_NOT_IN : RPNElement::FUNCTION_IN; + return true; } } else if (function_name == "equals" || @@ -807,7 +802,8 @@ bool MergeTreeConditionBloomFilterText::tryPrepareSetBloomFilter( for (const auto & prepared_set_data_type : prepared_set->getDataTypes()) { - auto prepared_set_data_type_id = prepared_set_data_type->getTypeId(); + /// A `Nullable` key keeps the wrapper on its elements at `transform_null_in = 1`. + auto prepared_set_data_type_id = removeNullable(prepared_set_data_type)->getTypeId(); if (prepared_set_data_type_id != TypeIndex::String && prepared_set_data_type_id != TypeIndex::FixedString) return false; } @@ -828,6 +824,10 @@ bool MergeTreeConditionBloomFilterText::tryPrepareSetBloomFilter( for (size_t row = 0; row < prepared_set_total_row_count; ++row) { + /// A NULL element also matches the column's NULL rows, which the filter cannot express. + if (column->isNullAt(row)) + return false; + bloom_filters.back().emplace_back(params); auto ref = column->getDataAt(row); forEachTokenToBloomFilter(*tokenizer, ref.data(), ref.size(), bloom_filters.back().back()); diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index d8c5094ce71c2..d64931e0ed45c 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -650,7 +651,8 @@ bool MergeTreeIndexConditionText::traverseAtomNode(const RPNBuilderTreeNode & no auto lhs_argument = function.getArgumentAt(0); auto rhs_argument = function.getArgumentAt(1); - if ((function_name == "in" || function_name == "globalIn") + if ((function_name == "in" || function_name == "globalIn" + || function_name == "nullIn" || function_name == "globalNullIn") && tryPrepareSetForTextSearch(lhs_argument, rhs_argument, function_name, out)) { out.function = RPNElement::FUNCTION_HAS_ANY_ELEMENTS; @@ -894,6 +896,19 @@ static void validateRegexpPatterns(const Array & patterns, const Settings & sett #endif } +/// The value an absent map key reads: `''`, or all NUL when the value type is `FixedString`. +/// `mapValues` stores neither. +static bool isMapValueDefault(std::string_view value, const Block & header) +{ + /// A text index is always defined on a single expression. + chassert(header.columns() == 1); + auto value_type = removeNullable(removeLowCardinality(header.getByPosition(0).type)); + if (const auto * array_type = typeid_cast(value_type.get())) + value_type = removeNullable(removeLowCardinality(array_type->getNestedType())); + + return value.empty() || (isFixedString(value_type) && value.find_first_not_of('\0') == std::string_view::npos); +} + bool MergeTreeIndexConditionText::traverseFunctionNode( const RPNBuilderFunctionTreeNode & function_node, const RPNBuilderTreeNode & index_column_node, @@ -943,7 +958,7 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( auto & [map_column_name, _] = *parsed; if (header.has(fmt::format("mapValues({})", map_column_name)) && value_field.getType() == Field::Types::String - && !value_field.safeGet().empty()) + && !isMapValueDefault(value_field.safeGet(), header)) { has_index_column = true; direct_read_mode = getHintOrNoneMode(); @@ -1641,7 +1656,7 @@ bool MergeTreeIndexConditionText::traverseMapElementValueNode(const RPNBuilderTr /// for functions like `func(arrayElement(m, 'const_key'), ...)`. /// If index can be used, than we can analyze the index as for scalar string column /// because `arrayElement(m, 'const_key')` projects Array(String) to String. - if (const_value.getType() != Field::Types::String || const_value.safeGet().empty()) + if (const_value.getType() != Field::Types::String || isMapValueDefault(const_value.safeGet(), header)) return false; return hasIndexForMapElementValue(index_column_node); @@ -1721,10 +1736,17 @@ bool MergeTreeIndexConditionText::tryPrepareSetForTextSearch( { std::optional set_key_position; + /// `m['key']` answered by a `mapValues(m)` index: an absent key reads the value type's default. + bool has_index_for_map_element_value = false; + auto has_index = [&](const RPNBuilderTreeNode & node) { + if (hasIndexForMapElementValue(node)) + { + has_index_for_map_element_value = true; + return true; + } return hasIndexForColumn(node.getColumnName()) - || hasIndexForMapElementValue(node) || tryMatchNodeToJSONIndex(node, header, "JSONAllValues"); }; @@ -1773,19 +1795,32 @@ bool MergeTreeIndexConditionText::tryPrepareSetForTextSearch( return false; const auto & set_column = *columns[*set_key_position]; - if (!WhichDataType(set_column.getDataType()).isStringOrFixedString()) + + /// With setting `transform_null_in = 1`, the IN set can be nullable. + const auto * set_column_nullable = typeid_cast(&set_column); + const auto & set_column_values = set_column_nullable ? set_column_nullable->getNestedColumn() : set_column; + + if (!WhichDataType(set_column_values.getDataType()).isStringOrFixedString()) return false; size_t total_row_count = prepared_set->getTotalRowCount(); for (size_t row = 0; row < total_row_count; ++row) { + /// The atom is an OR over the elements, and the index skips NULL rows when building a + /// granule, so a NULL element is a disjunct it cannot bind. Decline the atom. + if (set_column.isNullAt(row)) + { + out.text_search_queries.clear(); + return false; + } + auto ref = set_column.getDataAt(row); /// Reject the index usage when there is an empty string in the set. /// The condition with such a predicate will be always true on granule. /// See MergeTreeIndexGranuleText::hasAllQueryTokensOrEmpty. - if (ref.empty()) + if (ref.empty() || (has_index_for_map_element_value && isMapValueDefault(ref, header))) { out.text_search_queries.clear(); return false; diff --git a/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference new file mode 100644 index 0000000000000..7f82e6a6845d9 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference @@ -0,0 +1,35 @@ +Granules: 1/2 +1 +1 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +1 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +Granules: 1/2 +2 +3 +1 +1 +2 +2 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql new file mode 100644 index 0000000000000..6127b05004cdb --- /dev/null +++ b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql @@ -0,0 +1,136 @@ +-- Tests that the `text`, `tokenbf_v1`, `ngrambf_v1` and `sparse_grams` skip indexes prune an `IN` +-- when `transform_null_in = 1`, where the predicate arrives as `nullIn`/`globalNullIn`, and that a +-- set holding a NULL element is refused because `nullIn` also matches the column's NULL rows. +-- +-- `index_granularity = 4` over 8 rows, so every granule is mixed: granule 0 holds word0..word3. + +SET enable_full_text_index = 1; +SET transform_null_in = 1; + +DROP TABLE IF EXISTS tab; + +-- The same block runs for every index type. `tokenbf_v1`, `ngrambf_v1` and `sparse_grams` reject a +-- `Nullable` column at DDL, so the common block uses `String` and NULL enters through the set only. + +-- text + +CREATE TABLE tab (s String, INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- The text index does not model NOT IN at all, unlike the token bloom filter family below. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- tokenbf_v1 + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE tokenbf_v1(256, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- ngrambf_v1 + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE ngrambf_v1(3, 256, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- A FixedString element has its NUL padding stripped before tokenization, so it keeps pruning. +SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12)) SETTINGS force_data_skipping_indices = 'idx'; +SELECT (SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12))) = (SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12)) SETTINGS use_skip_indexes = 0); + +-- sparse_grams + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE sparse_grams(3, 100, 512, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- A Nullable column: only the text index accepts one, and only there can a NULL element of the set +-- meet a NULL row of the column. The set is a disjunction, so one NULL element refuses the index +-- whatever else the set holds. + +DROP TABLE tab; +CREATE TABLE tab (s Nullable(String), INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT if(number = 7, NULL, 'word' || toString(number)) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s IN ('word1', NULL) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE s IN ('word1', NULL); +SELECT count() FROM tab WHERE s IN ('word1', NULL, 'word2') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE s IN ('word1', NULL, 'word2'); + +-- A tuple set is one Tuple column unpacked by position, so the indexed component is addressed by it. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, s String, INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 4; +INSERT INTO tab SELECT number, 'word' || toString(number) FROM numbers(8); + +SELECT count() FROM tab WHERE (id, s) IN ((1, 'word1')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE (id, s) IN (SELECT tuple(number, 'word1') FROM numbers(8)) SETTINGS force_data_skipping_indices = 'idx'; + +-- An absent map key reads the value type's default, which mapValues never stores. The FixedString default +-- is all NUL and the array tokenizer keeps the padding, so it must be recognised as the default. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, m Map(String, FixedString(4)), INDEX idx mapValues(m) TYPE text(tokenizer = array)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; +INSERT INTO tab VALUES (0, {'k':'val0'}), (1, {'other':'xxxx'}), (2, {'k':'val2'}), (3, {}); + +SELECT count() FROM tab WHERE m['k'] IN (toFixedString('', 4)) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] IN (toFixedString('', 4)); +SELECT count() FROM tab WHERE m['k'] = toFixedString('', 4) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] = toFixedString('', 4); +SELECT count() FROM tab WHERE m['k'] IN ('val0') SETTINGS force_data_skipping_indices = 'idx'; + +-- A String map value defaults to '' only, so an all-NUL value is a real value and must prune. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, m Map(String, String), INDEX idx mapValues(m) TYPE text(tokenizer = array)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; +INSERT INTO tab SELECT 0, map('k', unhex('00')); +INSERT INTO tab SELECT number, map('k', 'val' || toString(number)) FROM numbers(1, 2); +INSERT INTO tab SELECT 3, map('other', 'x'); + +SELECT count() FROM tab WHERE m['k'] = unhex('00') SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE m['k'] IN (unhex('00')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE m['k'] = '' SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] IN ('') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] = ''; + +DROP TABLE tab; From 7b3ca31f99fd6ec8d2eaf0a983006dea8df72956 Mon Sep 17 00:00:00 2001 From: Jimmy Aguilar Mena Date: Fri, 25 Sep 2026 13:29:37 +0000 Subject: [PATCH 101/296] Add missing includes --- src/Storages/MergeTree/MergeTreeIndexConditionText.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index d64931e0ed45c..1640e2b731c19 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include From c8e1f583abbb0a0581f550eee23add17d1a1a658 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 13:49:46 +0000 Subject: [PATCH 102/296] Backport #111287 to 26.8: Fix double free when finalizing -State aggregates under looping combinators --- .../Combinators/AggregateFunctionArray.h | 5 + ...gregateFunctionCombinatorsArgMinArgMax.cpp | 5 + .../Combinators/AggregateFunctionDistinct.h | 5 + .../Combinators/AggregateFunctionForEach.h | 57 +++++++- .../Combinators/AggregateFunctionIf.h | 5 + .../Combinators/AggregateFunctionMap.cpp | 132 ++++++++++++++---- .../Combinators/AggregateFunctionMerge.h | 5 + .../Combinators/AggregateFunctionNull.h | 39 +++++- .../Combinators/AggregateFunctionOrFill.h | 63 +++++++-- .../Combinators/AggregateFunctionResample.h | 71 +++++++++- .../AggregateFunctionSimpleState.h | 5 + .../Combinators/AggregateFunctionState.h | 31 +++- .../Combinators/AggregateFunctionTuple.cpp | 7 + .../Combinators/AggregateFunctionTuple.h | 65 ++++++++- src/AggregateFunctions/IAggregateFunction.h | 9 ++ src/Columns/ColumnAggregateFunction.cpp | 8 ++ src/Columns/ColumnAggregateFunction.h | 16 +++ src/Common/FailPoint.cpp | 4 +- ...state_resample_finalize_no_crash.reference | 14 ++ ...array_state_resample_finalize_no_crash.sql | 102 ++++++++++++++ 20 files changed, 590 insertions(+), 58 deletions(-) create mode 100644 tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference create mode 100644 tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h index 614bf5aa1470a..a98b076c532f1 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h @@ -231,6 +231,11 @@ class AggregateFunctionArray final : public IAggregateFunctionHelperinsertMergeResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(place, to); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionCombinatorsArgMinArgMax.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionCombinatorsArgMinArgMax.cpp index 2dd9f6bfe11f1..f73eed07a9abc 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionCombinatorsArgMinArgMax.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionCombinatorsArgMinArgMax.cpp @@ -222,6 +222,11 @@ class AggregateFunctionCombinatorArgMinArgMax final : public IAggregateFunctionH nested_function->insertMergeResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_function->rollbackInsertResult(place, to); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } }; diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h b/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h index 69c2af03fef17..0368843766814 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h @@ -291,6 +291,11 @@ class AggregateFunctionDistinct final : public IAggregateFunctionDataHelperinsertMergeResultInto(getNestedPlace(place), to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(getNestedPlace(place), to); + } + size_t sizeOfData() const override { return prefix_size + nested_func->sizeOfData(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionForEach.h b/src/AggregateFunctions/Combinators/AggregateFunctionForEach.h index 219c94ec9cd31..17b499acd3bf6 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionForEach.h @@ -342,17 +342,15 @@ class AggregateFunctionForEach final : public IAggregateFunctionDataHelper - void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + void transferElements(AggregateDataPtr __restrict place, ColumnArray & arr_to, size_t & transferred, Arena * arena) const { AggregateFunctionForEachData & state = data(place); - - ColumnArray & arr_to = assert_cast(to); - ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); IColumn & elems_to = arr_to.getData(); - char * nested_state = state.array_of_aggregate_datas; - for (size_t i = 0; i < state.dynamic_array_size; ++i) + char * nested_state = state.array_of_aggregate_datas + transferred * nested_size_of_data; + for (; transferred < state.dynamic_array_size; ++transferred) { if constexpr (merge) nested_func->insertMergeResultInto(nested_state, elems_to, arena); @@ -361,9 +359,43 @@ class AggregateFunctionForEach final : public IAggregateFunctionDataHelper + void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + { + ColumnArray & arr_to = assert_cast(to); + size_t transferred = 0; + + if constexpr (!merge) + { + /// A nested function that is not a state aliases nothing and need not be atomic. + if (nested_func->isState()) + { + const size_t offsets_before = arr_to.getOffsets().size(); + + try + { + transferElements(place, arr_to, transferred, arena); + } + catch (...) + { + arr_to.getOffsets().resize_assume_reserved(offsets_before); + const char * nested_state = data(place).array_of_aggregate_datas; + for (size_t i = transferred; i-- > 0;) + nested_func->rollbackInsertResult(nested_state + i * nested_size_of_data, arr_to.getData()); + throw; + } + + return; + } + } + + transferElements(place, arr_to, transferred, arena); + } + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const override { insertResultIntoImpl(place, to, arena); @@ -374,6 +406,19 @@ class AggregateFunctionForEach final : public IAggregateFunctionDataHelper(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + const AggregateFunctionForEachData & state = data(place); + + ColumnArray & arr_to = assert_cast(to); + ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); + + offsets_to.resize_assume_reserved(offsets_to.size() - 1); + const char * nested_state = state.array_of_aggregate_datas; + for (size_t i = state.dynamic_array_size; i-- > 0;) + nested_func->rollbackInsertResult(nested_state + i * nested_size_of_data, arr_to.getData()); + } + bool allocatesMemoryInArena() const override { return true; diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h index ed6c8e84f8a6b..f5cdb6b30e844 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h @@ -240,6 +240,11 @@ class AggregateFunctionIf final : public IAggregateFunctionHelperinsertMergeResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(place, to); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp index b711b12886242..1cdc6b1881f23 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -154,6 +155,23 @@ class AggregateFunctionMap final return map_type->getKeyType(); } + /// Reads one key out of a key column in the representation `merged_maps` is keyed by. + typename Data::SearchType keyAt(const IColumn & key_column, size_t row) const + { + if constexpr (std::is_same_v) + { + if (key_type->getTypeId() == TypeIndex::FixedString) + return assert_cast(key_column).getDataAt(row); + if (key_type->getTypeId() == TypeIndex::IPv6) + return assert_cast(key_column).getDataAt(row); + return assert_cast(key_column).getDataAt(row); + } + else + { + return assert_cast &>(key_column).getData()[row]; + } + } + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { const auto & map_column = assert_cast(*columns[0]); @@ -170,24 +188,7 @@ class AggregateFunctionMap final for (size_t i = 0; i < size; ++i) { - typename Data::SearchType key; - - if constexpr (std::is_same_v) - { - std::string_view key_ref; - if (key_type->getTypeId() == TypeIndex::FixedString) - key_ref = assert_cast(key_column).getDataAt(offset + i); - else if (key_type->getTypeId() == TypeIndex::IPv6) - key_ref = assert_cast(key_column).getDataAt(offset + i); - else - key_ref = assert_cast(key_column).getDataAt(offset + i); - - key = key_ref; - } - else - { - key = assert_cast &>(key_column).getData()[offset + i]; - } + const typename Data::SearchType key = keyAt(key_column, offset + i); auto it = merged_maps.find(key); @@ -339,13 +340,40 @@ class AggregateFunctionMap final } } + /// `transferred` counts the values whose transfer returned, so a caller that catches can undo those. + template + void transferValues( + AggregateDataPtr __restrict place, + ColumnMap & map_column, + const VectorWithMemoryTracking & keys, + size_t & transferred, + Arena * arena) const + { + auto & nested_data_column = map_column.getNestedData(); + auto & key_column = nested_data_column.getColumn(0); + auto & val_column = nested_data_column.getColumn(1); + + auto & merged_maps = this->data(place).merged_maps; + + // insert using sorted keys to result column + for (; transferred < keys.size(); ++transferred) + { + key_column.insert(keys[transferred]); + if constexpr (merge) + nested_func->insertMergeResultInto(merged_maps[keys[transferred]], val_column, arena); + else + nested_func->insertResultInto(merged_maps[keys[transferred]], val_column, arena); + } + + IColumn::Offsets & res_offsets = map_column.getNestedColumn().getOffsets(); + res_offsets.push_back(val_column.size()); + } + template void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const { auto & map_column = assert_cast(to); - auto & nested_column = map_column.getNestedColumn(); auto & nested_data_column = map_column.getNestedData(); - auto & key_column = nested_data_column.getColumn(0); auto & val_column = nested_data_column.getColumn(1); @@ -360,18 +388,37 @@ class AggregateFunctionMap final } ::sort(keys.begin(), keys.end()); - // insert using sorted keys to result column - for (auto & key : keys) + size_t transferred = 0; + + if constexpr (!merge) { - key_column.insert(key); - if constexpr (merge) - nested_func->insertMergeResultInto(merged_maps[key], val_column, arena); - else - nested_func->insertResultInto(merged_maps[key], val_column, arena); + /// A nested function that is not a state aliases nothing and need not be atomic. + if (nested_func->isState()) + { + /// `ColumnString::insert` grows `chars` before it appends the offset, so a row count + /// cannot undo an interrupted key insert but a checkpoint, which restores both, can. + const auto keys_checkpoint = key_column.getCheckpoint(); + IColumn::Offsets & res_offsets = map_column.getNestedColumn().getOffsets(); + const size_t offsets_before = res_offsets.size(); + + try + { + transferValues(place, map_column, keys, transferred, arena); + } + catch (...) + { + res_offsets.resize_assume_reserved(offsets_before); + for (size_t i = transferred; i-- > 0;) + nested_func->rollbackInsertResult(merged_maps[keys[i]], val_column); + key_column.rollback(*keys_checkpoint); + throw; + } + + return; + } } - IColumn::Offsets & res_offsets = nested_column.getOffsets(); - res_offsets.push_back(val_column.size()); + transferValues(place, map_column, keys, transferred, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const override @@ -384,6 +431,33 @@ class AggregateFunctionMap final insertResultIntoImpl(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + auto & map_column = assert_cast(to); + auto & nested_data_column = map_column.getNestedData(); + auto & key_column = nested_data_column.getColumn(0); + auto & val_column = nested_data_column.getColumn(1); + + auto & merged_maps = this->data(place).merged_maps; + const size_t appended = merged_maps.size(); + + IColumn::Offsets & res_offsets = map_column.getNestedColumn().getOffsets(); + res_offsets.resize_assume_reserved(res_offsets.size() - 1); + + /// The values were appended in sorted key order while `merged_maps` is unordered, and a row's + /// undo depends on which place produced it, so recover that order from the keys themselves. + const size_t appended_end = key_column.size(); + for (size_t pos = appended_end; pos-- > appended_end - appended;) + { + auto it = merged_maps.find(keyAt(key_column, pos)); + if (it == merged_maps.end()) + abortOnFailedAssertion("AggregateFunctionMap::rollbackInsertResult: appended key is missing from merged_maps"); + nested_func->rollbackInsertResult(it->second, val_column); + } + + key_column.popBack(appended); + } + bool allocatesMemoryInArena() const override { return true; } AggregateFunctionPtr getNestedFunction() const override { return nested_func; } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h index b80eb99657f7b..113cb24963d02 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h @@ -139,6 +139,11 @@ class AggregateFunctionMerge final : public IAggregateFunctionHelperinsertMergeResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(place, to); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h index 232fe5d8fac62..18cd17272fa4d 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h @@ -248,10 +248,25 @@ class AggregateFunctionNullBase : public IAggregateFunctionHelper if (getFlag(place)) { if constexpr (merge) + { nested_function->insertMergeResultInto(nestedPlace(place), to_concrete.getNestedColumn(), arena); + to_concrete.getNullMapData().push_back(false); + } else + { nested_function->insertResultInto(nestedPlace(place), to_concrete.getNestedColumn(), arena); - to_concrete.getNullMapData().push_back(false); + + /// A nested call that threw has already restored the nested column itself. + try + { + to_concrete.getNullMapData().push_back(false); + } + catch (...) + { + nested_function->rollbackInsertResult(nestedPlace(place), to_concrete.getNestedColumn()); + throw; + } + } } else { @@ -277,6 +292,28 @@ class AggregateFunctionNullBase : public IAggregateFunctionHelper insertResultIntoImpl(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + if constexpr (result_is_nullable) + { + ColumnNullable & to_concrete = assert_cast(to); + if (getFlag(place)) + { + to_concrete.getNullMapData().pop_back(); + nested_function->rollbackInsertResult(nestedPlace(place), to_concrete.getNestedColumn()); + } + else + { + /// insertResultInto appended a state the column itself owns, so this pop must destroy it. + to_concrete.popBack(1); + } + } + else + { + nested_function->rollbackInsertResult(nestedPlace(place), to); + } + } + bool allocatesMemoryInArena() const override { return nested_function->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h b/src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h index 3f51d7b5723e4..1bf5cbc176dd9 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h @@ -319,10 +319,20 @@ class AggregateFunctionOrFill final : public IAggregateFunctionHelper(to); col.getNullMapColumn().insertDefault(); - if constexpr (merge) - nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena); - else - nested_function->insertResultInto(place, col.getNestedColumn(), arena); + /// The null map entry is this call's own, and the nested transfer restores the nested + /// column itself when it throws, so only the entry has to go. + try + { + if constexpr (merge) + nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena); + else + nested_function->insertResultInto(place, col.getNestedColumn(), arena); + } + catch (...) + { + col.getNullMapColumn().getData().pop_back(); + throw; + } } } else @@ -356,10 +366,18 @@ class AggregateFunctionOrFill final : public IAggregateFunctionHelper(to); col.getNullMapColumn().getData().push_back(static_cast(1)); - if constexpr (merge) - nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena); - else - nested_function->insertResultInto(place, col.getNestedColumn(), arena); + try + { + if constexpr (merge) + nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena); + else + nested_function->insertResultInto(place, col.getNestedColumn(), arena); + } + catch (...) + { + col.getNullMapColumn().getData().pop_back(); + throw; + } } } else @@ -384,6 +402,35 @@ class AggregateFunctionOrFill final : public IAggregateFunctionHelper(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + /// The flag-unset branches are indistinguishable from the appended row alone: with a state-nested + /// function they alias like the flag-set branch, otherwise they append a default `to` owns. + if (!place[size_of_data] && !nested_function->isState()) + { + to.popBack(1); + return; + } + + if constexpr (UseNull) + { + if (!result_is_nullable || inner_nullable) + { + nested_function->rollbackInsertResult(place, to); + } + else + { + ColumnNullable & col = typeid_cast(to); + col.getNullMapColumn().getData().pop_back(); + nested_function->rollbackInsertResult(place, col.getNestedColumn()); + } + } + else + { + nested_function->rollbackInsertResult(place, to); + } + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } /// After `Nullable(Tuple)` was introduced, Tuple's `canBeInsideNullable` now returns true, diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionResample.h b/src/AggregateFunctions/Combinators/AggregateFunctionResample.h index b418906d739a7..21defecebde76 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionResample.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionResample.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,12 @@ struct Settings; namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; + extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace FailPoints +{ +extern const char aggregate_function_state_transfer_throw[]; } template @@ -229,23 +236,63 @@ class AggregateFunctionResample final : public IAggregateFunctionHelper(nested_function_->getResultType()); } + /// `transferred` counts the buckets whose transfer returned, so a caller that catches can undo those. template - void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + void transferBuckets(AggregateDataPtr __restrict place, ColumnArray & col, size_t & transferred, Arena * arena) const { - auto & col = assert_cast(to); - auto & col_offsets = assert_cast(col.getOffsetsColumn()); - - for (size_t i = 0; i < total; ++i) + for (; transferred < total; ++transferred) { if constexpr (merge) - nested_function->insertMergeResultInto(place + i * size_of_data, col.getData(), arena); + nested_function->insertMergeResultInto(place + transferred * size_of_data, col.getData(), arena); else - nested_function->insertResultInto(place + i * size_of_data, col.getData(), arena); + nested_function->insertResultInto(place + transferred * size_of_data, col.getData(), arena); } + if constexpr (!merge) + { + fiu_do_on(FailPoints::aggregate_function_state_transfer_throw, + { + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected failure in AggregateFunctionResample::insertResultInto"); + }); + } + + auto & col_offsets = assert_cast(col.getOffsetsColumn()); col_offsets.getData().push_back(col.getData().size()); } + template + void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + { + auto & col = assert_cast(to); + size_t transferred = 0; + + if constexpr (!merge) + { + /// A nested function that is not a state aliases nothing and need not be atomic. + if (nested_function->isState()) + { + const size_t offsets_before = assert_cast(col.getOffsetsColumn()).size(); + + try + { + transferBuckets(place, col, transferred, arena); + } + catch (...) + { + auto & col_offsets = assert_cast(col.getOffsetsColumn()); + col_offsets.getData().resize_assume_reserved(offsets_before); + for (size_t i = transferred; i-- > 0;) + nested_function->rollbackInsertResult(place + i * size_of_data, col.getData()); + throw; + } + + return; + } + } + + transferBuckets(place, col, transferred, arena); + } + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const override { insertResultIntoImpl(place, to, arena); @@ -256,6 +303,16 @@ class AggregateFunctionResample final : public IAggregateFunctionHelper(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + auto & col = assert_cast(to); + auto & col_offsets = assert_cast(col.getOffsetsColumn()); + + col_offsets.getData().resize_assume_reserved(col_offsets.size() - 1); + for (size_t i = total; i-- > 0;) + nested_function->rollbackInsertResult(place + i * size_of_data, col.getData()); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } }; diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionSimpleState.h b/src/AggregateFunctions/Combinators/AggregateFunctionSimpleState.h index 19a786db1f206..7385a3e944dbe 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionSimpleState.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionSimpleState.h @@ -103,6 +103,11 @@ class AggregateFunctionSimpleState final : public IAggregateFunctionHelperinsertResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(place, to); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); } AggregateFunctionPtr getNestedFunction() const override { return nested_func; } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionState.h b/src/AggregateFunctions/Combinators/AggregateFunctionState.h index 5cb41d1d8716f..7318c51022c04 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionState.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionState.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -10,6 +11,16 @@ namespace DB { struct Settings; +namespace ErrorCodes +{ +extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace FailPoints +{ +extern const char aggregate_function_state_transfer_throw[]; +} + /** Not an aggregate function, but an adapter of aggregate functions, * Aggregate functions with the `State` suffix differ from the corresponding ones in that their states are not finalized. @@ -144,7 +155,18 @@ class AggregateFunctionState final : public IAggregateFunctionHelper(to).getData().push_back(place); + auto & column = assert_cast(to); + + /// Only once the column holds an aliased state, which is the partial transfer to undo. + if (unlikely(!column.empty())) + { + fiu_do_on(FailPoints::aggregate_function_state_transfer_throw, + { + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected failure in AggregateFunctionState::insertResultInto"); + }); + } + + column.getData().push_back(place); } void insertMergeResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override @@ -152,6 +174,13 @@ class AggregateFunctionState final : public IAggregateFunctionHelper(to).insertFrom(place); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict, IColumn & to) const noexcept override + { + /// insertResultInto aliased a state that `place` still owns, so the row must go without the + /// destroy that ColumnAggregateFunction::popBack performs. + assert_cast(to).popBackWithoutDestroy(1); + } + /// Aggregate function or aggregate function state. bool isState() const override { return true; } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp index 4001af145fa10..13b9d034887e5 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp @@ -499,6 +499,13 @@ void AggregateFunctionTuple::insertMergeResultInto(AggregateDataPtr __restrict p insertResultIntoImpl(place, to, arena); } +void AggregateFunctionTuple::rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept +{ + auto & tuple_to = assert_cast(to); + for (size_t i = nested_functions.size(); i-- > 0;) + nested_functions[i]->rollbackInsertResult(place + state_offsets[i], tuple_to.getColumn(i)); +} + bool AggregateFunctionTuple::allocatesMemoryInArena() const { for (const auto & func : nested_functions) diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.h b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.h index 4cefe838a21b4..3c03a82531c3c 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -11,6 +12,16 @@ namespace DB { struct Settings; +namespace ErrorCodes +{ +extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace FailPoints +{ +extern const char aggregate_function_state_transfer_throw_after_child[]; +} + /** Adaptor for aggregate functions. * Adding -Tuple suffix to aggregate function * will convert that aggregate function to a function, accepting Tuples, @@ -130,6 +141,7 @@ class AggregateFunctionTuple final : public IAggregateFunctionHelper - void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + void transferElements(AggregateDataPtr __restrict place, ColumnTuple & tuple_to, size_t & transferred, Arena * arena) const { - auto & tuple_to = assert_cast(to); - for (size_t i = 0; i < nested_functions.size(); ++i) + for (; transferred < nested_functions.size(); ++transferred) { + if constexpr (!for_merge) + { + /// Only past the first element, whose transfer returned, is there a completed child to undo. + if (unlikely(transferred > 0)) + { + fiu_do_on(FailPoints::aggregate_function_state_transfer_throw_after_child, + { + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected failure in AggregateFunctionTuple::insertResultInto"); + }); + } + } + if constexpr (for_merge) - nested_functions[i]->insertMergeResultInto(place + state_offsets[i], tuple_to.getColumn(i), arena); + nested_functions[transferred]->insertMergeResultInto( + place + state_offsets[transferred], tuple_to.getColumn(transferred), arena); else - nested_functions[i]->insertResultInto(place + state_offsets[i], tuple_to.getColumn(i), arena); + nested_functions[transferred]->insertResultInto( + place + state_offsets[transferred], tuple_to.getColumn(transferred), arena); } } + template + void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + { + auto & tuple_to = assert_cast(to); + size_t transferred = 0; + + if constexpr (!for_merge) + { + /// An element that is not a state aliases nothing and need not be atomic. + if (isState()) + { + try + { + transferElements(place, tuple_to, transferred, arena); + } + catch (...) + { + for (size_t i = transferred; i-- > 0;) + nested_functions[i]->rollbackInsertResult(place + state_offsets[i], tuple_to.getColumn(i)); + throw; + } + + return; + } + } + + transferElements(place, tuple_to, transferred, arena); + } + /// Shared implementation of the batch add overrides. Hoists the per-element column pointers, so /// no per-row unwrapping work remains in the row loop. /// `get_place` returns the aggregation state for a row, or nullptr when the row has none. diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 7f7e321d3ad0e..ffa802107efed 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -294,6 +294,15 @@ class IAggregateFunction : public std::enable_shared_from_this size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot pop {} rows from {}: there are only {} rows", n, getName(), size()); + + data.resize_assume_reserved(data.size() - n); +} + ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const { size_t size = data.size(); diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 2689cb00816a8..c9da4d7f56473 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -131,6 +131,18 @@ class ColumnAggregateFunction final : public COWHelper arrayMap(y -> length(finalizeAggregation(y)), x), groupArrayStateForEachResample(0, 2, 1)([number], number % 2)) FROM numbers(2000) SETTINGS max_threads = 1; + +-- -ForEach alone: the throw is in its own loop, with the first element already aliased. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEach([number, number + 1]) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), groupArrayStateForEach([number, number + 1])) FROM numbers(2000) SETTINGS max_threads = 1; + +-- -Tuple over -Map with String keys: the first element completes, the second throws on its second +-- key, so -Tuple has to undo a completed -Map child in a different subcolumn. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateMapTuple((map('a', number), map('b', number, 'c', number))) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT mapKeys(t.1), arrayMap(x -> length(finalizeAggregation(x)), mapValues(t.1)), mapKeys(t.2), arrayMap(x -> length(finalizeAggregation(x)), mapValues(t.2)) FROM (SELECT groupArrayStateMapTuple((map('a', number), map('b', number, 'c', number))) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- -Tuple over -ForEach, first element shorter than the second. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEachTuple(([number], [number, number])) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateForEachTuple(([number], [number, number])) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- Exactly one bucket: the -State transfer cannot throw (the column is still empty), so the throw +-- comes from -Resample appending its own offset with the single sub-state already aliased. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResample(0, 1, 1)(number, 0) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), groupArrayStateResample(0, 1, 1)(number, 0)) FROM numbers(2000) SETTINGS max_threads = 1; + +-- A -Map whose value rows are not uniform: the key with non-NULL values aliases a sub-state, the +-- all-NULL keys make the null adapter insert a default the column itself owns. Undoing such a map in +-- any order other than the reverse of the sorted-key append order applies the wrong row's semantics. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateTupleMapTuple((CAST(map('b', NULL, 'a', (number, number + 1), 'c', NULL), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'), CAST(map('x', (number, number + 1), 'y', (number + 2, number + 3)), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'))) FROM numbers(2000) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT mapKeys(t.1), arrayMap(x -> isNull(x), mapValues(t.1)), arrayMap(x -> length(finalizeAggregation(assumeNotNull(x).1)), mapValues(t.1)), mapKeys(t.2), arrayMap(x -> isNull(x), mapValues(t.2)) FROM (SELECT groupArrayStateTupleMapTuple((CAST(map('b', NULL, 'a', (number, number + 1), 'c', NULL), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'), CAST(map('x', (number, number + 1), 'y', (number + 2, number + 3)), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'))) AS t FROM numbers(2000)) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1; + +-- -Tuple over -Resample. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same, through a transparent -If wrapper. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleIfTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2), (number > 0, number > 0)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleIfTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2), (number > 0, number > 0)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- -OrNull around a Tuple of -State results: Tuple can be inside Nullable, so the transfer goes into +-- the nested column of a ColumnNullable. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTupleOrNullTuple(0, 2, 1)(((number, number + 1), (number + 2, number + 3)), ((number % 2, number % 2), (number % 2, number % 2))) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.1).1), arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.2).2) FROM (SELECT groupArrayStateResampleTupleOrNullTuple(0, 2, 1)(((number, number + 1), (number + 2, number + 3)), ((number % 2, number % 2), (number % 2, number % 2))) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same window through the implicit null adapter instead of -OrNull, plus a -Distinct wrapper. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTupleDistinctTuple(0, 2, 1)(CAST(((number, number + 1), (number + 2, number + 3)), 'Tuple(Nullable(Tuple(UInt64, UInt64)), Nullable(Tuple(UInt64, UInt64)))'), ((number % 2, number % 2), (number % 2, number % 2))) FROM numbers(2000) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.1).1), arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.2).2) FROM (SELECT groupArrayStateResampleTupleDistinctTuple(0, 2, 1)(CAST(((number, number + 1), (number + 2, number + 3)), 'Tuple(Nullable(Tuple(UInt64, UInt64)), Nullable(Tuple(UInt64, UInt64)))'), ((number % 2, number % 2), (number % 2, number % 2))) AS t FROM numbers(2000)) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1; + +-- A completed -Resample child undone by its parent: with the second failpoint the first tuple element +-- transfers all of its buckets, and the throw lands before the second element. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same completed child behind an -OrNull that forwards: Array cannot be inside Nullable, so -OrFill +-- passes the array column straight through and has to forward the undo as well. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT groupArrayStateResampleOrNullTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleOrNullTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; From b6dd300fa9b56a0478e85e08f71932bcd1ecc375 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 14:32:50 +0000 Subject: [PATCH 103/296] Backport #121894 to 26.8: Fix a server crash when the flameGraph aggregate function is constructed without a query context --- .../AggregateFunctionFactory.h | 2 ++ .../AggregateFunctionFlameGraph.cpp | 4 +++- ...graph_attach_without_query_context.reference | 1 + ...3_flamegraph_attach_without_query_context.sh | 17 +++++++++++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05243_flamegraph_attach_without_query_context.reference create mode 100755 tests/queries/0_stateless/05243_flamegraph_attach_without_query_context.sh diff --git a/src/AggregateFunctions/AggregateFunctionFactory.h b/src/AggregateFunctions/AggregateFunctionFactory.h index 53248dae079f9..c3323b1a2177c 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.h +++ b/src/AggregateFunctions/AggregateFunctionFactory.h @@ -31,6 +31,8 @@ class ASTFunction; * The invoker has arguments: name of aggregate function, types of arguments, values of parameters. * Parameters are for "parametric" aggregate functions. * For example, in quantileWeighted(0.9)(x, weight), 0.9 is "parameter" and x, weight are "arguments". + * `Settings` is null when the function is constructed outside a query, e.g. while a background + * thread parses an `AggregateFunction(...)` type name. */ using AggregateFunctionCreator = std::function; diff --git a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp index eb5f188de5458..b84aaa5daf125 100644 --- a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp +++ b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp @@ -691,7 +691,9 @@ static void check(const std::string & name, const DataTypes & argument_types, co static AggregateFunctionPtr createAggregateFunctionFlameGraph(const std::string & name, const DataTypes & argument_types, const Array & params, const Settings * settings) { - if (!(*settings)[Setting::allow_introspection_functions]) + /// The factory passes null when there is no query context to take settings from, and with no + /// principal there is nothing to authorize. + if (settings && !(*settings)[Setting::allow_introspection_functions]) throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0"); diff --git a/tests/queries/0_stateless/05243_flamegraph_attach_without_query_context.reference b/tests/queries/0_stateless/05243_flamegraph_attach_without_query_context.reference new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/tests/queries/0_stateless/05243_flamegraph_attach_without_query_context.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05243_flamegraph_attach_without_query_context.sh b/tests/queries/0_stateless/05243_flamegraph_attach_without_query_context.sh new file mode 100755 index 0000000000000..3e7efa42e6b85 --- /dev/null +++ b/tests/queries/0_stateless/05243_flamegraph_attach_without_query_context.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# A stored table whose column type names AggregateFunction(flameGraph, ...) used to crash the +# process when a second run opened it. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +cd "${CLICKHOUSE_TMP}" || exit +rm -rf "05243_flamegraph_attach" + +$CLICKHOUSE_LOCAL --path "05243_flamegraph_attach" --query "SET allow_introspection_functions = 1; CREATE TABLE t (c AggregateFunction(flameGraph, Array(UInt64))) ENGINE = Memory" + +# Read the table rather than a constant, so that opening it is what the output depends on. +$CLICKHOUSE_LOCAL --path "05243_flamegraph_attach" --query "SELECT count() FROM t" + +rm -rf "05243_flamegraph_attach" From 974a1c0b3cae4f8a951c9f8dcd6c90a30adfe2b6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 15:23:43 +0000 Subject: [PATCH 104/296] Backport #120376 to 26.8: Enforce settings constraints installed by a profile in the same statement --- src/Interpreters/Context.cpp | 77 +++++++++++++++++++ src/Interpreters/Context.h | 2 + src/Interpreters/InterpreterSetQuery.cpp | 4 +- src/Server/ArrowFlight/ArrowFlightServer.cpp | 17 +++- src/Server/GRPCServer.cpp | 25 +++--- .../test_session_options_settings_profile.py | 76 ++++++++++++++++++ tests/integration/test_grpc_protocol/test.py | 47 +++++++++++ ...raints_set_profile_in_same_query.reference | 1 + ...s_constraints_set_profile_in_same_query.sh | 39 ++++++++++ ...aints_profile_in_nested_settings.reference | 3 + ..._constraints_profile_in_nested_settings.sh | 18 +++++ ...nstraints_profile_in_http_params.reference | 6 ++ ...ings_constraints_profile_in_http_params.sh | 28 +++++++ 13 files changed, 328 insertions(+), 15 deletions(-) create mode 100644 tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py create mode 100644 tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference create mode 100755 tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh create mode 100644 tests/queries/0_stateless/05219_settings_constraints_profile_in_nested_settings.reference create mode 100755 tests/queries/0_stateless/05219_settings_constraints_profile_in_nested_settings.sh create mode 100644 tests/queries/0_stateless/05220_settings_constraints_profile_in_http_params.reference create mode 100755 tests/queries/0_stateless/05220_settings_constraints_profile_in_http_params.sh diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 5649d6f637b97..4bcc3b6bc7672 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -3398,6 +3399,41 @@ Settings Context::getSettingsCopy() const return *settings; } +namespace +{ +bool isProfileChange(const SettingChange & change) +{ + return change.name == "profile"; +} + +/// Enforces the constraints on `changes` the way `applySettingsChanges` applies them: a `profile` change +/// installs a new constraint set for the changes after it. Each run of changes up to the next `profile` +/// change is enforced against the constraints in force before it, then applied together with that `profile` +/// change to a scratch copy of `context`, so a rejected list leaves `context` untouched. Returns the enforced list. +template +SettingsChanges enforceConstraintsAlongProfileChanges(const ContextPtr & context, const SettingsChanges & changes, Enforce && enforce) +{ + auto scratch_context = Context::createCopy(context); + SettingsChanges enforced; + for (auto begin = changes.begin(); begin != changes.end();) + { + auto profile = std::find_if(begin, changes.end(), isProfileChange); + SettingsChanges segment(begin, profile); + enforce(*scratch_context, segment); + begin = profile; + if (profile != changes.end()) + { + segment.push_back(*profile); + ++begin; + } + /// `setCurrentProfile` checks the profile's own settings against the constraints in force before it. + scratch_context->applySettingsChanges(segment); + enforced.insert(enforced.end(), segment.begin(), segment.end()); + } + return enforced; +} +} + void Context::setSettings(const Settings & settings_) { std::lock_guard lock(mutex); @@ -3558,6 +3594,15 @@ void Context::checkSettingsConstraints(const SettingChange & change, SettingSour void Context::checkSettingsConstraints(const SettingsChanges & changes, SettingSource source) { + if (std::ranges::any_of(changes, isProfileChange)) + { + enforceConstraintsAlongProfileChanges(shared_from_this(), changes, [source](Context & context, SettingsChanges & segment) + { + context.checkSettingsConstraints(std::as_const(segment), source); + }); + return; + } + SharedLockGuard lock(mutex); settings->checkShorthandChanges(changes); getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(*settings, changes, source); @@ -3570,14 +3615,46 @@ void Context::checkSettingsConstraintsForSettingsReset(const std::vector getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.checkResetToDefault(*settings, names, source); } +void Context::checkSettingsConstraintsForSettingsReset( + const std::vector & names, const SettingsChanges & changes_applied_first, SettingSource source) +{ + if (std::ranges::none_of(changes_applied_first, isProfileChange)) + { + checkSettingsConstraintsForSettingsReset(names, source); + return; + } + /// The resets take effect after the rest of the statement, so a `profile` change in it decides the constraints. + auto scratch_context = Context::createCopy(shared_from_this()); + scratch_context->applySettingsChanges(changes_applied_first); + scratch_context->checkSettingsConstraintsForSettingsReset(names, source); +} + void Context::checkSettingsConstraints(SettingsChanges & changes, SettingSource source) { + if (std::ranges::any_of(changes, isProfileChange)) + { + changes = enforceConstraintsAlongProfileChanges(shared_from_this(), changes, [source](Context & context, SettingsChanges & segment) + { + context.checkSettingsConstraints(segment, source); + }); + return; + } + SharedLockGuard lock(mutex); checkSettingsConstraintsWithLock(changes, source); } void Context::clampToSettingsConstraints(SettingsChanges & changes, SettingSource source) { + if (std::ranges::any_of(changes, isProfileChange)) + { + changes = enforceConstraintsAlongProfileChanges(shared_from_this(), changes, [source](Context & context, SettingsChanges & segment) + { + context.clampToSettingsConstraints(segment, source); + }); + return; + } + SharedLockGuard lock(mutex); clampToSettingsConstraintsWithLock(changes, source); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index f40b65b73f913..6c2dcdcff8f59 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1228,6 +1228,8 @@ class Context: public ContextData, public std::enable_shared_from_this void checkSettingsConstraints(const SettingsChanges & changes, SettingSource source); void checkSettingsConstraints(SettingsChanges & changes, SettingSource source); void checkSettingsConstraintsForSettingsReset(const std::vector & names, SettingSource source); + /// For the resets of a statement that also changes `profile`: `changes_applied_first` decides their constraints. + void checkSettingsConstraintsForSettingsReset(const std::vector & names, const SettingsChanges & changes_applied_first, SettingSource source); void clampToSettingsConstraints(SettingsChanges & changes, SettingSource source); void checkMergeTreeSettingsConstraints(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const; diff --git a/src/Interpreters/InterpreterSetQuery.cpp b/src/Interpreters/InterpreterSetQuery.cpp index fd7ff81b7a77a..cef2ade6d97d4 100644 --- a/src/Interpreters/InterpreterSetQuery.cpp +++ b/src/Interpreters/InterpreterSetQuery.cpp @@ -89,7 +89,7 @@ BlockIO InterpreterSetQuery::execute() /// explicitly set to its current value. The original code applies const `ast.changes`. getContext()->checkSettingsConstraints(std::as_const(changes), SettingSource::QUERY); /// Checked before anything is applied, so that a violation leaves the whole statement without effect. - getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, SettingSource::QUERY); + getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, changes, SettingSource::QUERY); auto session_context = getContext()->getSessionContext(); session_context->applySettingsChanges(changes); session_context->addQueryParameters(NameToNameMap{ast.query_parameters.begin(), ast.query_parameters.end()}); @@ -109,7 +109,7 @@ void InterpreterSetQuery::executeForCurrentContext(bool ignore_setting_constrain if (!ignore_setting_constraints) { getContext()->checkSettingsConstraints(std::as_const(changes), SettingSource::QUERY); - getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, SettingSource::QUERY); + getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, changes, SettingSource::QUERY); rejectHTTPOnlyConstructionSettings(ast); } getContext()->applySettingsChanges(changes); diff --git a/src/Server/ArrowFlight/ArrowFlightServer.cpp b/src/Server/ArrowFlight/ArrowFlightServer.cpp index 87627bbce6af6..c560ba7838ed4 100644 --- a/src/Server/ArrowFlight/ArrowFlightServer.cpp +++ b/src/Server/ArrowFlight/ArrowFlightServer.cpp @@ -1376,14 +1376,14 @@ arrow::Status ArrowFlightServer::DoAction( } }; - for (const auto & [setting, value] : request.session_options) + auto apply_option = [&](const std::string & setting, const auto & value) { if (!isValidIdentifier(setting)) { result.errors[setting] = arrow::flight::SetSessionOptionsResult::Error{ arrow::flight::SetSessionOptionErrorValue::kInvalidName }; - continue; + return; } try @@ -1391,14 +1391,14 @@ arrow::Status ArrowFlightServer::DoAction( if (std::holds_alternative(value)) { /// std::monostate means "reset to default" (SET setting = DEFAULT). - query_context->checkSettingsConstraintsForSettingsReset({setting}, SettingSource::QUERY); + session_context->checkSettingsConstraintsForSettingsReset({setting}, SettingSource::QUERY); session_context->resetSettingsToDefaultValue({setting}); } else { auto string_value = std::visit(to_string_value, value); SettingChange change{setting, Field{string_value}}; - query_context->checkSettingsConstraints(change, SettingSource::QUERY); + session_context->checkSettingsConstraints(change, SettingSource::QUERY); session_context->setSetting(setting, string_value); } } @@ -1416,6 +1416,15 @@ arrow::Status ArrowFlightServer::DoAction( result.errors[setting] = arrow::flight::SetSessionOptionsResult::Error{error_value}; } + }; + + /// The options arrive in a map with no order, so `profile` goes first for its constraints to bind the rest. + if (auto profile = request.session_options.find("profile"); profile != request.session_options.end()) + apply_option(profile->first, profile->second); + for (const auto & [setting, value] : request.session_options) + { + if (setting != "profile") + apply_option(setting, value); } ARROW_ASSIGN_OR_RAISE(auto serialized, result.SerializeToString()) diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index f8dd4d717c2a5..ae5a796bfe126 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -310,6 +310,20 @@ namespace } }; + /// A protobuf map has no order, so `profile` goes first for its constraints to bind the other settings. + SettingsChanges settingsChangesFromMap(const google::protobuf::Map & map) + { + SettingsChanges changes; + for (const auto & [key, value] : map) + { + if (key == "profile") + changes.insert(changes.begin(), {key, value}); + else + changes.push_back({key, value}); + } + return changes; + } + /// Gets session's timeout from query info or from the server config. std::chrono::steady_clock::duration getSessionTimeout(const GRPCQueryInfo & query_info, const Poco::Util::AbstractConfiguration & config) { @@ -945,12 +959,7 @@ namespace query_context = session->makeQueryContext(std::move(client_info)); - /// Prepare settings. - SettingsChanges settings_changes; - for (const auto & [key, value] : query_info.settings()) - { - settings_changes.push_back({key, value}); - } + auto settings_changes = settingsChangesFromMap(query_info.settings()); query_context->checkSettingsConstraints(settings_changes, SettingSource::QUERY); query_context->applySettingsChanges(settings_changes); @@ -1277,9 +1286,7 @@ namespace { temp_context = Context::createCopy(query_context); external_table_context = temp_context; - SettingsChanges settings_changes; - for (const auto & [key, value] : external_table.settings()) - settings_changes.push_back({key, value}); + auto settings_changes = settingsChangesFromMap(external_table.settings()); external_table_context->checkSettingsConstraints(settings_changes, SettingSource::QUERY); external_table_context->applySettingsChanges(settings_changes); } diff --git a/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py b/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py new file mode 100644 index 0000000000000..d998b1690edf9 --- /dev/null +++ b/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +import pytest +import random +import string + +from .flight_sql_client import FlightSQLClient, SetSessionOptionsResult + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=[ + "configs/flight_port.xml", + ], +) + +PROFILE_NAME = "profile_arrowflight_session_options_constraints" + + +def get_client(): + session_id = ''.join(random.choices(string.ascii_letters + string.digits, k=16)) + return FlightSQLClient( + host=node.ip_address, + port=8888, + insecure=True, + disable_server_verification=True, + metadata={'x-clickhouse-session-id': session_id}, + features={'metadata-reflection': 'true'}, + ) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + node.wait_until_port_is_ready(8888, timeout=10) + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def settings_profile(): + node.query(f"DROP SETTINGS PROFILE IF EXISTS {PROFILE_NAME}") + node.query( + f"CREATE SETTINGS PROFILE {PROFILE_NAME} SETTINGS max_execution_time = 10 CONST" + ) + try: + yield PROFILE_NAME + finally: + node.query(f"DROP SETTINGS PROFILE IF EXISTS {PROFILE_NAME}") + + +def _query_scalar(client, query): + flight_info = client.execute(query) + table = client.do_get(flight_info.endpoints[0].ticket).read_all() + return table.column(0)[0].as_py() + + +def test_profile_constraints_apply_within_same_request(): + """A profile set in a SetSessionOptions request constrains the other options of the same request.""" + client = get_client() + + result = client.set_session_options( + {"profile": PROFILE_NAME, "max_execution_time": "999"} + ) + + assert "profile" not in result.errors + # A constraint violation is neither a parse nor an unknown-setting error, so it maps to UNSPECIFIED. + assert ( + result.errors["max_execution_time"].value + == SetSessionOptionsResult.UNSPECIFIED + ) + assert float(_query_scalar(client, "SELECT getSetting('max_execution_time')")) == 10 diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 915eaef3ad15f..9aa9080bf7379 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -611,6 +611,53 @@ def test_external_table(): ) +def test_settings_profile_constraints(): + profile = "grpc_profile_constraints" + query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") + query( + f"CREATE SETTINGS PROFILE {profile} SETTINGS max_execution_time = 10 CONST, max_result_rows = 12345, format_csv_delimiter = '|'" + ) + try: + # `settings` is a protobuf map with no order: the profile goes first, so its constraints bind the + # other settings and they override the values it sets. + e = query_and_get_error( + "SELECT 1", settings={"profile": profile, "max_execution_time": "999"} + ) + assert "Setting max_execution_time should not be changed" in e.display_text + assert ( + query( + "SELECT getSetting('max_result_rows')", + settings={"profile": profile, "max_result_rows": "7"}, + ) + == "7\n" + ) + + columns = [ + clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), + clickhouse_grpc_pb2.NameAndType(name="UserName", type="String"), + ] + + def ext(settings): + return clickhouse_grpc_pb2.ExternalTable( + name="ext1", columns=columns, data=b"1;Alex\n", format="CSV", settings=settings + ) + + e = query_and_get_error( + "SELECT * FROM ext1", + external_tables=[ext({"profile": profile, "max_execution_time": "999"})], + ) + assert "Setting max_execution_time should not be changed" in e.display_text + assert ( + query( + "SELECT * FROM ext1", + external_tables=[ext({"profile": profile, "format_csv_delimiter": ";"})], + ) + == "1\tAlex\n" + ) + finally: + query(f"DROP SETTINGS PROFILE {profile}") + + def test_external_table_streaming(): columns = [ clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), diff --git a/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference new file mode 100644 index 0000000000000..17c209230693f --- /dev/null +++ b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference @@ -0,0 +1 @@ +7 8 diff --git a/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh new file mode 100755 index 0000000000000..e4d11a0e11bf4 --- /dev/null +++ b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# A `SET` statement that changes `profile` installs a new constraint set halfway through itself, and the +# assignments and resets after that point must pass it, as they would in a separate statement. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +CONST_PROFILE="profile_const_$CLICKHOUSE_DATABASE" +READONLY_PROFILE="profile_readonly_$CLICKHOUSE_DATABASE" +MAX_PROFILE="profile_max_$CLICKHOUSE_DATABASE" +RELAXED_PROFILE="profile_relaxed_$CLICKHOUSE_DATABASE" + +$CLICKHOUSE_CLIENT -m -q " +DROP SETTINGS PROFILE IF EXISTS $CONST_PROFILE, $READONLY_PROFILE, $MAX_PROFILE, $RELAXED_PROFILE; +CREATE SETTINGS PROFILE $CONST_PROFILE SETTINGS max_execution_time = 10 CONST, SQL_tenant_id = 1 CONST; +CREATE SETTINGS PROFILE $READONLY_PROFILE SETTINGS readonly = 1; +CREATE SETTINGS PROFILE $MAX_PROFILE SETTINGS max_memory_usage MAX 1000000; +CREATE SETTINGS PROFILE $RELAXED_PROFILE SETTINGS max_memory_usage MAX 1099511627776; +" + +# One session on purpose: a rejected statement must leave it untouched, which the statements after it prove. +$CLICKHOUSE_CLIENT -m -q " +SET profile = '$CONST_PROFILE', max_execution_time = 999; -- { serverError SETTING_CONSTRAINT_VIOLATION } +SET profile = '$CONST_PROFILE', max_execution_time = DEFAULT; -- { serverError SETTING_CONSTRAINT_VIOLATION } +SET profile = '$READONLY_PROFILE', max_memory_usage = 1099511627776; -- { serverError READONLY } +SET profile = '$MAX_PROFILE', max_memory_usage = 1099511627776; -- { serverError SETTING_CONSTRAINT_VIOLATION } +-- a later, looser profile does not lift the constraint in force at the assignment +SET profile = '$MAX_PROFILE', max_memory_usage = 5000000, profile = '$RELAXED_PROFILE'; -- { serverError SETTING_CONSTRAINT_VIOLATION } +-- none of the rejected statements applied its profile, which would have installed this setting +SELECT getSetting('SQL_tenant_id'); -- { serverError UNKNOWN_SETTING } +-- assigning the value the profile installs is a no-op and stays allowed +SET profile = '$CONST_PROFILE', max_execution_time = 10; +-- what is not constrained still applies, before and after the profile change +SET SQL_before = 7, profile = '$CONST_PROFILE', SQL_after = 8; +SELECT getSetting('SQL_before'), getSetting('SQL_after'); +" + +$CLICKHOUSE_CLIENT -q "DROP SETTINGS PROFILE $CONST_PROFILE, $READONLY_PROFILE, $MAX_PROFILE, $RELAXED_PROFILE" diff --git a/tests/queries/0_stateless/05219_settings_constraints_profile_in_nested_settings.reference b/tests/queries/0_stateless/05219_settings_constraints_profile_in_nested_settings.reference new file mode 100644 index 0000000000000..4f2ffd17cb8dd --- /dev/null +++ b/tests/queries/0_stateless/05219_settings_constraints_profile_in_nested_settings.reference @@ -0,0 +1,3 @@ +10 +1000000 +10 diff --git a/tests/queries/0_stateless/05219_settings_constraints_profile_in_nested_settings.sh b/tests/queries/0_stateless/05219_settings_constraints_profile_in_nested_settings.sh new file mode 100755 index 0000000000000..613eece11dcd5 --- /dev/null +++ b/tests/queries/0_stateless/05219_settings_constraints_profile_in_nested_settings.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# A `profile` change in a nested `SETTINGS` clause constrains the settings after it. Settings crossing into +# another execution context are clamped rather than rejected, so the nested value loses to the constraint. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +PROFILE="profile_nested_$CLICKHOUSE_DATABASE" + +$CLICKHOUSE_CLIENT -m -q " +DROP SETTINGS PROFILE IF EXISTS $PROFILE; +CREATE SETTINGS PROFILE $PROFILE SETTINGS max_execution_time = 10 CONST, max_memory_usage MAX 1000000; +SELECT * FROM (SELECT getSetting('max_execution_time') SETTINGS profile = '$PROFILE', max_execution_time = 999); +SELECT * FROM (SELECT getSetting('max_memory_usage') SETTINGS profile = '$PROFILE', max_memory_usage = 1099511627776); +WITH w AS (SELECT getSetting('max_execution_time') SETTINGS profile = '$PROFILE', max_execution_time = 999) SELECT * FROM w; +DROP SETTINGS PROFILE $PROFILE; +" diff --git a/tests/queries/0_stateless/05220_settings_constraints_profile_in_http_params.reference b/tests/queries/0_stateless/05220_settings_constraints_profile_in_http_params.reference new file mode 100644 index 0000000000000..5521a7189933b --- /dev/null +++ b/tests/queries/0_stateless/05220_settings_constraints_profile_in_http_params.reference @@ -0,0 +1,6 @@ +-- a parameter after the profile is checked against the constraints it installs +SETTING_CONSTRAINT_VIOLATION +-- a parameter before the profile is overridden by it +12345 +-- a parameter equal to the value before the profile still overrides the profile +0 diff --git a/tests/queries/0_stateless/05220_settings_constraints_profile_in_http_params.sh b/tests/queries/0_stateless/05220_settings_constraints_profile_in_http_params.sh new file mode 100755 index 0000000000000..d060d5ee55c48 --- /dev/null +++ b/tests/queries/0_stateless/05220_settings_constraints_profile_in_http_params.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# HTTP URL parameters are a settings change list in URL order, so a `profile` parameter constrains the +# parameters after it. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +PROFILE="profile_http_$CLICKHOUSE_DATABASE" + +$CLICKHOUSE_CLIENT -m -q " +DROP SETTINGS PROFILE IF EXISTS $PROFILE; +CREATE SETTINGS PROFILE $PROFILE SETTINGS max_execution_time = 10 CONST, max_result_rows = 12345; +" + +function run_with_params() +{ + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$1" -d "$2" +} + +echo '-- a parameter after the profile is checked against the constraints it installs' +run_with_params "profile=$PROFILE&max_execution_time=999" "SELECT 1" | grep -o -m1 SETTING_CONSTRAINT_VIOLATION +echo '-- a parameter before the profile is overridden by it' +run_with_params "max_result_rows=8&profile=$PROFILE" "SELECT getSetting('max_result_rows')" +echo '-- a parameter equal to the value before the profile still overrides the profile' +run_with_params "profile=$PROFILE&max_result_rows=0" "SELECT getSetting('max_result_rows')" + +$CLICKHOUSE_CLIENT -q "DROP SETTINGS PROFILE $PROFILE" From 8137e730c439f2c183619be1876be4356b6156b4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 25 Sep 2026 17:55:16 +0000 Subject: [PATCH 105/296] Update autogenerated version to 26.8.11.7 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index d14f61b49bdc0..16b893d9fd0d8 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54523) +SET(VERSION_REVISION 54524) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 11) -SET(VERSION_GITHASH bedf2ab54b8a0c34afbf2d907eaf933324f43cd6) -SET(VERSION_DESCRIBE v26.8.11.1-lts) -SET(VERSION_STRING 26.8.11.1) +SET(VERSION_PATCH 12) +SET(VERSION_GITHASH 97da26e305d9764402a30702c9b779745bf557c4) +SET(VERSION_DESCRIBE v26.8.12.1-lts) +SET(VERSION_STRING 26.8.12.1) # end of autochange From 17861e3f82cefdcff6e0ff2ebc5d66afc143e1be Mon Sep 17 00:00:00 2001 From: Jimmy Aguilar Mena Date: Fri, 25 Sep 2026 20:47:55 +0000 Subject: [PATCH 106/296] Strip the padding of an set element before tokenizing it --- .../MergeTree/MergeTreeIndexBloomFilterText.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp index d9d478b11dfbb..e8e299d1a6995 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp @@ -822,6 +822,9 @@ bool MergeTreeConditionBloomFilterText::tryPrepareSetBloomFilter( size_t tuple_idx = elem.tuple_index; const auto & column = columns[tuple_idx]; + const DataTypePtr & element_type = prepared_set->getElementsTypes()[tuple_idx]; + const bool is_fixed_string_element = WhichDataType(removeNullable(element_type)).isFixedString(); + for (size_t row = 0; row < prepared_set_total_row_count; ++row) { /// A NULL element also matches the column's NULL rows, which the filter cannot express. @@ -829,7 +832,12 @@ bool MergeTreeConditionBloomFilterText::tryPrepareSetBloomFilter( return false; bloom_filters.back().emplace_back(params); - auto ref = column->getDataAt(row); + + /// `FixedString` element carries its padding, which the comparison ignores but the tokenizer would not. + std::string_view ref = column->getDataAt(row); + if (is_fixed_string_element) + ref = ref.substr(0, ref.find_last_not_of('\0') + 1); + forEachTokenToBloomFilter(*tokenizer, ref.data(), ref.size(), bloom_filters.back().back()); } } From 7504aa878eebe7f65c6735e36268ef9b060371d7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 26 Sep 2026 04:07:42 +0000 Subject: [PATCH 107/296] Backport #119302 to 26.8: Revert "Revert "Proper partition pruning with dates"" --- src/Functions/FunctionsConversion.h | 16 +- .../Iceberg/ManifestFilesPruning.cpp | 206 +++++++++++++- .../test_partition_pruning_with_functions.py | 268 ++++++++++++++++++ ...nullable_key_todate_monotonicity.reference | 8 + ...05054_nullable_key_todate_monotonicity.sql | 33 +++ 5 files changed, 523 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py create mode 100644 tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference create mode 100644 tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index e3158387728c0..477c0e010bbf7 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -4365,8 +4365,14 @@ struct ToDateMonotonicity { static bool has() { return true; } - static IFunction::Monotonicity get(const IDataType & type, const Field & left, const Field & right) + static IFunction::Monotonicity get(const IDataType & type_with_wrappers, const Field & left, const Field & right) { + const IDataType * type_without_wrappers = &type_with_wrappers; + if (const auto * low_cardinality_type = typeid_cast(type_without_wrappers)) + type_without_wrappers = low_cardinality_type->getDictionaryType().get(); + if (const auto * nullable_type = typeid_cast(type_without_wrappers)) + type_without_wrappers = nullable_type->getNestedType().get(); + const IDataType & type = *type_without_wrappers; auto which = WhichDataType(type); if (which.isDateOrDate32() || which.isTime() || which.isTime64() || which.isDateTime() || which.isDateTime64() || which.isInt8() || which.isInt16() || which.isUInt8() || which.isUInt16()) @@ -4404,8 +4410,14 @@ struct ToDateTimeMonotonicity { static bool has() { return true; } - static IFunction::Monotonicity get(const IDataType & type, const Field &, const Field &) + static IFunction::Monotonicity get(const IDataType & type_with_wrappers, const Field &, const Field &) { + const IDataType * type_without_wrappers = &type_with_wrappers; + if (const auto * low_cardinality_type = typeid_cast(type_without_wrappers)) + type_without_wrappers = low_cardinality_type->getDictionaryType().get(); + if (const auto * nullable_type = typeid_cast(type_without_wrappers)) + type_without_wrappers = nullable_type->getNestedType().get(); + const IDataType & type = *type_without_wrappers; if (type.isValueRepresentedByNumber()) { auto which = WhichDataType(type); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp index 011cc96f16a01..c7c76cbd14836 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -187,12 +190,185 @@ Field decodePartitionDecimalByType(const String & bytes, const IDataType & type) } +namespace +{ + +enum class PartitionTransformKind : uint8_t +{ + Day, + Month, + Year, + Hour, + NotInvertible, +}; + +PartitionTransformKind parsePartitionTransformKind(const String & transform_name_src) +{ + const String transform_name = Poco::toLower(transform_name_src); + + if (transform_name == "day" || transform_name == "days" || transform_name == "date" || transform_name == "dates") + return PartitionTransformKind::Day; + if (transform_name == "month" || transform_name == "months") + return PartitionTransformKind::Month; + if (transform_name == "year" || transform_name == "years") + return PartitionTransformKind::Year; + if (transform_name == "hour" || transform_name == "hours") + return PartitionTransformKind::Hour; + return PartitionTransformKind::NotInvertible; +} + +struct Interval +{ + Int64 first; + Int64 past_last; +}; + +std::optional unitInterval(Int64 value) +{ + Int64 past_last = 0; + if (common::addOverflow(value, Int64{1}, past_last)) + return {}; + return Interval{value, past_last}; +} + +std::optional refineInterval(std::optional interval, Int64 factor) +{ + Int64 first = 0; + Int64 past_last = 0; + if (!interval || common::mulOverflow(interval->first, factor, first) || common::mulOverflow(interval->past_last, factor, past_last)) + return {}; + return Interval{first, past_last}; +} + +std::optional closedRange(std::optional interval, std::optional decimal_scale) +{ + Int64 last = 0; + if (!interval || common::subOverflow(interval->past_last, Int64{1}, last)) + return {}; + + if (decimal_scale) + return Range( + DecimalField(interval->first, *decimal_scale), true, DecimalField(last, *decimal_scale), true); + return Range(interval->first, true, last, true); +} + +std::optional dayIntervalOfMonthNum(Int64 month) +{ + auto months = unitInterval(month); + if (!months) + return {}; + + const auto & utc = DateLUT::instance("UTC"); + const auto epoch = ExtendedDayNum(0); + const auto first = utc.addMonths(epoch, months->first); + const auto past_last = utc.addMonths(epoch, months->past_last); + if (utc.toMonthNumSinceEpoch(first) != months->first || utc.toMonthNumSinceEpoch(past_last) != months->past_last) + return {}; + + return Interval{Int64{first}, Int64{past_last}}; +} + +std::optional dayIntervalOfYearNum(Int64 year) +{ + auto years = unitInterval(year); + if (!years) + return {}; + + const auto & utc = DateLUT::instance("UTC"); + const auto epoch = ExtendedDayNum(0); + const auto first = utc.addYears(epoch, years->first); + const auto past_last = utc.addYears(epoch, years->past_last); + if (utc.toYearSinceEpoch(first) != years->first || utc.toYearSinceEpoch(past_last) != years->past_last) + return {}; + + return Interval{Int64{first}, Int64{past_last}}; +} + +std::optional dayIntervalOfPartitionValue(PartitionTransformKind kind, Int64 value) +{ + switch (kind) + { + case PartitionTransformKind::Day: + return unitInterval(value); + case PartitionTransformKind::Month: + return dayIntervalOfMonthNum(value); + case PartitionTransformKind::Year: + return dayIntervalOfYearNum(value); + case PartitionTransformKind::Hour: + case PartitionTransformKind::NotInvertible: + return {}; + } + UNREACHABLE(); +} + +std::optional secondIntervalOfPartitionValue(PartitionTransformKind kind, Int64 value) +{ + static constexpr Int64 seconds_per_hour = 3600; + static constexpr Int64 seconds_per_day = 86400; + + switch (kind) + { + case PartitionTransformKind::Hour: + return refineInterval(unitInterval(value), seconds_per_hour); + case PartitionTransformKind::Day: + case PartitionTransformKind::Month: + case PartitionTransformKind::Year: + return refineInterval(dayIntervalOfPartitionValue(kind, value), seconds_per_day); + case PartitionTransformKind::NotInvertible: + return {}; + } + UNREACHABLE(); +} + +std::optional partitionValueAsInt64(const Field & partition_value) +{ + if (partition_value.getType() == Field::Types::Int64) + return partition_value.safeGet(); + + if (partition_value.getType() == Field::Types::UInt64) + { + const UInt64 value = partition_value.safeGet(); + if (value <= static_cast(std::numeric_limits::max())) + return static_cast(value); + } + + return {}; +} + +std::optional rangeOfPartitionValue(const String & transform_name, const Field & partition_value, const IDataType & source_type) +{ + const auto value = partitionValueAsInt64(partition_value); + if (!value) + return {}; + + const PartitionTransformKind kind = parsePartitionTransformKind(transform_name); + const WhichDataType which(source_type); + + if (which.isDateOrDate32()) + return closedRange(dayIntervalOfPartitionValue(kind, *value), std::nullopt); + + if (which.isDateTime()) + return closedRange(secondIntervalOfPartitionValue(kind, *value), std::nullopt); + + if (which.isDateTime64()) + { + const UInt32 scale = getDecimalScale(source_type); + return closedRange( + refineInterval(secondIntervalOfPartitionValue(kind, *value), DecimalUtils::scaleMultiplier(scale)), scale); + } + + return {}; +} + +} + PruningReturnStatus ManifestFilesPruner::canBePruned( const ProcessedManifestFileEntryPtr & entry, const std::unordered_map & entry_hyperrectangles) const { + const auto & partition_value = entry->parsed_entry->partition_key_value; + if (partition_key_condition.has_value()) { - const auto & partition_value = entry->parsed_entry->partition_key_value; std::vector index_value(partition_value.begin(), partition_value.end()); for (size_t i = 0; i < index_value.size(); ++i) { @@ -226,15 +402,33 @@ PruningReturnStatus ManifestFilesPruner::canBePruned( continue; } - auto rect_it = entry_hyperrectangles.find(column_id); - if (rect_it == entry_hyperrectangles.end()) - continue; - auto info_it = entry->parsed_entry->columns_infos.find(column_id); bool has_no_nulls = info_it != entry->parsed_entry->columns_infos.end() && info_it->second.nulls_count.has_value() && *info_it->second.nulls_count == 0; - if (has_no_nulls && !key_condition.mayBeTrueInRange(1, &rect_it->second.left, &rect_it->second.right, {name_and_type->type})) + const DataTypes data_types{name_and_type->type}; + + if (entry->common_partition_specification) + { + for (const auto & partition_field : *entry->common_partition_specification) + { + if (partition_field.source_id != column_id || partition_field.tuple_index < 0 + || static_cast(partition_field.tuple_index) >= partition_value.size()) + continue; + + auto range = rangeOfPartitionValue( + partition_field.transform_name, + partition_value[partition_field.tuple_index], + *removeNullable(name_and_type->type)); + + if (range && !key_condition.mayBeTrueInRange(1, &range->left, &range->right, data_types)) + return PruningReturnStatus::PARTITION_PRUNED; + } + } + + auto rect_it = entry_hyperrectangles.find(column_id); + if (has_no_nulls && rect_it != entry_hyperrectangles.end() + && !key_condition.mayBeTrueInRange(1, &rect_it->second.left, &rect_it->second.right, data_types)) { return PruningReturnStatus::MIN_MAX_INDEX_PRUNED; } diff --git a/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py b/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py new file mode 100644 index 0000000000000..51bb446b5f316 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py @@ -0,0 +1,268 @@ +import pytest + +from helpers.iceberg_utils import ( + check_validity_and_get_prunned_files_general, + execute_spark_query_general, + get_creation_expression, + get_uuid_str, +) + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_with_functions_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + ts_day TIMESTAMP, + ts_hour TIMESTAMP, + ts_month TIMESTAMP, + d_year DATE, + tag INT + ) + USING iceberg + PARTITIONED BY (days(ts_day), hours(ts_hour), months(ts_month), years(d_year)) + OPTIONS('format-version'='2') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 1), + (TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2024-01-20 11:00:00', TIMESTAMP '2024-02-20 10:00:00', DATE '2025-01-20', 2), + (TIMESTAMP '2024-02-20 10:00:00', TIMESTAMP '2024-01-21 10:00:00', TIMESTAMP '2024-03-20 10:00:00', DATE '2026-01-20', 3), + (TIMESTAMP '2025-02-20 10:00:00', TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2025-01-20 10:00:00', DATE '2027-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + # A filter that wraps the partition source column in a monotonic function must still be able to + # use the partition value, which is what https://github.com/ClickHouse/ClickHouse/issues/103433 + # reported for `toDate`. + assert check_validity_and_get_prunned_files(select("toDate(ts_day) = toDate('2024-01-20')")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfDay(ts_day) = toDateTime64('2024-01-20 00:00:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toYYYYMMDD(ts_day) = 20240120")) == 3 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) IN (toDate('2024-01-20'), toDate('2024-02-20'))")) == 2 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) > toDate('2024-02-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(ts_day) = 2025")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) = toDateTime64('2024-01-20 10:00:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toDate(ts_hour) = toDate('2024-01-21')")) == 2 + assert check_validity_and_get_prunned_files(select("toStartOfMonth(ts_month) = toDate('2024-02-01')")) == 3 + assert check_validity_and_get_prunned_files(select("toYYYYMM(ts_month) = 202503")) == 4 + assert check_validity_and_get_prunned_files(select("toYear(d_year) = 2026")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfYear(d_year) = toDate('2028-01-01')")) == 4 + + # A constant that is not aligned to the transform's granularity: 10:30 is inside the 10:00 hour, + # so the file of that hour must survive, and the same for a day and a month boundary. + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) < toDateTime64('2024-01-20 10:30:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) >= toDateTime64('2024-01-20 10:30:00', 6)")) == 1 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) < toDate('2024-01-20') + INTERVAL 12 HOUR")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfMonth(ts_month) > toDate('2024-02-10')")) == 2 + + # A single day is one weekday, so the partition value answers this too: only 2024-01-20 is a Saturday. + assert check_validity_and_get_prunned_files(select("toDayOfWeek(ts_day) = 6")) == 3 + + # The partition value of a `day` transform says nothing about the hour of the day, so a filter on + # it must not prune anything. + assert check_validity_and_get_prunned_files(select("toHour(ts_day) = 10")) == 0 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions_before_epoch(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_before_epoch_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} (ts TIMESTAMP, d DATE, tag INT) + USING iceberg + PARTITIONED BY (days(ts), years(d)) + OPTIONS('format-version'='2') + """ + ) + + # The transforms count from 1970, so these partition values are negative. + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '1969-11-15 07:00:00', DATE '1969-11-15', 1), + (TIMESTAMP '1969-12-31 23:00:00', DATE '1970-01-01', 2), + (TIMESTAMP '1970-01-01 00:30:00', DATE '1971-06-01', 3), + (TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("toDate32(ts) = toDate32('1969-11-15')")) == 3 + assert check_validity_and_get_prunned_files(select("toDate32(ts) >= toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toDate32(ts) < toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(d) = 1969")) == 3 + assert check_validity_and_get_prunned_files(select("toYear(d) >= 1971")) == 2 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions_before_epoch_separate_commits(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_before_epoch_separate_commits_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} (ts TIMESTAMP, d DATE, tag INT) + USING iceberg + PARTITIONED BY (days(ts), years(d)) + OPTIONS('format-version'='2') + """ + ) + + # One manifest file per commit, so the partition summaries of the first manifest are negative on + # both of their sides, while the ones of the second manifest are positive on both. + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '1969-11-15 07:00:00', DATE '1969-11-15', 1), + (TIMESTAMP '1969-12-31 23:00:00', DATE '1969-06-01', 2); + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 3); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("toDate32(ts) = toDate32('1969-11-15')")) == 2 + assert check_validity_and_get_prunned_files(select("toDate32(ts) >= toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(d) = 1969")) == 1 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_without_functions(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_without_functions_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + ts_day TIMESTAMP, + ts_hour TIMESTAMP, + ts_month TIMESTAMP, + d_year DATE, + tag INT + ) + USING iceberg + PARTITIONED BY (days(ts_day), hours(ts_hour), months(ts_month), years(d_year)) + OPTIONS('format-version'='2') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 1), + (TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2024-01-20 11:00:00', TIMESTAMP '2024-02-20 10:00:00', DATE '2025-01-20', 2), + (TIMESTAMP '2024-02-20 10:00:00', TIMESTAMP '2024-01-21 10:00:00', TIMESTAMP '2024-03-20 10:00:00', DATE '2026-01-20', 3), + (TIMESTAMP '2025-02-20 10:00:00', TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2025-01-20 10:00:00', DATE '2027-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("ts_day = '2024-01-20 10:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_day < '2024-01-21 00:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_day >= '2024-02-01 00:00:00'")) == 2 + assert check_validity_and_get_prunned_files(select("ts_day BETWEEN '2024-01-21 00:00:00' AND '2024-02-21 00:00:00'")) == 2 + assert check_validity_and_get_prunned_files(select("ts_day IN ('2024-01-20 10:00:00', '2024-02-20 10:00:00')")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_hour = '2024-01-20 11:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_hour > '2024-01-21 09:00:00'")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_month = '2024-02-20 10:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_month < '2024-03-01 00:00:00'")) == 2 + + assert check_validity_and_get_prunned_files(select("d_year = '2026-01-20'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year < '2025-01-01'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year >= '2026-01-01'")) == 2 + assert check_validity_and_get_prunned_files(select("d_year IN ('2024-01-20', '2026-01-20')")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_day = '2024-01-20 12:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year = '2026-06-01'")) == 3 diff --git a/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference new file mode 100644 index 0000000000000..19219d2f6180d --- /dev/null +++ b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference @@ -0,0 +1,8 @@ +Granules: 2/10 +Granules: 2/10 +Granules: 2/10 +3 15 +3 6 +6 105 +6 105 +9 120 diff --git a/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql new file mode 100644 index 0000000000000..aae7082ce17b0 --- /dev/null +++ b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql @@ -0,0 +1,33 @@ +-- The monotonicity of a conversion does not depend on the `Nullable` wrapper of its argument, so a +-- `toDate`/`toDateTime` predicate over a `Nullable` key must still be usable for index analysis. + +SET session_timezone = 'UTC'; + +DROP TABLE IF EXISTS t_nullable_key; +CREATE TABLE t_nullable_key (x Nullable(DateTime64(6)), y Int64) +ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 4, allow_nullable_key = 1; + +INSERT INTO t_nullable_key +SELECT if(number % 7 = 0, NULL, toDateTime64('2026-03-01 00:00:00', 6) + INTERVAL number * 6 HOUR), number +FROM numbers(40); + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDate(x) = toDate('2026-03-02') +) WHERE granules != ''; + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDate32(x) = toDate32('2026-03-02') +) WHERE granules != ''; + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDateTime(x) = toDateTime('2026-03-02 06:00:00') +) WHERE granules != ''; + +-- The rows the index analysis keeps must be the rows the predicate selects, `NULL`s included. +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) = toDate('2026-03-02'); +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) < toDate('2026-03-02'); +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) IS NULL; +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) IN (toDate('2026-03-02'), toDate('2026-03-08')); +SELECT count(), sum(y) FROM t_nullable_key WHERE isNull(x) OR toDate(x) = toDate('2026-03-02'); + +DROP TABLE t_nullable_key; From 2e64dd6dcd788d25c8fa269a2dc20e232bae8cbd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 26 Sep 2026 09:16:29 +0000 Subject: [PATCH 108/296] Backport #122222 to 26.8: Do not wait for the peer in TLS shutdown when a write is still pending --- .../include/Poco/Net/SecureSocketImpl.h | 3 + .../NetSSL_OpenSSL/src/SecureSocketImpl.cpp | 8 +- src/Common/tests/gtest_ssl_send_timeout.cpp | 126 ++++++++++++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h index 82b38890b7917..9e295ce5d299a 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h @@ -286,6 +286,9 @@ namespace Net Context::Ptr _pContext; bool _needHandshake; bool _fatalError; + bool _pendingWrite = false; + /// Whether the last `SSL_write` returned `SSL_ERROR_WANT_WRITE`. OpenSSL keeps that record pending + /// until `SSL_write` is retried, even after other operations change what `SSL_get_error` reports. std::string _peerHostName; Session::Ptr _pSession; const BIO_METHOD * _bioMethod = nullptr; diff --git a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp index 3636d4b5fdd36..61e0a720ba610 100644 --- a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp @@ -171,6 +171,7 @@ void SecureSocketImpl::acceptSSL() ScopedLock lock(*_mutex); poco_assert (!_pSSL); _fatalError = false; + _pendingWrite = false; BIO* pBIO = BIO_new(getBioMethod()); if (!pBIO) throw SSLException("Cannot create BIO object"); @@ -241,6 +242,7 @@ void SecureSocketImpl::connectSSL(bool performHandshake) poco_assert (!_pSSL); poco_assert (_pSocket->initialized()); _fatalError = false; + _pendingWrite = false; BIO* pBIO = BIO_new(getBioMethod()); if (!pBIO) throw SSLException("Cannot create SSL BIO object"); @@ -364,7 +366,10 @@ void SecureSocketImpl::shutdown() return SSL_shutdown(_pSSL); }, false); } - while (result.rc < 0 && mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + /// OpenSSL does not dispatch the `close_notify` alert while a record write from an + /// earlier `SSL_write` is still pending, so retrying cannot make progress then. + while (!_pendingWrite && result.rc < 0 + && mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); if (result.rc < 0) handleError(result.rc, result.sslError, result.socketError, result.errorCode); if (_pSocket->getBlocking()) @@ -419,6 +424,7 @@ int SecureSocketImpl::sendBytes(const void* buffer, int length, int flags) { return SSL_write(_pSSL, buffer, length); }); + _pendingWrite = result.sslError == SSL_ERROR_WANT_WRITE; } while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); rc = result.rc; diff --git a/src/Common/tests/gtest_ssl_send_timeout.cpp b/src/Common/tests/gtest_ssl_send_timeout.cpp index 7f56fd6059452..e17f43b4add71 100644 --- a/src/Common/tests/gtest_ssl_send_timeout.cpp +++ b/src/Common/tests/gtest_ssl_send_timeout.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,12 @@ #include +#include + +#include + +#include +#include #include #include #include @@ -100,6 +107,125 @@ TEST(SSLSocketTimeout, SendBytesThrowsTimeoutOnBlockingSocket) } +namespace +{ + +/// Checks that shutting a blocking SSL socket down after its write timed out returns +/// promptly, instead of waiting for the peer for another full I/O timeout and throwing. +void checkShutdownAfterSendTimeout(bool receive_timeout_before_shutdown) +{ + EphemeralCert cert; + auto server_ctx = cert.makeContext(Poco::Net::Context::SERVER_USE); + auto client_ctx = cert.makeContext(Poco::Net::Context::CLIENT_USE); + + Poco::Net::SecureServerSocket server_socket( + Poco::Net::SocketAddress("127.0.0.1", 0), 1, server_ctx); + auto port = server_socket.address().port(); + + std::atomic server_done{false}; + + /// Server thread: accept and handshake, then sit idle (never read). + std::jthread server_thread([&] + { + try + { + auto accepted = server_socket.acceptConnection(); + /// Handshake happens on first I/O. Do a small read to trigger it. + char buf[1]; + try { accepted.receiveBytes(buf, 1); } catch (...) {} /// Ok: handshake may fail. NOLINT(bugprone-empty-catch) + /// Keep the connection open, and unread, until the test completes. + while (!server_done.load()) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + catch (...) {} /// Ok: server thread cleanup, test checks client-side behavior. NOLINT(bugprone-empty-catch) + }); + + /// Declared after the thread so that it runs before the thread is joined, on every exit path. + /// Closing the listening socket also unblocks acceptConnection if the client never connected. + SCOPE_EXIT(server_done.store(true); server_socket.close()); + + std::optional client; + try + { + client.emplace(Poco::Net::SocketAddress("127.0.0.1", port), client_ctx); + } + catch (const Poco::Exception & e) + { + /// Connection setup can fail on some systems; skip gracefully. + GTEST_SKIP() << "SSL setup failed: " << e.displayText(); + } + + /// Very short send timeout so the test doesn't wait long. + client->setSendTimeout(Poco::Timespan(0, 200'000)); /// 200ms + + /// Write enough data to fill the TCP send buffer and SSL buffer. + /// Typical TCP buffer is 128KB-256KB. Write 4MB to be sure. + std::vector data(4 * 1024 * 1024, 'X'); + + bool got_timeout = false; + try + { + size_t offset = 0; + while (offset < data.size()) + { + int sent = client->sendBytes(data.data() + offset, static_cast(data.size() - offset)); + if (sent > 0) + offset += sent; + else + break; + } + } + catch (const Poco::TimeoutException &) + { + got_timeout = true; + } + + /// Without a timed out write there is no unsent data left behind and nothing to test. + ASSERT_TRUE(got_timeout) << "Expected Poco::TimeoutException when writing to a non-reading SSL peer"; + + auto * client_impl = static_cast(client->impl()); + SSL * ssl = client_impl->ssl(); + ASSERT_NE(ssl, nullptr); + + if (receive_timeout_before_shutdown) + { + /// The peer never writes either, so the read times out too. + client->setReceiveTimeout(Poco::Timespan(0, 200'000)); /// 200ms + char buf[1]; + EXPECT_THROW(client->receiveBytes(buf, 1), Poco::TimeoutException); + /// The read replaces the state that `SSL_want_write` reports, while the write stays pending. + ASSERT_FALSE(SSL_want_write(ssl)); + } + + /// The shutdown budget is max(send timeout, receive timeout), so raising the receive + /// timeout now separates a shutdown that waits for the peer from one that does not. + client->setReceiveTimeout(Poco::Timespan(10, 0)); /// 10s + + const auto started = std::chrono::steady_clock::now(); + EXPECT_NO_THROW(client->shutdown()); + const auto elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - started).count(); + + EXPECT_LT(elapsed_ms, 2000) << "shutdown() blocked for " << elapsed_ms + << "ms waiting for a peer that is not reading"; + + /// The orderly TLS shutdown must still be attempted once, so skipping it entirely does not pass. + EXPECT_TRUE(SSL_get_shutdown(ssl) & SSL_SENT_SHUTDOWN); +} + +} + +TEST(SSLSocketTimeout, ShutdownAfterSendTimeoutDoesNotWaitForPeer) +{ + checkShutdownAfterSendTimeout(/* receive_timeout_before_shutdown= */ false); +} + +TEST(SSLSocketTimeout, ShutdownAfterSendAndReceiveTimeoutsDoesNotWaitForPeer) +{ + checkShutdownAfterSendTimeout(/* receive_timeout_before_shutdown= */ true); +} + + /// Test that SSL handshake throws TimeoutException when the peer /// is a plain TCP listener that never speaks SSL. /// No server thread needed -- the kernel's listen backlog completes the From d7f2b255cfa15d6e1e7f2b421743ad66f4b58758 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 26 Sep 2026 15:54:22 +0000 Subject: [PATCH 109/296] Update autogenerated version to 26.8.12.53 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 16b893d9fd0d8..88fe83ae7ce7f 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54524) +SET(VERSION_REVISION 54525) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 12) -SET(VERSION_GITHASH 97da26e305d9764402a30702c9b779745bf557c4) -SET(VERSION_DESCRIBE v26.8.12.1-lts) -SET(VERSION_STRING 26.8.12.1) +SET(VERSION_PATCH 13) +SET(VERSION_GITHASH 1968835eab6e2d2030697e4ff1ec5692d628fb13) +SET(VERSION_DESCRIBE v26.8.13.1-lts) +SET(VERSION_STRING 26.8.13.1) # end of autochange From 18eed45f781a95d4b9540f1cb7a2a8e83c72d172 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 26 Sep 2026 17:33:37 +0000 Subject: [PATCH 110/296] Backport #121711 to 26.8: Split a small single-level `GROUP BY` result into several chunks when the aggregation output fans out --- src/Interpreters/Aggregator.cpp | 62 ++++++---- src/Interpreters/Aggregator.h | 23 ++-- src/Processors/QueryPlan/AggregatingStep.cpp | 14 ++- .../QueryPlan/MergingAggregatedStep.cpp | 3 +- .../Transforms/AggregatingTransform.cpp | 29 ++++- .../Transforms/AggregatingTransform.h | 8 +- .../Transforms/MergingAggregatedTransform.cpp | 6 +- .../Transforms/MergingAggregatedTransform.h | 5 +- ...aggregation_single_level_result_fanout.xml | 111 ++++++++++++++++++ ...ation_single_level_result_fanout.reference | 6 + ...aggregation_single_level_result_fanout.sql | 30 +++++ 11 files changed, 255 insertions(+), 42 deletions(-) create mode 100644 tests/performance/aggregation_single_level_result_fanout.xml create mode 100644 tests/queries/0_stateless/05257_aggregation_single_level_result_fanout.reference create mode 100644 tests/queries/0_stateless/05257_aggregation_single_level_result_fanout.sql diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 0d5901bff0a57..b526a8ac129d6 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -117,6 +117,21 @@ bool worthConvertToTwoLevel( || (group_by_two_level_threshold_bytes && result_size_bytes >= static_cast(group_by_two_level_threshold_bytes)); } +/// The row capacity of each chunk that `convertToBlockImpl` emits. +1 for `nullKeyData`: if the table +/// doesn't have it, that's not a problem, just memory for one excessive row is preallocated. +/// A non-zero `max_rows_per_block` lowers the `max_block_size` bound so that a table smaller than +/// one block can still be emitted as several chunks. +size_t convertedBlockSize(size_t table_size, size_t max_block_size, size_t max_rows_per_block, bool return_single_block) +{ + if (return_single_block) + return table_size + 1; + + if (max_rows_per_block) + max_block_size = std::min(max_block_size, max_rows_per_block); + + return std::min(max_block_size, table_size) + 1; +} + void initDataVariantsWithSizeHint( DB::AggregatedDataVariants & result, DB::AggregatedDataVariants::Type method_chosen, const DB::Aggregator::Params & params) { @@ -3334,7 +3349,7 @@ void Aggregator::disableMinMaxOptimizationForFixedHashMaps(ManyAggregatedDataVar template requires SetAggregationMethod Chunks -Aggregator::convertToBlockImpl(Method & method, Table & data, Arena *, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block) const +Aggregator::convertToBlockImpl(Method & method, Table & data, Arena *, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block, size_t max_rows_per_block) const { if (data.empty()) { @@ -3344,7 +3359,7 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena *, Arenas & return result; } - Chunks res = convertToBlockImplKeysOnly(method, data, aggregates_pools, final, return_single_block); + Chunks res = convertToBlockImplKeysOnly(method, data, aggregates_pools, final, return_single_block, max_rows_per_block); /// In order to release memory early. data.clearAndShrink(); @@ -3355,7 +3370,7 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena *, Arenas & template requires MapAggregationMethod Chunks -Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final,size_t rows, bool return_single_block) const +Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block, size_t max_rows_per_block) const { if (data.empty()) { @@ -3368,8 +3383,7 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Are if (is_simple_count) { - /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated - const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1; + const size_t max_block_size = convertedBlockSize(data.size(), params.max_block_size, max_rows_per_block, return_single_block); std::optional out_cols; std::optional shuffled_key_sizes; @@ -3483,11 +3497,11 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Are #if USE_EMBEDDED_COMPILER use_compiled_functions = compiled_aggregate_functions_holder != nullptr && !Method::low_cardinality_optimization; #endif - res = convertToBlockImplFinal(method, data, arena, aggregates_pools, use_compiled_functions, return_single_block); + res = convertToBlockImplFinal(method, data, arena, aggregates_pools, use_compiled_functions, return_single_block, max_rows_per_block); } else { - res = convertToBlockImplNotFinal(method, data, aggregates_pools, rows, return_single_block); + res = convertToBlockImplNotFinal(method, data, aggregates_pools, rows, return_single_block, max_rows_per_block); } /// In order to release memory early. @@ -3661,10 +3675,9 @@ Chunk Aggregator::insertResultsIntoColumns( template requires SetAggregationMethod Chunks Aggregator::convertToBlockImplKeysOnly( - Method & method, Table & data, Arenas & aggregates_pools, bool final, bool return_single_block) const + Method & method, Table & data, Arenas & aggregates_pools, bool final, bool return_single_block, size_t max_rows_per_block) const { - /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated - const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1; + const size_t max_block_size = convertedBlockSize(data.size(), params.max_block_size, max_rows_per_block, return_single_block); std::optional out_cols; std::optional shuffled_key_sizes; @@ -3730,10 +3743,10 @@ Chunks Aggregator::convertToBlockImplFinal( Arena * arena, Arenas & aggregates_pools, bool use_compiled_functions [[maybe_unused]], - bool return_single_block) const + bool return_single_block, + size_t max_rows_per_block) const { - /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated - const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1; + const size_t max_block_size = convertedBlockSize(data.size(), params.max_block_size, max_rows_per_block, return_single_block); const bool final = true; std::optional out_cols; @@ -3811,10 +3824,9 @@ Chunks Aggregator::convertToBlockImplFinal( template Chunks NO_INLINE -Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t, bool return_single_block) const +Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t, bool return_single_block, size_t max_rows_per_block) const { - /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated - const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1; + const size_t max_block_size = convertedBlockSize(data.size(), params.max_block_size, max_rows_per_block, return_single_block); const bool final = false; Chunks res_chunks; @@ -3989,7 +4001,7 @@ Aggregator::AggregatedChunk Aggregator::prepareChunkAndFillWithoutKey(Aggregated template std::conditional_t -Aggregator::prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const +Aggregator::prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final, size_t max_rows_per_block) const { Chunks res_variant; const size_t rows = data_variants.sizeWithoutOverflowRow(); @@ -3997,7 +4009,7 @@ Aggregator::prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variant else if (data_variants.type == AggregatedDataVariants::Type::NAME) \ { \ res_variant = convertToBlockImpl( \ - *data_variants.NAME, data_variants.NAME->data, data_variants.aggregates_pool, data_variants.aggregates_pools, final, rows, return_single_block); \ + *data_variants.NAME, data_variants.NAME->data, data_variants.aggregates_pool, data_variants.aggregates_pools, final, rows, return_single_block, max_rows_per_block); \ } if (false) {} // NOLINT @@ -4092,7 +4104,7 @@ Aggregator::AggregatedChunks Aggregator::prepareChunksAndFillTwoLevelImpl(Aggreg } -Aggregator::AggregatedChunks Aggregator::convertToChunks(AggregatedDataVariants & data_variants, bool final) const +Aggregator::AggregatedChunks Aggregator::convertToChunks(AggregatedDataVariants & data_variants, bool final, size_t max_rows_per_block) const { LOG_TRACE(log, "Converting aggregated data to chunks"); @@ -4111,7 +4123,7 @@ Aggregator::AggregatedChunks Aggregator::convertToChunks(AggregatedDataVariants if (data_variants.type != AggregatedDataVariants::Type::without_key) { if (!data_variants.isTwoLevel()) - chunks.splice(chunks.end(), prepareChunkAndFillSingleLevel(data_variants, final)); + chunks.splice(chunks.end(), prepareChunkAndFillSingleLevel(data_variants, final, max_rows_per_block)); else chunks.splice(chunks.end(), prepareChunksAndFillTwoLevel(data_variants, final)); } @@ -4142,6 +4154,16 @@ Aggregator::AggregatedChunks Aggregator::convertToChunks(AggregatedDataVariants return chunks; } +size_t Aggregator::singleLevelChunkRowsForFanOut(size_t rows, size_t output_streams) +{ + static constexpr size_t MIN_ROWS_PER_CHUNK{512}; + const size_t num_chunks = std::clamp(rows / MIN_ROWS_PER_CHUNK, 1, std::max(output_streams, 1)); + if (num_chunks <= 1) + return 0; + + return (rows + num_chunks - 1) / num_chunks; +} + template void NO_INLINE Aggregator::mergeDataNullKey( diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 2ae75015b007b..48c2e5c8a9f26 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -460,8 +460,15 @@ class Aggregator final * If final = false, then ColumnAggregateFunction is created as the aggregation columns with the state of the calculations, * which can then be combined with other states (for distributed query processing). * If final = true, then columns with ready values are created as aggregate columns. + * A non-zero `max_rows_per_block` caps the size of the emitted single-level chunks below `max_block_size`. */ - AggregatedChunks convertToChunks(AggregatedDataVariants & data_variants, bool final) const; + AggregatedChunks convertToChunks(AggregatedDataVariants & data_variants, bool final, size_t max_rows_per_block = 0) const; + + /// A single-level result smaller than `max_block_size` is converted to one chunk, and a `Resize` + /// hands out whole chunks, so everything downstream of it runs in one thread. Returns a chunk size + /// that splits `rows` into about one chunk per output stream, never below 512 rows per chunk, or 0 + /// to leave the result as is. + static size_t singleLevelChunkRowsForFanOut(size_t rows, size_t output_streams); /// `adaptive_session` (or nullptr when the adaptive aggregation is off) feeds the /// thaw verdict into the hash-table statistics next to the observed sizes. @@ -1034,13 +1041,13 @@ class Aggregator final template requires MapAggregationMethod Chunks - convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block) const; + convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block, size_t max_rows_per_block = 0) const; /// A set method skips the inline-count and compiled-function paths; it only emits keys. template requires SetAggregationMethod Chunks - convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block) const; + convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block, size_t max_rows_per_block = 0) const; template void insertAggregatesIntoColumns( @@ -1060,7 +1067,7 @@ class Aggregator final template requires SetAggregationMethod Chunks convertToBlockImplKeysOnly( - Method & method, Table & data, Arenas & aggregates_pools, bool final, bool return_single_block) const; + Method & method, Table & data, Arenas & aggregates_pools, bool final, bool return_single_block, size_t max_rows_per_block) const; template Chunks convertToBlockImplFinal( @@ -1069,11 +1076,12 @@ class Aggregator final Arena * arena, Arenas & aggregates_pools, bool use_compiled_functions, - bool return_single_block) const; + bool return_single_block, + size_t max_rows_per_block) const; template Chunks - convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t rows, bool return_single_block) const; + convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t rows, bool return_single_block, size_t max_rows_per_block) const; /// `topk_full_key_bytes`, when non-null and the bucket goes through the Top-K conversion, /// receives the byte size all of the bucket's keys would occupy materialized: the runtime @@ -1124,9 +1132,10 @@ class Aggregator final AggregatedChunk prepareChunkAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final, bool is_overflows) const; AggregatedChunks prepareChunksAndFillTwoLevel(AggregatedDataVariants & data_variants, bool final) const; + /// A non-zero `max_rows_per_block` caps the size of the emitted chunks below `max_block_size`. template std::conditional_t - prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const; + prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final, size_t max_rows_per_block = 0) const; template AggregatedChunks prepareChunksAndFillTwoLevelImpl(AggregatedDataVariants & data_variants, Method & method, bool final) const; diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 272d0f4f0e352..08fe3c68f92fd 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -563,6 +563,10 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B }); } + /// The per-set results are spread over `max_threads` streams below, so split a small single-level + /// result of each set for that width as for an ordinary aggregation. + const size_t grouping_sets_output_streams = should_produce_results_in_order_of_bucket_number ? 1 : params.max_threads; + pipeline.transform([&](OutputPortRawPtrs ports) { chassert(streams * grouping_sets_size == ports.size()); @@ -586,7 +590,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B new_temporary_data_merge_threads, should_produce_results_in_order_of_bucket_number, skip_merging, - nullptr); + nullptr, + grouping_sets_output_streams); // For each input stream we have `grouping_sets_size` copies, so port index // for transform #j should skip ports of first (j-1) streams. connect(*ports[i + grouping_sets_size * j], aggregation_for_set->getInputs().front()); @@ -597,7 +602,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B else { auto aggregation_for_set - = std::make_shared(input_header, transform_params_for_set, dataflow_cache_updater); + = std::make_shared(input_header, transform_params_for_set, dataflow_cache_updater, grouping_sets_output_streams); connect(*ports[i], aggregation_for_set->getInputs().front()); ports[i] = &aggregation_for_set->getOutputs().front(); processors.push_back(aggregation_for_set); @@ -859,7 +864,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B new_temporary_data_merge_threads, should_produce_results_in_order_of_bucket_number, skip_merging, - dataflow_cache_updater); + dataflow_cache_updater, + streams_after_aggregation); }); pipeline.resize(streams_after_aggregation, false, settings.min_outstreams_per_resize_after_split); @@ -869,7 +875,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B else { pipeline.addSimpleTransform([&](const SharedHeader & header) - { return std::make_shared(header, transform_params, dataflow_cache_updater); }); + { return std::make_shared(header, transform_params, dataflow_cache_updater, streams_after_aggregation); }); pipeline.resize(streams_after_aggregation); diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index 95528696962e6..3f64bfea764a5 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -151,7 +151,8 @@ void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, c pipeline.resize(1); /// Now merge the aggregated blocks - auto transform = std::make_shared(pipeline.getSharedHeader(), params, final, grouping_sets_params); + auto transform = std::make_shared(pipeline.getSharedHeader(), params, final, grouping_sets_params, + should_produce_results_in_order_of_bucket_number ? 1 : max_threads); pipeline.addTransform(std::move(transform)); } else diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index e03ad193b77ea..6a7c49764e0f7 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -546,6 +546,7 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor AggregatingTransformParamsPtr params_, ManyAggregatedDataVariantsPtr data_, size_t num_threads_, + size_t output_streams_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_, AdaptiveAggregationSessionPtr adaptive_session_) : IProcessor({}, {params_->getHeader()}) @@ -553,6 +554,7 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor , data(std::move(data_)) , shared_data(std::make_shared()) , num_threads(num_threads_) + , output_streams(output_streams_) , updater(std::move(updater_)) , adaptive_session(std::move(adaptive_session_)) { @@ -907,6 +909,11 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor size_t num_threads; + /// How many streams the output is spread over downstream. It is not `num_threads`. That is capped by the + /// number of aggregating streams (1 for a single input stream), while the `Resize` after the aggregation + /// fans out to `max_threads`. 1 when the results must go out in bucket order. + size_t output_streams; + RuntimeDataflowStatisticsCacheUpdaterPtr updater; AdaptiveAggregationSessionPtr adaptive_session; @@ -1003,7 +1010,11 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor throw Exception(ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT, "Unknown aggregated data variant."); } - auto agg_chunks = params->aggregator.prepareChunkAndFillSingleLevel(*first, params->final); + const size_t max_rows_per_block = Aggregator::singleLevelChunkRowsForFanOut(first->sizeWithoutOverflowRow(), output_streams); + if (max_rows_per_block) + LOG_TRACE(getLogger("AggregatingTransform"), "Split single level result into chunks of at most {} rows.", max_rows_per_block); + + auto agg_chunks = params->aggregator.prepareChunkAndFillSingleLevel(*first, params->final, max_rows_per_block); for (auto & agg_chunk : agg_chunks) { if (agg_chunk.chunk.getNumRows() > 0) @@ -1086,7 +1097,7 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor }; AggregatingTransform::AggregatingTransform( - SharedHeader header, AggregatingTransformParamsPtr params_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_) + SharedHeader header, AggregatingTransformParamsPtr params_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_, size_t output_streams_) : AggregatingTransform( std::move(header), std::move(params_), @@ -1096,7 +1107,8 @@ AggregatingTransform::AggregatingTransform( 1, true /* should_produce_results_in_order_of_bucket_number */, false /* skip_merging */, - updater_) + updater_, + output_streams_) { } @@ -1109,7 +1121,8 @@ AggregatingTransform::AggregatingTransform( size_t temporary_data_merge_threads_, bool should_produce_results_in_order_of_bucket_number_, bool skip_merging_, - RuntimeDataflowStatisticsCacheUpdaterPtr updater_) + RuntimeDataflowStatisticsCacheUpdaterPtr updater_, + size_t output_streams_) : IProcessor({std::move(header)}, {params_->getHeader()}) , params(std::move(params_)) , key_columns(params->params.keys_size) @@ -1121,6 +1134,7 @@ AggregatingTransform::AggregatingTransform( , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) , skip_merging(skip_merging_) , updater(std::move(updater_)) + , output_streams(output_streams_) { /// `AggregatingStep` leaves its engagement verdict in the flag. Without a producer nothing is ever /// staged, so the merge-time drains find empty backlogs and do nothing. @@ -1412,7 +1426,12 @@ void AggregatingTransform::initGenerate() std::move(many_data->variants), adaptive_context ? adaptive_context->session.get() : nullptr); auto prepared_data_ptr = std::make_shared(std::move(prepared_data)); processors.emplace_back(std::make_shared( - params, std::move(prepared_data_ptr), max_threads, updater, adaptive_engaged ? adaptive_context->session : nullptr)); + params, + std::move(prepared_data_ptr), + max_threads, + output_streams, + updater, + adaptive_engaged ? adaptive_context->session : nullptr)); } else { diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 6be36c2f76770..9842346a5e2ff 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -115,7 +115,7 @@ using ManyAggregatedDataPtr = std::shared_ptr; class AggregatingTransform final : public IProcessor { public: - AggregatingTransform(SharedHeader header, AggregatingTransformParamsPtr params_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_); + AggregatingTransform(SharedHeader header, AggregatingTransformParamsPtr params_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_, size_t output_streams_ = 1); /// For Parallel aggregating. AggregatingTransform( @@ -127,7 +127,8 @@ class AggregatingTransform final : public IProcessor size_t temporary_data_merge_threads, bool should_produce_results_in_order_of_bucket_number_ = true, bool skip_merging_ = false, - RuntimeDataflowStatisticsCacheUpdaterPtr updater_ = nullptr); + RuntimeDataflowStatisticsCacheUpdaterPtr updater_ = nullptr, + size_t output_streams_ = 1); ~AggregatingTransform() override; @@ -195,6 +196,9 @@ class AggregatingTransform final : public IProcessor RuntimeDataflowStatisticsCacheUpdaterPtr updater; + /// How many streams `AggregatingStep` spreads this transform's output over; 1 when it doesn't. + size_t output_streams = 1; + void initGenerate(); }; diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 6c3a11a3944fb..4e080b4931aee 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -64,8 +64,9 @@ static ActionsDAG makeReorderingActions(const Block & in_header, const GroupingS MergingAggregatedTransform::~MergingAggregatedTransform() = default; MergingAggregatedTransform::MergingAggregatedTransform( - SharedHeader header_, Aggregator::Params params, bool final, GroupingSetsParamsList grouping_sets_params) + SharedHeader header_, Aggregator::Params params, bool final, GroupingSetsParamsList grouping_sets_params, size_t output_streams_) : IAccumulatingTransform(header_, std::make_shared(appendGroupingIfNeeded(*header_, params.getHeader(*header_, final)))) + , output_streams(output_streams_) { if (!grouping_sets_params.empty()) { @@ -258,7 +259,8 @@ Chunk MergingAggregatedTransform::generate() /// TODO: this operation can be made async. Add async for IAccumulatingTransform. params->aggregator.mergeBlocks(std::move(bucket_to_chunks), data_variants, is_cancelled); - auto merged_chunks = params->aggregator.convertToChunks(data_variants, params->final); + const size_t max_rows_per_block = Aggregator::singleLevelChunkRowsForFanOut(data_variants.sizeWithoutOverflowRow(), output_streams); + auto merged_chunks = params->aggregator.convertToChunks(data_variants, params->final, max_rows_per_block); if (grouping_set.creating_missing_keys_actions) { diff --git a/src/Processors/Transforms/MergingAggregatedTransform.h b/src/Processors/Transforms/MergingAggregatedTransform.h index 69388c021dd33..4ace709bafe9e 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.h +++ b/src/Processors/Transforms/MergingAggregatedTransform.h @@ -15,7 +15,7 @@ using ExpressionActionsPtr = std::shared_ptr; class MergingAggregatedTransform final : public IAccumulatingTransform { public: - MergingAggregatedTransform(SharedHeader header_, Aggregator::Params params_, bool final_, GroupingSetsParamsList grouping_sets_params); + MergingAggregatedTransform(SharedHeader header_, Aggregator::Params params_, bool final_, GroupingSetsParamsList grouping_sets_params, size_t output_streams_ = 1); ~MergingAggregatedTransform() override; @@ -50,6 +50,9 @@ class MergingAggregatedTransform final : public IAccumulatingTransform bool consume_started = false; bool generate_started = false; + /// How many streams `MergingAggregatedStep` spreads the output over; 1 when it doesn't. + size_t output_streams = 1; + void addChunk(Columns columns, size_t num_rows, Int32 bucket_num, bool is_overflows); }; diff --git a/tests/performance/aggregation_single_level_result_fanout.xml b/tests/performance/aggregation_single_level_result_fanout.xml new file mode 100644 index 0000000000000..e7a9f16cc9f3e --- /dev/null +++ b/tests/performance/aggregation_single_level_result_fanout.xml @@ -0,0 +1,111 @@ + + + + 8 + 0 + 0 + + + CREATE TABLE single_level_fanout_t1 (oid UInt64, cid Int32, cr DateTime, h String, s1 Int32) ENGINE = MergeTree ORDER BY oid + CREATE TABLE single_level_fanout_t2 (oid UInt64) ENGINE = MergeTree ORDER BY oid + CREATE TABLE single_level_fanout_t3 (cid Int32, h String, ref DateTime, s2 Float64) ENGINE = MergeTree ORDER BY (cid, h, ref) + + + INSERT INTO single_level_fanout_t1 + SELECT + intDiv(number, 5), + toInt32(intDiv(number, 5) % 4), + toDateTime('2026-09-08 00:00:00') + toIntervalMinute(intDiv(number, 5) % 1440) + toIntervalSecond(number % 5), + toString(intDiv(number, 5) % 200), + toInt32(number % 300) + FROM numbers(300000) + + INSERT INTO single_level_fanout_t2 SELECT number FROM numbers(60000) + + INSERT INTO single_level_fanout_t3 + SELECT + toInt32(number % 4), + toString(intDiv(number, 4) % 200), + toDateTime('2026-09-08 00:00:00') + toIntervalMinute(intDiv(number, 800)), + toFloat64(number % 5) / 2 + FROM numbers(1152000) + + + + + join_algorithm + + hash + parallel_hash + + + + + = e.t0 - toIntervalMinute(3) AND p.ref <= e.t0 + toIntervalMinute(1) + GROUP BY e.oid, e.cid, e.h, e.s + ) + GROUP BY cid + FORMAT Null + SETTINGS join_algorithm = '{join_algorithm}' + ]]> + + + = e.t0 - toIntervalMinute(3) AND p.ref <= e.t0 + toIntervalMinute(1) + GROUP BY e.oid, e.cid, e.h, e.s + ) + GROUP BY cid + FORMAT Null + SETTINGS enable_parallel_single_level_merge = 1, enable_adaptive_aggregator = 1 + ]]> + + DROP TABLE IF EXISTS single_level_fanout_t1 + DROP TABLE IF EXISTS single_level_fanout_t2 + DROP TABLE IF EXISTS single_level_fanout_t3 + diff --git a/tests/queries/0_stateless/05257_aggregation_single_level_result_fanout.reference b/tests/queries/0_stateless/05257_aggregation_single_level_result_fanout.reference new file mode 100644 index 0000000000000..a9e2f17562ae7 --- /dev/null +++ b/tests/queries/0_stateless/05257_aggregation_single_level_result_fanout.reference @@ -0,0 +1,6 @@ +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/05257_aggregation_single_level_result_fanout.sql b/tests/queries/0_stateless/05257_aggregation_single_level_result_fanout.sql new file mode 100644 index 0000000000000..73843a1473830 --- /dev/null +++ b/tests/queries/0_stateless/05257_aggregation_single_level_result_fanout.sql @@ -0,0 +1,30 @@ +-- A single-level result smaller than max_block_size is split into several chunks when the aggregation +-- output fans out, so the step after it does not run in one thread. 20000 groups stay single-level. +DROP TABLE IF EXISTS t_fanout; +CREATE TABLE t_fanout (n UInt64) ENGINE = MergeTree ORDER BY n AS SELECT number FROM numbers(200000); + +-- Several aggregating streams, serial single-level merge. +SELECT max(bs) < 20000 FROM (SELECT blockSize() AS bs FROM (SELECT number % 20000 AS k, count() FROM numbers_mt(200000) GROUP BY k)) +SETTINGS max_threads = 4, enable_parallel_single_level_merge = 0; + +-- One aggregating stream. +SELECT max(bs) < 20000 FROM (SELECT blockSize() AS bs FROM (SELECT number % 20000 AS k, count() FROM numbers(200000) GROUP BY k)) +SETTINGS max_threads = 4; + +-- Merge of results from remote shards. +SELECT max(bs) < 20000 FROM (SELECT blockSize() AS bs FROM (SELECT n % 20000 AS k, count() FROM remote('127.0.0.{1,2}', currentDatabase(), t_fanout) GROUP BY k)) +SETTINGS max_threads = 4, distributed_aggregation_memory_efficient = 0; + +-- Results are unchanged by the split. +SELECT sum(c) = 200000 AND count() = 20000 FROM (SELECT number % 20000 AS k, count() AS c FROM numbers(200000) GROUP BY k) SETTINGS max_threads = 4; + +-- A result above max_block_size is also split into about one chunk per output stream, not into +-- max_block_size chunks: 3000 groups used to come out as 2001 + 999 rows, now as four chunks of about 750. +SELECT max(bs) < 1000 FROM (SELECT blockSize() AS bs FROM (SELECT number % 3000 AS k, count() FROM numbers_mt(30000) GROUP BY k)) +SETTINGS max_threads = 4, max_block_size = 2000, enable_parallel_single_level_merge = 0; + +-- The same split applies to each set of GROUPING SETS. +SELECT max(bs) < 20000 FROM (SELECT blockSize() AS bs FROM (SELECT number % 20000 AS k, number % 3 AS g, count() FROM numbers_mt(200000) GROUP BY GROUPING SETS ((k), (g)))) +SETTINGS max_threads = 4; + +DROP TABLE t_fanout; From 783d557efa25bfee0b5d35c86787b819f5f5ddbf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 27 Sep 2026 13:33:29 +0000 Subject: [PATCH 111/296] Update autogenerated version to 26.8.13.2 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 88fe83ae7ce7f..af630df25c06c 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54525) +SET(VERSION_REVISION 54526) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 13) -SET(VERSION_GITHASH 1968835eab6e2d2030697e4ff1ec5692d628fb13) -SET(VERSION_DESCRIBE v26.8.13.1-lts) -SET(VERSION_STRING 26.8.13.1) +SET(VERSION_PATCH 14) +SET(VERSION_GITHASH 3eac80eef9b1f81e88b581a6c107c59f21d9e15c) +SET(VERSION_DESCRIBE v26.8.14.1-lts) +SET(VERSION_STRING 26.8.14.1) # end of autochange From b44da491bb423ce04ede58be4a607ff8b83b4baf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 27 Sep 2026 15:36:45 +0000 Subject: [PATCH 112/296] Backport #117387 to 26.8: Reload TLS CA certificates (caConfig) without restart --- .../features/security/tls/configuring-tls.mdx | 13 + .../_server_settings_outside_source.mdx | 4 +- programs/keeper/Keeper.cpp | 14 +- programs/server/Server.cpp | 33 ++- src/Common/Config/ConfigReloader.cpp | 13 +- src/Coordination/KeeperServer.cpp | 6 +- src/Core/ServerSettings.cpp | 4 +- src/Server/CertificateReloader.cpp | 215 +++++++++++++- src/Server/CertificateReloader.h | 54 +++- .../System/StorageSystemCertificates.cpp | 17 +- .../test_reload_ca_certificate/__init__.py | 0 .../test_reload_ca_certificate/certs/ca.crt | 20 ++ .../test_reload_ca_certificate/certs/ca1.crt | 20 ++ .../test_reload_ca_certificate/certs/ca1.key | 28 ++ .../test_reload_ca_certificate/certs/ca2.crt | 20 ++ .../test_reload_ca_certificate/certs/ca2.key | 28 ++ .../certs/cert1.crt | 21 ++ .../certs/cert1.key | 28 ++ .../certs/cert2.crt | 21 ++ .../certs/cert2.key | 28 ++ .../certs/generate_certs.sh | 41 +++ .../test_reload_ca_certificate/certs/node.crt | 21 ++ .../test_reload_ca_certificate/certs/node.key | 28 ++ .../configs/keeper1.xml | 36 +++ .../configs/keeper2.xml | 36 +++ .../configs/keeper3.xml | 36 +++ .../configs/ssl.xml | 31 ++ .../configs/ssl_with_default_cas.xml | 29 ++ .../test_reload_ca_certificate/test.py | 275 ++++++++++++++++++ 29 files changed, 1078 insertions(+), 42 deletions(-) create mode 100644 tests/integration/test_reload_ca_certificate/__init__.py create mode 100644 tests/integration/test_reload_ca_certificate/certs/ca.crt create mode 100644 tests/integration/test_reload_ca_certificate/certs/ca1.crt create mode 100644 tests/integration/test_reload_ca_certificate/certs/ca1.key create mode 100644 tests/integration/test_reload_ca_certificate/certs/ca2.crt create mode 100644 tests/integration/test_reload_ca_certificate/certs/ca2.key create mode 100644 tests/integration/test_reload_ca_certificate/certs/cert1.crt create mode 100644 tests/integration/test_reload_ca_certificate/certs/cert1.key create mode 100644 tests/integration/test_reload_ca_certificate/certs/cert2.crt create mode 100644 tests/integration/test_reload_ca_certificate/certs/cert2.key create mode 100755 tests/integration/test_reload_ca_certificate/certs/generate_certs.sh create mode 100644 tests/integration/test_reload_ca_certificate/certs/node.crt create mode 100644 tests/integration/test_reload_ca_certificate/certs/node.key create mode 100644 tests/integration/test_reload_ca_certificate/configs/keeper1.xml create mode 100644 tests/integration/test_reload_ca_certificate/configs/keeper2.xml create mode 100644 tests/integration/test_reload_ca_certificate/configs/keeper3.xml create mode 100644 tests/integration/test_reload_ca_certificate/configs/ssl.xml create mode 100644 tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml create mode 100644 tests/integration/test_reload_ca_certificate/test.py diff --git a/docs/concepts/features/security/tls/configuring-tls.mdx b/docs/concepts/features/security/tls/configuring-tls.mdx index be54610c4cdcd..ca80687a4e259 100644 --- a/docs/concepts/features/security/tls/configuring-tls.mdx +++ b/docs/concepts/features/security/tls/configuring-tls.mdx @@ -645,6 +645,19 @@ For `clickhouse-client`, you can also use the `--accept-invalid-certificate` CLI ``` +## Rotating certificates and CA certificates without a restart {#rotating-certificates-without-restart} + +ClickHouse server and ClickHouse Keeper watch the files referenced by `certificateFile`, `privateKeyFile` and `caConfig` +in the `openSSL.server` and `openSSL.client` sections (and in `protocols.*` for composable protocols), or the files in the +directory if `caConfig` is a directory. When one of these files changes, or when `SYSTEM RELOAD CONFIG` is executed, the +certificates are reloaded and used for all new TLS connections, including HTTPS, the secure native protocol, interserver +connections, connections to Keeper and the Raft connections between Keeper nodes. Established connections keep using the +certificates they were opened with. + +To rotate a CA without downtime, first replace the `caConfig` file with a bundle that contains both the old and the new +CA certificate, then switch the node and client certificates to ones issued by the new CA, and finally replace the bundle +with the new CA certificate only. + ## Summary {#summary} This article focused on getting a ClickHouse environment configured with TLS. The settings will differ for different requirements in production environments; for example, certificate verification levels, protocols, ciphers, etc. But you should now have a good understanding of the steps involved in configuring and implementing secure connections. diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 03357b3470b04..23c3fedd9bc7d 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1159,12 +1159,14 @@ SSL client/server configuration. Support for SSL is provided by the `libpoco` library. The available configuration options are explained in [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h). Default values can be found in [SSLManager.cpp](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/src/SSLManager.cpp). +The files referenced by `certificateFile`, `privateKeyFile` and `caConfig` are reloaded without a restart when they change or on `SYSTEM RELOAD CONFIG`. New connections use the reloaded certificates, established connections are not affected. + Keys for server/client settings: | Option | Description | Default Value | |-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| | `cacheSessions` | Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Acceptable values: `true`, `false`. | `false` | -| `caConfig` | Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://www.openssl.org/docs/man3.0/man3/SSL_CTX_load_verify_locations.html). | | +| `caConfig` | Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://www.openssl.org/docs/man3.0/man3/SSL_CTX_load_verify_locations.html). Like `certificateFile` and `privateKeyFile`, the CA certificates are reloaded without a restart when the file changes or on `SYSTEM RELOAD CONFIG`. | | | `certificateFile` | Path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contains the certificate. | | | `cipherList` | Supported OpenSSL encryptions. | `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH` | | `disableProtocols` | Protocols that are not allowed to be used. | | diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index fe530bd9d2f62..f86cb3a015219 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -626,14 +626,14 @@ try Coordination::EventPtr unused_event = std::make_shared(); - const std::string cert_path = config().getString("openSSL.server.certificateFile", ""); - const std::string key_path = config().getString("openSSL.server.privateKeyFile", ""); - + /// TLS certificates, keys and CA certificates are reloaded by CertificateReloader when these files change. std::vector extra_paths = {include_from_path}; - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); + for (const auto * key : {"openSSL.server.certificateFile", "openSSL.server.privateKeyFile", "openSSL.server.caConfig", + "openSSL.client.certificateFile", "openSSL.client.privateKeyFile", "openSSL.client.caConfig"}) + { + if (auto file_path = config().getString(key, ""); !file_path.empty()) + extra_paths.emplace_back(std::move(file_path)); + } /// ConfigReloader have to strict parameters which are redundant in our case auto main_config_reloader = std::make_unique( diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ad78a0d204a21..8877da89122b2 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -487,6 +487,10 @@ namespace ServerSetting extern const ServerSettingsString logger_shutdown_level; extern const ServerSettingsString openssl_server_certificate_file; extern const ServerSettingsString openssl_server_private_key_file; + extern const ServerSettingsString openssl_server_ca_config; + extern const ServerSettingsString openssl_client_certificate_file; + extern const ServerSettingsString openssl_client_private_key_file; + extern const ServerSettingsString openssl_client_ca_config; extern const ServerSettingsString distributed_ddl_path; extern const ServerSettingsString distributed_ddl_replicas_path; extern const ServerSettingsInt32 distributed_ddl_pool_size; @@ -2668,26 +2672,25 @@ try tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); } - std::string cert_path = server_settings[ServerSetting::openssl_server_certificate_file]; - std::string key_path = server_settings[ServerSetting::openssl_server_private_key_file]; - + /// TLS certificates, keys and CA certificates are reloaded by CertificateReloader when these files change. std::vector extra_paths = {include_from_path}; - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); + auto watch_path = [&](const std::string & file_path) + { + if (!file_path.empty()) + extra_paths.emplace_back(file_path); + }; + watch_path(server_settings[ServerSetting::openssl_server_certificate_file]); + watch_path(server_settings[ServerSetting::openssl_server_private_key_file]); + watch_path(server_settings[ServerSetting::openssl_server_ca_config]); + watch_path(server_settings[ServerSetting::openssl_client_certificate_file]); + watch_path(server_settings[ServerSetting::openssl_client_private_key_file]); + watch_path(server_settings[ServerSetting::openssl_client_ca_config]); Poco::Util::AbstractConfiguration::Keys protocols; config().keys("protocols", protocols); for (const auto & protocol : protocols) - { - cert_path = config().getString("protocols." + protocol + ".certificateFile", ""); - key_path = config().getString("protocols." + protocol + ".privateKeyFile", ""); - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); - } + for (const auto * key : {"certificateFile", "privateKeyFile", "caConfig"}) + watch_path(config().getString("protocols." + protocol + "." + key, "")); DNSResolver::instance().setFilterSettings(server_settings[ServerSetting::dns_allow_resolve_names_to_ipv4], server_settings[ServerSetting::dns_allow_resolve_names_to_ipv6]); /// DNSCacheUpdater uses BackgroundSchedulePool which lives in shared context diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index dd9531c6493dc..1dc2fd0ab747c 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -222,8 +222,17 @@ struct ConfigReloader::FileWithTimestamp void ConfigReloader::FilesChangesTracker::addIfExists(const std::string & path_to_add) { - if (!path_to_add.empty() && fs::exists(path_to_add)) - files.emplace(path_to_add); + if (path_to_add.empty() || !fs::exists(path_to_add)) + return; + + files.emplace(path_to_add); + + /// E.g. a directory with CA certificates, a change of a file in it should be noticed too. + std::error_code ec; + if (fs::is_directory(path_to_add, ec)) + for (const auto & entry : fs::directory_iterator(path_to_add, ec)) + if (entry.is_regular_file(ec)) + files.emplace(entry.path().string()); } bool ConfigReloader::FilesChangesTracker::isDifferOrNewerThan(const FilesChangesTracker & rhs) diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index c1d19bbc92bd1..b453b94dea7c8 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -148,7 +148,9 @@ auto getSslContextProvider(const Poco::Util::AbstractConfiguration & config, std if (config.has(root_ca_file_property)) params.caLocation = config.getString(root_ca_file_property); - params.loadDefaultCAs = config.getBool(load_default_ca_file_property, false); + /// Unlike `Poco::Net::SSLManager`, the default CA certificates are not trusted unless `loadDefaultCAFile` is set. + constexpr bool load_default_cas_default = false; + params.loadDefaultCAs = config.getBool(load_default_ca_file_property, load_default_cas_default); params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString(verification_mode_property, "none")); const String cipher_list_property = config_prefix + "cipherList"; @@ -194,7 +196,7 @@ auto getSslContextProvider(const Poco::Util::AbstractConfiguration & config, std /// Try to register with CertificateReloader for hot-reload support. /// If registration fails, fall back to static certificate loading. - if (!CertificateReloader::instance().registerAdditionalContext(ssl_ctx, config_prefix)) + if (!CertificateReloader::instance().registerAdditionalContext(ssl_ctx, config_prefix, load_default_cas_default)) { /// For passphrase-protected keys, load certificates manually if (certificate_data) diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 0f94a06ba3bc3..209c0521f77b5 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -1886,7 +1886,7 @@ Configured as `named_collections_storage.type` (`` contains the certificate.)", 0, "openSSL.server.certificateFile") \ - DECLARE(String, openssl_server_ca_config, "", R"(Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://docs.openssl.org/3.0/man3/SSL_CTX_load_verify_locations/).)", 0, "openSSL.server.caConfig") \ + DECLARE(String, openssl_server_ca_config, "", R"(Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://docs.openssl.org/3.0/man3/SSL_CTX_load_verify_locations/). The CA certificates are reloaded without a restart when the file changes or on `SYSTEM RELOAD CONFIG`; new connections are verified against the reloaded certificates.)", 0, "openSSL.server.caConfig") \ DECLARE(String, openssl_server_verification_mode, "relaxed", R"(The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/ClickHouse/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: ``, ``, ``, ``.)", 0, "openSSL.server.verificationMode") \ DECLARE(UInt64, openssl_server_verification_depth, 9, R"(The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.)", 0, "openSSL.server.verificationDepth") \ DECLARE(Bool, openssl_server_load_default_ca_file, true, R"(Determines whether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `` (resp. the directory ``) or in file (resp. directory) specified by the environment variable `` (resp. ``).)", 0, "openSSL.server.loadDefaultCAFile") \ @@ -1906,7 +1906,7 @@ Configured as `named_collections_storage.type` (`` contains the certificate.)", 0, "openSSL.client.certificateFile") \ - DECLARE(String, openssl_client_ca_config, "", R"(Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://docs.openssl.org/3.0/man3/SSL_CTX_load_verify_locations/).)", 0, "openSSL.client.caConfig") \ + DECLARE(String, openssl_client_ca_config, "", R"(Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://docs.openssl.org/3.0/man3/SSL_CTX_load_verify_locations/). The CA certificates are reloaded without a restart when the file changes or on `SYSTEM RELOAD CONFIG`; new connections are verified against the reloaded certificates.)", 0, "openSSL.client.caConfig") \ DECLARE(String, openssl_client_verification_mode, "relaxed", R"(The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/ClickHouse/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: ``, ``, ``, ``.)", 0, "openSSL.client.verificationMode") \ DECLARE(UInt64, openssl_client_verification_depth, 9, R"(The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.)", 0, "openSSL.client.verificationDepth") \ DECLARE(Bool, openssl_client_load_default_ca_file, true, R"(Determines whether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `` (resp. the directory ``) or in file (resp. directory) specified by the environment variable `` (resp. ``).)", 0, "openSSL.client.loadDefaultCAFile") \ diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 787c4e1fba8ab..543508d2c9e43 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -3,6 +3,7 @@ #if USE_SSL #include +#include #include #include #include @@ -23,6 +24,7 @@ CertificateReloader & CertificateReloader::instance() namespace ErrorCodes { extern const int INVALID_CONFIG_PARAMETER; + extern const int OPENSSL_ERROR; } namespace @@ -38,6 +40,91 @@ int callSetCertificate(SSL * ssl, void * arg) return CertificateReloader::instance().setCertificate(ssl, pdata); } +/// Called by OpenSSL instead of `X509_verify_cert` to verify the peer's certificate. +int callVerifyCertificate(X509_STORE_CTX * store_ctx, void * arg) +{ + const MultiVersion * ca_store = reinterpret_cast *>(arg); + return CertificateReloader::instance().verifyCertificate(store_ctx, ca_store); +} + +int defaultVerifyCertificate(X509_STORE_CTX * store_ctx) +{ + /// Same as libssl does when no callback is installed: an error is treated as a verification failure. + int ok = X509_verify_cert(store_ctx); + return ok < 0 ? 0 : ok; +} + +/// Verify the certificate from `original_ctx` like `X509_verify_cert(original_ctx)` would, but against the trusted certificates in `store`. +/// libssl prepares `original_ctx` in `ssl_verify_internal` (ssl/ssl_cert.c) before calling the application callback. +/// The same preparation is carried over to a new `X509_STORE_CTX` that uses `store`, and the outcome is reported back +/// through `original_ctx`, because that is what libssl looks at after the callback returns. +int verifyCertificateWithStore(X509_STORE_CTX * original_ctx, X509 * certificate, X509_STORE * store) +{ + std::unique_ptr verify_ctx(X509_STORE_CTX_new(), X509_STORE_CTX_free); + if (!verify_ctx) + { + X509_STORE_CTX_set_error(original_ctx, X509_V_ERR_OUT_OF_MEM); + return 0; + } + + if (X509_STORE_CTX_init(verify_ctx.get(), store, certificate, X509_STORE_CTX_get0_untrusted(original_ctx)) != 1) + { + X509_STORE_CTX_set_error(original_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } + + /// The connection, for the per-connection verification callbacks (e.g. the ones of Poco and boost::asio) that look it up. + int ssl_idx = SSL_get_ex_data_X509_STORE_CTX_idx(); + SSL * ssl = static_cast(X509_STORE_CTX_get_ex_data(original_ctx, ssl_idx)); + if (ssl) + { + if (X509_STORE_CTX_set_ex_data(verify_ctx.get(), ssl_idx, ssl) != 1) + { + X509_STORE_CTX_set_error(original_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } + /// Has an effect only if DANE is enabled for the connection. + X509_STORE_CTX_set0_dane(verify_ctx.get(), SSL_get0_dane(ssl)); + } + + /// The default purpose depends on which side is verified. Everything libssl derived from the connection and its `SSL_CTX` + /// (verification depth, expected host name, security level, flags, ...) is already in the parameters of `original_ctx`. + X509_STORE_CTX_set_default(verify_ctx.get(), (ssl && SSL_is_server(ssl)) ? "ssl_client" : "ssl_server"); + if (X509_VERIFY_PARAM_set1(X509_STORE_CTX_get0_param(verify_ctx.get()), X509_STORE_CTX_get0_param(original_ctx)) != 1) + { + X509_STORE_CTX_set_error(original_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } + X509_STORE_CTX_set_verify_cb(verify_ctx.get(), X509_STORE_CTX_get_verify_cb(original_ctx)); + + int ok = defaultVerifyCertificate(verify_ctx.get()); + + X509_STORE_CTX_set_error(original_ctx, X509_STORE_CTX_get_error(verify_ctx.get())); + X509_STORE_CTX_set_error_depth(original_ctx, X509_STORE_CTX_get_error_depth(verify_ctx.get())); + if (STACK_OF(X509) * chain = X509_STORE_CTX_get1_chain(verify_ctx.get())) + X509_STORE_CTX_set0_verified_chain(original_ctx, chain); + X509_VERIFY_PARAM_move_peername(X509_STORE_CTX_get0_param(original_ctx), X509_STORE_CTX_get0_param(verify_ctx.get())); + + return ok; +} + +/// Load the trusted CA certificates the same way `Poco::Net::Context` does it for the contexts created at startup, +/// so that a reload results in the same set of trusted certificates as a restart would. +std::unique_ptr loadCAStore(const std::string & ca_path, bool load_default_cas) +{ + Poco::Net::Context::Params params; + params.caLocation = ca_path; + params.loadDefaultCAs = load_default_cas; + /// Only the certificate store is taken from this context, so the usage does not matter. + Poco::Net::Context context(Poco::Net::Context::CLIENT_USE, params); + + X509_STORE * store = SSL_CTX_get_cert_store(context.sslContext()); + if (!store || X509_STORE_up_ref(store) != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Cannot get CA certificates from SSL context: {}", Poco::Net::Utility::getLastError()); + + return std::make_unique(store, context.getCAPaths()); +} + } /// This is callback for OpenSSL. It will be called on every connection to obtain a certificate and private key. @@ -50,6 +137,28 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu return setCertificateCallback(ssl, current.get(), log); } +/// This is callback for OpenSSL. It will be called on every connection that verifies the certificate of the peer. +int CertificateReloader::verifyCertificate(X509_STORE_CTX * store_ctx, const MultiVersion * ca_store) const +{ + try + { + auto current = ca_store->get(); + X509 * certificate = X509_STORE_CTX_get0_cert(store_ctx); + + /// Raw public keys (RFC 7250) are verified without CA certificates. + if (!current || !certificate) + return defaultVerifyCertificate(store_ctx); + + return verifyCertificateWithStore(store_ctx, certificate, current->store); + } + catch (...) + { + LOG_ERROR(log, getCurrentExceptionMessageAndPattern(/* with_stacktrace */ false)); + X509_STORE_CTX_set_error(store_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } +} + int setCertificateCallback(SSL * ssl, const CertificateReloader::Data * current_data, LoggerPtr log) { if (current_data->certs_chain.empty()) @@ -142,6 +251,10 @@ std::list::iterator CertificateReloader::findOrI data.push_back(MultiData(ctx)); --it; data_index[prefix] = it; + + /// Verify peer certificates against the reloadable CA certificates of this prefix. + /// Until (and unless) they are loaded, the callback does exactly what OpenSSL does without it. + SSL_CTX_set_cert_verify_callback(ctx, callVerifyCertificate, reinterpret_cast(&it->ca.store)); } return it; } @@ -179,8 +292,51 @@ void CertificateReloader::tryLoadACMECertificate(SSL_CTX * ctx, const std::strin } } +void CertificateReloader::tryLoadCAImpl(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix) +{ + std::string new_ca_path = config.getString(prefix + Poco::Net::SSLManager::CFG_CA_LOCATION, ""); + + /// Without `caConfig` the trusted certificates come only from the system locations (if at all), there is nothing to reload. + /// But if `caConfig` was there before, keep following the configuration, like a restart would. + if (new_ca_path.empty()) + { + auto index_it = data_index.find(prefix); + if (index_it == data_index.end() || index_it->second->ca_file.path.empty()) + return; + } + + try + { + auto it = findOrInsert(ctx, prefix); + bool ca_file_changed = it->ca_file.changeIfModified(std::move(new_ca_path), log); + + auto load = [&](CAData & ca, bool load_default_cas_default) + { + bool new_load_default_cas = config.getBool(prefix + Poco::Net::SSLManager::CFG_ENABLE_DEFAULT_CA, load_default_cas_default); + if (!ca_file_changed && ca.store.get() && new_load_default_cas == ca.load_default_cas) + return; + + LOG_DEBUG(log, "Reloading CA certificates ({}), load default CAs: {}.", it->ca_file.path, new_load_default_cas); + ca.store.set(loadCAStore(it->ca_file.path, new_load_default_cas)); + ca.load_default_cas = new_load_default_cas; + LOG_INFO(log, "Reloaded CA certificates ({}), load default CAs: {}.", it->ca_file.path, new_load_default_cas); + }; + + load(it->ca, Poco::Net::SSLManager::VAL_ENABLE_DEFAULT_CA); + if (it->ca_with_other_default) + load(*it->ca_with_other_default, it->other_load_default_cas_default); + } + catch (...) + { + LOG_ERROR(log, getCurrentExceptionMessageAndPattern(/* with_stacktrace */ false)); + } +} + void CertificateReloader::tryLoadImpl(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix) { + /// Trusted CA certificates do not depend on how the own certificate is configured. + tryLoadCAImpl(config, ctx, prefix); + /// If at least one of the files is modified - recreate std::string new_cert_path = config.getString(prefix + "certificateFile", ""); std::string new_key_path = config.getString(prefix + "privateKeyFile", ""); @@ -239,7 +395,7 @@ void CertificateReloader::tryReloadAll(const Poco::Util::AbstractConfiguration & } -bool CertificateReloader::registerAdditionalContext(SSL_CTX * ctx, const std::string & prefix) +bool CertificateReloader::registerAdditionalContext(SSL_CTX * ctx, const std::string & prefix, bool load_default_cas_default) { if (!ctx) return false; @@ -256,9 +412,24 @@ bool CertificateReloader::registerAdditionalContext(SSL_CTX * ctx, const std::st MultiData * pdata = &*(it->second); + /// Share the reloadable CA certificates of this prefix (see `findOrInsert`). A context that assumes another default + /// for `loadDefaultCAFile` may trust other CA certificates than the primary one, so it gets its own ones. + /// They are loaded by the next `tryLoad`, until then the context keeps using the store it was created with. + CAData * ca = &pdata->ca; + if (load_default_cas_default != Poco::Net::SSLManager::VAL_ENABLE_DEFAULT_CA) + { + if (!pdata->ca_with_other_default) + { + pdata->ca_with_other_default.emplace(); + pdata->other_load_default_cas_default = load_default_cas_default; + } + ca = &*pdata->ca_with_other_default; + } + SSL_CTX_set_cert_verify_callback(ctx, callVerifyCertificate, reinterpret_cast(&ca->store)); + /// Verify that certificate data was actually loaded, not just the entry created. /// If data is null, return false so caller can use fallback (static cert loading). - /// This can happen if initial cert parsing failed in tryLoadImpl. + /// This can happen if initial cert parsing failed in tryLoadImpl or if only `caConfig` is set for the prefix. if (!pdata->data.get()) { LOG_WARNING(log, "Cannot register additional context for prefix '{}': certificate data not loaded. " @@ -292,6 +463,22 @@ std::optional CertificateReloader::getCertificate(const std::st } +std::optional CertificateReloader::getCAPaths(const std::string & prefix) const +{ + std::lock_guard lock{data_mutex}; + + auto it = data_index.find(prefix); + if (it == data_index.end()) + return {}; + + auto current = it->second->ca.store.get(); + if (!current) + return {}; + + return current->paths; +} + + CertificateReloader::Data::Data(std::string cert_path, std::string key_path, std::string pass_phrase) : certs_chain(X509Certificate::fromFile(cert_path)), key(KeyPair::fromFile(key_path, pass_phrase)) { @@ -305,6 +492,14 @@ CertificateReloader::Data::Data(KeyPair _pkey, X509Certificate::List _certs_chai bool CertificateReloader::File::changeIfModified(std::string new_path, LoggerPtr logger) { + if (new_path.empty()) + { + bool changed = !path.empty(); + path.clear(); + modification_time = {}; + return changed; + } + std::error_code ec; std::filesystem::file_time_type new_modification_time = std::filesystem::last_write_time(new_path, ec); if (ec) @@ -318,10 +513,24 @@ bool CertificateReloader::File::changeIfModified(std::string new_path, LoggerPtr return false; } - if (new_path != path || new_modification_time != modification_time) + /// `caConfig` can be a directory with certificates, replacing one of them is a change too. + UInt64 new_directory_contents_hash = 0; + if (std::filesystem::is_directory(new_path, ec)) + { + SipHash hash; + for (const auto & entry : std::filesystem::directory_iterator(new_path, ec)) + { + hash.update(entry.path().filename().string()); + hash.update(std::filesystem::last_write_time(entry.path(), ec).time_since_epoch().count()); + } + new_directory_contents_hash = hash.get64(); + } + + if (new_path != path || new_modification_time != modification_time || new_directory_contents_hash != directory_contents_hash) { path = new_path; modification_time = new_modification_time; + directory_contents_hash = new_directory_contents_hash; return true; } diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 44330a1f7007a..735b40392d437 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -27,17 +28,34 @@ namespace DB { -/// The CertificateReloader singleton performs 2 functions: -/// 1. Dynamic reloading of TLS key-pair when requested by server: +/// The CertificateReloader singleton performs 3 functions: +/// 1. Dynamic reloading of TLS key-pair and of the trusted CA certificates (`caConfig`) when requested by server: /// Server config reloader notifies CertificateReloader when the config changes. /// On changed config, CertificateReloader reloads certs from disk. /// 2. Implement `SSL_CTX_set_cert_cb` to set certificate for a new connection: /// OpenSSL invokes a callback to setup a connection. +/// 3. Implement `SSL_CTX_set_cert_verify_callback` to verify the peer's certificate of a new connection +/// against the most recently loaded CA certificates. +/// +/// An `SSL_CTX` that is shared between threads must not be modified, so instead of touching the contexts on reload, +/// both callbacks apply the current immutable snapshot (`MultiVersion`) to each new connection. class CertificateReloader { public: using stat_t = struct stat; + /// Owns a reference to a set of trusted CA certificates and remembers where they were loaded from. + struct CAStore + { + CAStore(X509_STORE * store_, Poco::Net::Context::CAPaths paths_) : store(store_), paths(std::move(paths_)) {} + CAStore(const CAStore &) = delete; + CAStore & operator=(const CAStore &) = delete; + ~CAStore() { X509_STORE_free(store); } + + X509_STORE * const store; + const Poco::Net::Context::CAPaths paths; + }; + struct Data { X509Certificate::List certs_chain; @@ -56,10 +74,19 @@ class CertificateReloader std::string path; std::filesystem::file_time_type modification_time; + /// For a directory: the names and modification times of the files in it. + UInt64 directory_contents_hash = 0; bool changeIfModified(std::string new_path, LoggerPtr logger); }; + /// Trusted CA certificates from `caConfig` and, if `load_default_cas`, the default ones. + struct CAData + { + MultiVersion store; + bool load_default_cas = false; + }; + struct MultiData { SSL_CTX * ctx = nullptr; @@ -69,6 +96,14 @@ class CertificateReloader File cert_file{"certificate"}; File key_file{"key"}; + /// Empty if `caConfig` is not set for the prefix, then verification keeps using the store the context was created with. + CAData ca; + /// For additional contexts that assume another `loadDefaultCAFile` than Poco when it is not configured (the ones of Keeper). + std::optional ca_with_other_default; + bool other_load_default_cas_default = false; + + File ca_file{"CA"}; + explicit MultiData(SSL_CTX * ctx_) : ctx(ctx_) {} }; @@ -88,8 +123,11 @@ class CertificateReloader /// Handle configuration reload void tryLoad(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix); - /// Register an additional SSL_CTX to share certificates with the primary context - bool registerAdditionalContext(SSL_CTX * ctx, const std::string & prefix); + /// Register an additional SSL_CTX to share certificates and trusted CAs with the primary context of `prefix`. + /// `load_default_cas_default` is what the caller assumes for `loadDefaultCAFile` when it is not configured. + /// Returns true if the context will get its certificate and key from CertificateReloader, + /// false if the caller has to configure them on the context itself. + bool registerAdditionalContext(SSL_CTX * ctx, const std::string & prefix, bool load_default_cas_default); /// Handle configuration reload for all contexts void tryReloadAll(const Poco::Util::AbstractConfiguration & config); @@ -97,11 +135,18 @@ class CertificateReloader /// A callback for OpenSSL int setCertificate(SSL * ssl, const MultiData * pdata); + /// A callback for OpenSSL: verify the peer certificate in `store_ctx` against the current CA certificates in `ca_store`. + int verifyCertificate(X509_STORE_CTX * store_ctx, const MultiVersion * ca_store) const; + /// The leaf certificate that is currently served for `prefix` connections, if there is one. /// It is not necessarily the certificate of the corresponding `SSL_CTX`: certificates are installed /// per connection, and with `` the context itself never gets a certificate at all. std::optional getCertificate(const std::string & prefix) const; + /// Where the CA certificates that are currently used to verify peers of `prefix` connections were loaded from, + /// if they are managed by CertificateReloader (i.e. `caConfig` is set for the prefix). + std::optional getCAPaths(const std::string & prefix) const; + private: CertificateReloader() = default; @@ -111,6 +156,7 @@ class CertificateReloader /// Unsafe implementation void tryLoadImpl(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix) TSA_REQUIRES(data_mutex); void tryLoadACMECertificate(SSL_CTX * ctx, const std::string & prefix) TSA_REQUIRES(data_mutex); + void tryLoadCAImpl(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix) TSA_REQUIRES(data_mutex); std::list::iterator findOrInsert(SSL_CTX * ctx, const std::string & prefix) TSA_REQUIRES(data_mutex); diff --git a/src/Storages/System/StorageSystemCertificates.cpp b/src/Storages/System/StorageSystemCertificates.cpp index 3b8acceea62ae..2ae44bb8a5f35 100644 --- a/src/Storages/System/StorageSystemCertificates.cpp +++ b/src/Storages/System/StorageSystemCertificates.cpp @@ -15,6 +15,7 @@ #include #include #include + #include #endif #include @@ -140,12 +141,19 @@ void StorageSystemCertificates::fillData([[maybe_unused]] MutableColumns & res_c } }; + /// The CA certificates may have been reloaded since the context was created, `CertificateReloader` knows the current ones. + auto current_ca_paths = [](const std::string & prefix, Poco::Net::Context::Ptr ssl_context) + { + if (auto reloaded_ca_paths = CertificateReloader::instance().getCAPaths(prefix)) + return *reloaded_ca_paths; + return ssl_context->getCAPaths(); + }; + const auto & config = Context::getGlobalContextInstance()->getConfigRef(); try { - const auto & ca_paths = Poco::Net::SSLManager::instance().defaultServerContext()->getCAPaths(); - process_ca_paths(ca_paths, ""); + process_ca_paths(current_ca_paths(Poco::Net::SSLManager::CFG_SERVER_PREFIX, Poco::Net::SSLManager::instance().defaultServerContext()), ""); } catch (const Poco::Net::SSLException &) { @@ -162,10 +170,7 @@ void StorageSystemCertificates::fillData([[maybe_unused]] MutableColumns & res_c continue; if (auto ctx = Poco::Net::SSLManager::instance().getCustomServerContext(prefix)) - { - const auto & ca_paths = ctx->getCAPaths(); - process_ca_paths(ca_paths, protocol_name); - } + process_ca_paths(current_ca_paths(prefix, ctx), protocol_name); } #endif } diff --git a/tests/integration/test_reload_ca_certificate/__init__.py b/tests/integration/test_reload_ca_certificate/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_reload_ca_certificate/certs/ca.crt b/tests/integration/test_reload_ca_certificate/certs/ca.crt new file mode 100644 index 0000000000000..b836c066c43e6 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIURxZvEwx6gcq/IbNHsPts6W5c7P0wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDEwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCQMESmjuEmo6A25FY5Bt11iOrs +ySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5Uei/SsY0kLTiz +PZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJtwVf9E13eKGw7 +qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAgLoLUeVC/HpNQ +dVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4bJV53nOJ+1Jw +IEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTYdzicaA9JAgMB +AAGjUzBRMB0GA1UdDgQWBBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAfBgNVHSMEGDAW +gBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQBiopVu+5LqgH4FwKh6YLvWEWdtw5gWs1Eda27cN2b1nTnJ+Xfn +UlwAdaS1lzGBkgUDYfbL0OJo9wx/yg5MuakE2kNO1QALb4r5mpGqTBMYT1XVzcGx +6LCOweHBI4a/ff4s1rj1xRR+PIPIHnCd5mKEgscKMvCmR5kUzkkmjRPFnBJFLHH9 +Z88L4J5NBwF8wJsFCv9td8aeeeTc9rg19zKOn6vkR5AczeXVAeBDBxAtcmX2YRMt +ytIWGmKfW3OE8kxsZdJJn4b+9jE5pLbTEL/So0YuNELRgcpK+9MGApOKjcc3DJg0 +LhsYev318QV4SjluwR1T/jgRvAlFbwk8+10L +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca1.crt b/tests/integration/test_reload_ca_certificate/certs/ca1.crt new file mode 100644 index 0000000000000..b836c066c43e6 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca1.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIURxZvEwx6gcq/IbNHsPts6W5c7P0wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDEwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCQMESmjuEmo6A25FY5Bt11iOrs +ySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5Uei/SsY0kLTiz +PZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJtwVf9E13eKGw7 +qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAgLoLUeVC/HpNQ +dVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4bJV53nOJ+1Jw +IEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTYdzicaA9JAgMB +AAGjUzBRMB0GA1UdDgQWBBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAfBgNVHSMEGDAW +gBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQBiopVu+5LqgH4FwKh6YLvWEWdtw5gWs1Eda27cN2b1nTnJ+Xfn +UlwAdaS1lzGBkgUDYfbL0OJo9wx/yg5MuakE2kNO1QALb4r5mpGqTBMYT1XVzcGx +6LCOweHBI4a/ff4s1rj1xRR+PIPIHnCd5mKEgscKMvCmR5kUzkkmjRPFnBJFLHH9 +Z88L4J5NBwF8wJsFCv9td8aeeeTc9rg19zKOn6vkR5AczeXVAeBDBxAtcmX2YRMt +ytIWGmKfW3OE8kxsZdJJn4b+9jE5pLbTEL/So0YuNELRgcpK+9MGApOKjcc3DJg0 +LhsYev318QV4SjluwR1T/jgRvAlFbwk8+10L +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca1.key b/tests/integration/test_reload_ca_certificate/certs/ca1.key new file mode 100644 index 0000000000000..562b4debf1ac9 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca1.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCQMESmjuEmo6A2 +5FY5Bt11iOrsySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5U +ei/SsY0kLTizPZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJt +wVf9E13eKGw7qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAg +LoLUeVC/HpNQdVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4 +bJV53nOJ+1JwIEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTY +dzicaA9JAgMBAAECggEABbduL1gkknXkBcJw2IRec8OaBHQs2cdfwwbGM0b1xEL/ +2Dp9+j7ngcJTO9or1XVElCKoMNxmgaAzK9RSJfqtV3j2AaWKyOO6wk3YKwTKXy2+ +WvS6k7RgTwUnjm3TykwTa1xgZ2wQQ8EmC3CCCtScMPu+CnGgw1Y9GVXmqaQf5jQ1 +xAoxP8/POBLjco8PAFWtai+wK9X1OVxQ6gr5Tw3TwAIHrYGsTY19g5c5IM4Y2k34 +QoaSGHkmSFOrBZs2DaWCaJ08Iz3zWUBP0VV2/H0szdIIrkv/Tqw2K6peRbJB8mYN +TID/BpbAadbEllzfn31O3Si0urHNZMrJI5IS2IChEQKBgQDHXrc6AUQQArbEKCxB +dq6B9+tLukaiMctxLHNXhY2NYlJxtASxkmYJewmU/d16lMyo3S+IPkEueo0Q1u3Z +k/MotSiis5kkL2EJWJjHq4H/ktw6k2a0vRd4DtcoysJ2qkCcgubSGYqSE+YcI1KC +MyOoKfb2IffCucyYAzntKmh+2QKBgQC5JQH9+CdyPlcWTJsXMztgDwyhzBixSqIl +Ovfrzzi7QSIX9csccZoXWkOfDEUJp2CZImErASO06+RM2ETQ1n2art4eWxX3UubD +pLXfr1m8NL/LJb56k9sWT+YwLXm4JeHeaErK6xTQCDNxH6szXUXQdXUpnbrrjg0S +6xVG0tCt8QKBgBI+U4vmQ8EnTmwitPIElzFja0+Rqxb6cYBYrfFLUkmmvp6S9378 +Q4QIkzbkCBlIdnXZT5krATHsmu34jOlFBZIrCZ3hy1ipUTrWtZxH0Gx/ltFxXYua +ZgRhb0TXUPYk3Ca2P8Ln/WsikQLwJIOvhErGFEgvkYlrERKz8OAH6mn5AoGAXQxd +YO9bm8365KkhdNp5p8BIf/RcIJY6wW1OdkPR5kJIyTPtnWD2qW/i9kcrVzu4j524 +qe1Lrby0I265vx9dRuVFmon6ky8l7QOVqFKvTahRD97rSR2QCTmknWfteYAIcUeG +906ISjkk6WCaIRlqYeb2ODEeZQ4iQfTF369J03ECgYA3NPk8j5ma59w/TYUPgZWN +aFlEKWLO8xOal9kWhx5dpXi1YCv3YoQztrfYus5b2nj7iDshK5cK6TzZaxPahqxA +rW9ExDjN2q89AvqP9vKPDJZ8Rk7+vTlLSNkFSe9LvvtfnU6IHdNcj+t/CI2823GC +Myj/NocqikSkeRCcq4SGCA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca2.crt b/tests/integration/test_reload_ca_certificate/certs/ca2.crt new file mode 100644 index 0000000000000..eff5f7fd97bc9 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca2.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIUdDLEtjezpK/A2pIn/0YjbO3JhyAwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMjAeFw0yNjA4MzExNzI4MTVaFw0zNjA4MjgxNzI4MTVaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1bXrQUqqds9kXelFc7J+IrXsE +UdFZtxktFglh9eFrffqr0/+vthrcnqZ/OZrLmZV5PSW+gSNuaAVArbgGwYWW1lqL +0eRfQOktX/NcjoYlWUYlK7lWReFQj2CTCUrP3ojrdo/Oe0/0LsV3kctfUagcNxMX +oV5NAwHd9dDf9aL1ulljc+RO4ym8BlRqSApShkyH3xklkDLBneHOUJncwrWcC12p +1kV3fvt9iBURGK/MYAhhBJGnBeHM+OHLcyVZaUhqXcZ4+fFA2dEA5yGvPs2Gaf/1 +aAE/FsU/zLZkHmI8gslCoBrLtc10dgiUne07Hphu14wplVrSCWJImFMNN31fAgMB +AAGjUzBRMB0GA1UdDgQWBBRQyTcy9OASBuFcqS92wJ9LmtdwejAfBgNVHSMEGDAW +gBRQyTcy9OASBuFcqS92wJ9LmtdwejAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQCnd9ONRHb484+BSpRjfWpuDwGgjdqW9aglEFHOVsUestWe8rSl +9lZBveng/4zIqpIKZpOK04shPwP28lKwPim7EFp9iMSd7lnvSy0vHUhYoVzDtK/V ++QoYcrS/LgijOd+ohaszn2hzWWenNdWkgzSfjzp0WgiGg6j2R5pM9fmX/Wh0KtaC +XF23cM8eLaA5CyVCGRmH637qircr9wLbulld14GNBa9hTqcsp12TE+UX5ULwrGdO +PfiFeC8dmtiNW1p1dhdas52kB92e8EWU1wjFMZACyWpY6x7fxmdLexys5TqKkSn+ +JODwownsXCO4RGHTgQUuZNKFcyfbQGUbF3Ig +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca2.key b/tests/integration/test_reload_ca_certificate/certs/ca2.key new file mode 100644 index 0000000000000..14928a74fcad6 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca2.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC1bXrQUqqds9kX +elFc7J+IrXsEUdFZtxktFglh9eFrffqr0/+vthrcnqZ/OZrLmZV5PSW+gSNuaAVA +rbgGwYWW1lqL0eRfQOktX/NcjoYlWUYlK7lWReFQj2CTCUrP3ojrdo/Oe0/0LsV3 +kctfUagcNxMXoV5NAwHd9dDf9aL1ulljc+RO4ym8BlRqSApShkyH3xklkDLBneHO +UJncwrWcC12p1kV3fvt9iBURGK/MYAhhBJGnBeHM+OHLcyVZaUhqXcZ4+fFA2dEA +5yGvPs2Gaf/1aAE/FsU/zLZkHmI8gslCoBrLtc10dgiUne07Hphu14wplVrSCWJI +mFMNN31fAgMBAAECggEADGEHnovqwk88LSLrxvMKQPF9UaJ3g1nqk4NLvphf+iKi +u53/yzBvYsWccf/Z37HUsMIK/5WgzO63X7NlZBNI7ILRs3VDJ3tZWFQ8rwBhMiPb +lREHrgBJ9beNYlr29M9So6ZM/Pdi/DPuG1DNAe2jZ9Fw3GLmo/XkPGMtDlWcpIwm +DpJDpFDCPfpi0g0z5gIyqVQQW3m9dE8WFvBPUYoDALmKfj8xvPuw1ezEsSkUj06N +YD+/PTOvpQyDhJGVVS9fZyQvdfCszTPMYsNx7e3FVsDdrXRwCFr4gbzxkjQhcwoq +VeZc+1k9R6V17uY3lPL5hfUpLBz8CxfitBzHJjH7QQKBgQDhiAXlNGn0+ggKg3r0 +H2wJ9E6ecyhUVOXoS9f65b+V6sqPSEHWk0jKPJqMXZV6WY4lQHq90WU3hxx2sNiX +jIP6AZdTUVfqJ7gAvBIn14kkLrirmPYx3MURFYoPm+B4LIHcvMlnMtY7BzO3eiv9 +1GDZYDl0mNcx/m5ipYxYLIH34QKBgQDN8CKzju8R1Ogp7m7bbjxSv3PCEzV0jHrs +jQBTSjb60d+Lz/STI6N0H7BMkwvNewnMwQU1xZ8DgGsDXQa4/6x2UiDGHvsCDFTF +HHsXzQCHP8n10vaU4xeCFqyj9/KYElqS8WOn/faH+c1N246+ATTMRDiU3lhI64Bo +iXKmuQcdPwKBgHKYPwabf0su0G8nJ45reOYF8Pyp3tAa40cJYpDltFdkmc/8ExgI +dm/sI0s3MgCdCJD9FmDkyN1SFbBpY2R9zYF21YFMT7N2wxP8e+0qo1BzPPpUGqRz +XN61ZxVPSttFIica9est9ZTAsBKGTVwIUb2iGw+XqaCJe2U8YPdchh2BAoGBAJ+L +fAbqJHMHJDpgG4hqlddxtafUo+RAdXdQIcFlTMTy1aKGoK9hu99aMYaRoWI3ATed +DoFDMldPJRj8+BlZEu6z3+o91C8ZCI+Q6hhdXRxrIfcN0rU0XmENWgDKNir0hTE0 +TAW5LkbYE+NOxv6TBql97OwAehs8QEY8vhNGY6mXAoGASwKe9byhQKXk0X6Q31g+ +5qaDc+fAqirKnPkqexw+1pUv/j6pK0QFBmPAT46Iwt/znAXqXCfSCNwIpR2UW78n +2J9UQR076SUat71MvcbPhesoUhLw4ZyFQYZ1Lo3piokhzjI0zwlmMusDpUdOjdB6 +JJcZmm30hrRMYNncfobFvXc= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert1.crt b/tests/integration/test_reload_ca_certificate/certs/cert1.crt new file mode 100644 index 0000000000000..3c2faa29525a0 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert1.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUB2rETXTp2XN0iWipA9fQo9alsGQwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAN7xjZT2K9ocTmXgCKTgRU70rTvDrW9lpupOAASUm+pwWjtwtYs6AyI1eOlM +5ecZlMHC5zhgngxHlQ1MKqMmvFeMJmPZlpG4xPiwXQXxoFvbQ9T6nRvcnxhB2R/p +wSxBvVwwX2Tf7h+f03XwVndC3CkPwdmwR3VsnVld7WPsLaTemUDKefZ6yTR0OVYZ +fN86bw5Ao9ZzjyH079fAvL2BAQ27I64WJz2480dJX9xgp9wzo91oYyy/ox6cA+ls +lPxUIvV/Se9mV0NhtkkdEgC0DidXyKZM79PsDTxnQUYiqF8p6YGv+bDbvBgyaoTJ +uJQ5MyEDKEl6PcTWq06mHWGF7iMCAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCLJtr15jan+ipoJ4MHQ8hHuj5SmMB8G +A1UdIwQYMBaAFImx3vMM84mQETGNTEnxspVKDakdMA0GCSqGSIb3DQEBCwUAA4IB +AQAhoPNnfpwuqjZClvtMNGZz4He2g09XO7RgE6uFBPG4DKZyJoRwNkrUrdqpkG13 +2D1lV8IUUzgRBqzP4JmVjmm6Vb7106V6epaZXQPar9PaM9pS/whf6G+5lXTVbn0G +skjMAuDeqZLIPh+547hvdI3WguJ+Lt+iqtlpVpqjPqySgG21H7CFMEZ7pKz7bKgc +Q7qzlvVNJc/XDE8nkuxfdBDZODQD3HEQXOVVGx/1x82HXy/EsTS6nBZnO9gOyj5H +gd/+uN2S5oIuxs1a1b1b54MuWZaFz9OGaqGr3CAjSZO1imIfgT4Ftfoa0FAFrSdO +pXpfpZLT5iOYwG+WxcYu1fW5 +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert1.key b/tests/integration/test_reload_ca_certificate/certs/cert1.key new file mode 100644 index 0000000000000..61143a040b98e --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert1.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe8Y2U9ivaHE5l +4Aik4EVO9K07w61vZabqTgAElJvqcFo7cLWLOgMiNXjpTOXnGZTBwuc4YJ4MR5UN +TCqjJrxXjCZj2ZaRuMT4sF0F8aBb20PU+p0b3J8YQdkf6cEsQb1cMF9k3+4fn9N1 +8FZ3QtwpD8HZsEd1bJ1ZXe1j7C2k3plAynn2esk0dDlWGXzfOm8OQKPWc48h9O/X +wLy9gQENuyOuFic9uPNHSV/cYKfcM6PdaGMsv6MenAPpbJT8VCL1f0nvZldDYbZJ +HRIAtA4nV8imTO/T7A08Z0FGIqhfKemBr/mw27wYMmqEybiUOTMhAyhJej3E1qtO +ph1hhe4jAgMBAAECggEAOTrp3OTqseVVTLqbjXOS5ydRNwfOxEtkcz5Nq99YPPDV +gO+4csKUHlp6rO0UEWSUNr8pKuRGfiF8BjtYsKQXciPkkPpAuCylx69CWe3CfAIH +4irpXMcgQhJZQeN4Nruzd/Bk9Ji1YIHfPyXQlHHh4VqNqSui1GZq6A+ACogM2Ybe +Qu2tEKGYC7LIYATfcnWMF8QpdF+FDCQmjL8yZTz6WhCDTk+h58y/lDfFwzhCKEwx +Y3g69S16lgIRlzxqoXU9UR31ieGsG+op6hmgrdO57chTUF/+g3qS7fGoFN6g6gsm +GeanWfM2nHpM4A73+TF7aI8xf4IHIgkXYa/KEXXh4QKBgQDz+Kj7OEQvv2NIs+lY +RkYTl0ljof8IzFPp8qZEo9z0PpD/Nl9sphfoV/ZYj79Ijnb8bkPcK2aqn2f9Ortx +I28XtzZog/p4XVSLw0o6hRRENsZEpsxYebaGp6kighZM9JHPm67T4neQ5PcR0r4g +Kz6DpoN/bM+uiWiULIMBKeHlwwKBgQDp73x77kvFXGb/Xc67qjfT7H8gPdBcAajd +SGeQHP+AbvsxZaEp0hMSXHYoJRq4eSsA5h8BOHDjM39LZ4uvlcphQDpuCHC/cbUI +bt5PmuddY/YTDkvEi/WIZi9+HSuAZ+RnVgbZowuTV0/t776Fj8R/MefDuWSuiDvF +Bt2z1XZwIQKBgBGZxtcY4BJxxD/ietsbdsLDD1BYx4Vi+ErQbp5VFAOq39sJmSjF +csQYVHVfKXWakYr0iYDAwM9eYKosKomm/MTBOvOfUdqNISRUGm7OWv/w06zwO53G +ahycy97pc6JpontPx/URSX7yhcCLa5v2grQMtz/iIbl9wEWwUGMtGlbxAoGANQD3 +IplWf6w1Bg06JxklNxYxo5t91yrlGOYr2OJJHc+HiKSvRGt9uL5MY0Is8Lk7fiOl +yMACC+iCIhKe+rSkuy4zTvUInsfjrbp5Em5Vl7pradvmXO0dP79vaVKwpZJklOlP ++gXQPJ0e1hlpAJgXfH5RNe6OmmDxse2hU/q8sCECgYBmK1MbPoX0SbuzPmAT6iu8 +4spHZp/AsiWOAnXU42VgEJCRfH+086CvZdf8N7Q6cJDfxofO95syDIMkC/7bkTGT +WT+R+1s0ouZFBtcX+6uPf9eR0/5W3SCzV1YkQtw6iXhoFociRdXDGO8JON80+x8e +BWtbo3oEMlwc6/rNAELLbQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert2.crt b/tests/integration/test_reload_ca_certificate/certs/cert2.crt new file mode 100644 index 0000000000000..4fb702bf143b2 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert2.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUIcogGF2yo/W8Z+f1vXaSVWnpDO4wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMjAeFw0yNjA4MzExNzI4MTVaFw0zNjA4MjgxNzI4MTVaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAJKHwEUY5CUuRs7G9Zv+slTng5op18vSYWpj9m1/VagGhkRLlp6FGqf0+pnb +m1A0T0sXKVeuKFgJWqkTTJ5BuP2IOnS+HsfJ2oc0g2vRpO8v0mtZBdT8KA/W6fur +bxQeUOAoEIGca1r0ha/nRAdwvptFRIiIbDgIe9NXR7DQVBWe7Q+DxpzDhjR3/EvZ +W1/jjvTyg/WRhnM3WhrxfeBrHm6o1d8SsDR1t2FTe+7tiBh4ItI1ZWQUSpPhs9qf +LGe5jDx0+9fy4bj8isyMfGvnT+Ll5QFQn34DZrhQ+CAnU6AukZw2Bc3tIfLqtx28 +dh1VUDl0j/BRdbXerEzla1AVCb8CAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCjHe0KJ1rhLUhZfsMAtv5KSBYC0MB8G +A1UdIwQYMBaAFFDJNzL04BIG4VypL3bAn0ua13B6MA0GCSqGSIb3DQEBCwUAA4IB +AQAbU63n0EM1JWgCDaHYTo9FvnCmnmc1Ge2Nnz5lCwwom5kzB7GPjHljsx3L93OG +ETPbFD1On4OZI9ynURh+9mGlTAE3RiCHT+UjnHnQlKpn0mEjNihXGGM9iMIe4a/q +iVM2To3ayfjuMviLqgpRk48941rEbSSlnaZu+9bXG87QfRdRww2brRF1YOh3IvdT +lawdGc4Rlw/Hh9XjEBJ4u9dtRneoScKI//dzkTKuY0Jj3LeWRq8quX4+DnJZaBlx +vhU9YHstiNIooT1WGlHfNErOB7rsjWbAexWAdIo4gJVTM8caQtpaYPbOBxvCMaMX +/GD8THgD62RZxT80PWXdr4RW +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert2.key b/tests/integration/test_reload_ca_certificate/certs/cert2.key new file mode 100644 index 0000000000000..beeebe86de1fa --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert2.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCSh8BFGOQlLkbO +xvWb/rJU54OaKdfL0mFqY/Ztf1WoBoZES5aehRqn9PqZ25tQNE9LFylXrihYCVqp +E0yeQbj9iDp0vh7HydqHNINr0aTvL9JrWQXU/CgP1un7q28UHlDgKBCBnGta9IWv +50QHcL6bRUSIiGw4CHvTV0ew0FQVnu0Pg8acw4Y0d/xL2Vtf44708oP1kYZzN1oa +8X3gax5uqNXfErA0dbdhU3vu7YgYeCLSNWVkFEqT4bPanyxnuYw8dPvX8uG4/IrM +jHxr50/i5eUBUJ9+A2a4UPggJ1OgLpGcNgXN7SHy6rcdvHYdVVA5dI/wUXW13qxM +5WtQFQm/AgMBAAECggEAF9DYR21vbhBH8kV0gl69Zb8H7O9zw2KAcoDTJY072FFQ +fbAwHQ1dku071UmpbuSZpgbv3Brn3sCNIOGMo0TjstLIoX83KdodL7rcHhLsdPMi +XE/opUCN/bO8wwAO+pz0yyy9MLvCLSiExAguzpYcvMg02+DEf2pNlJY2hj6jXqyq +PKQXqTdtclrTfsjLBDJfqysZA5TrEV8tglU4uYr7fD2q3iFJ90VsBOKZaNs5jEsC +TJfXYWOf5OngwUD5nVCWHfsltJyysbhHUPQJcsk4G2zzQdX9ASS6wMc67rBuz25B +EMzR0ZP2eGAYqhbmYa6oVr0kE1GnukxLkQqtHOeAuQKBgQDEKEu4CRri/fw+RXOB +S33jdEPFP5hzPhx/41bKQzvZ/VkVRH7SLxMGIQjgkd1j7Klx7o42iUJPUYtlN8Gc +0+HxmnUjAh81F2fAq+C3rqRo2D0KQT3izHPGKxfmyPhPXVKrL8FVXGWMv1wvSHpJ +HQ3M4KaD9GMKmnswwyPArM0OBwKBgQC/O5/bqJf+ltCVeE4q77VxsHNcrJfxwgn6 +1I6ZcoAUPPyywOAG4WLaiH41I3CPydFYLfDk6AnbO9JQybvnu69OPa+At++Dggm6 +MSAYvYigQ4RsZuc+kzaV9//zhWIVa5kcfljyo1L49cLf2uBz3JWziEqYsl4TmCtp +bs8oWME4iQKBgAJU5EmEujAWisgGtU/FIPLyL9gJYHuGMnqGrkJrOCvoKgXpsYQ4 +EQbSn7NjqHkGmCEFj+UwDny44GpMll2R2y6vAlNvNAXCiHYu1NX6GnQwldEoY17t +xTaGzprsqp7u4gus3qRwG7jnkWXye5mg4cgcp34MCp1Wpr42o5cntqxDAoGAKP89 +XDgercPjX8f06huNyJvNf5a41GmG/jFHiPoVH0Gb4y6aWJ9FNBiDBh1c6laX/NGM +jWZ5hniitBMrp5iDEsECuRO103mzYClb+jHX8pPG9f5xoOaqkyghxTFZP8JbhtJH +e20sQpddeeRQrkYiCeU0KNxEcuryk53f54RvmBECgYAO8Yo/NA6Cz7FIidmb4tL5 +n3frL1fdA7bY74ltjMJp4dCmvlezbKG2xdSKqDlEvfoWgRrZEBtjH/i5esZhzH/H +6z0rK/TmoLpW5HLmBJ5Fqmj5lWOvmtNVh63QcW/hTFOGRSzuESzTD1qafMjtAAvZ +HBvkHzbEbZ6tVlKpNnNZzg== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh b/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh new file mode 100755 index 0000000000000..c487b3941426a --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Generates the certificates for test_reload_ca_certificate: +# ca1.crt / ca1.key - first root CA +# ca2.crt / ca2.key - second, unrelated root CA +# cert1.crt / cert1.key - leaf certificate issued by ca1 +# cert2.crt / cert2.key - leaf certificate issued by ca2 +# Both leaf certificates are valid for all host names used in the test and carry no extended key usage, +# so they can be used as server, client and Keeper (Raft) certificates. +# ca.crt, node.crt, node.key - copies of ca1.crt, cert1.crt, cert1.key: the initial content of the files +# that the configs point to and that the test overwrites to rotate certificates. +set -e +cd "$(dirname "${BASH_SOURCE[0]}")" + +DAYS=3650 + +cat > leaf.cnf << 'EOC' +[req] +distinguished_name = dn +prompt = no +[dn] +CN = clickhouse-test +[v3_leaf] +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = DNS:localhost, DNS:node, DNS:node1, DNS:node2, DNS:node3, IP:127.0.0.1 +EOC + +for i in 1 2; do + openssl req -x509 -newkey rsa:2048 -nodes -batch -sha256 -days $DAYS \ + -subj "/O=ClickHouse Test/CN=Test Root CA $i" -keyout ca$i.key -out ca$i.crt + + openssl req -newkey rsa:2048 -nodes -batch -config leaf.cnf -keyout cert$i.key -out cert$i.csr + openssl x509 -req -in cert$i.csr -CA ca$i.crt -CAkey ca$i.key -CAcreateserial -sha256 -days $DAYS \ + -extfile leaf.cnf -extensions v3_leaf -out cert$i.crt + rm -f cert$i.csr ca$i.srl +done +rm -f leaf.cnf + +cp ca1.crt ca.crt +cp cert1.crt node.crt +cp cert1.key node.key diff --git a/tests/integration/test_reload_ca_certificate/certs/node.crt b/tests/integration/test_reload_ca_certificate/certs/node.crt new file mode 100644 index 0000000000000..3c2faa29525a0 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/node.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUB2rETXTp2XN0iWipA9fQo9alsGQwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAN7xjZT2K9ocTmXgCKTgRU70rTvDrW9lpupOAASUm+pwWjtwtYs6AyI1eOlM +5ecZlMHC5zhgngxHlQ1MKqMmvFeMJmPZlpG4xPiwXQXxoFvbQ9T6nRvcnxhB2R/p +wSxBvVwwX2Tf7h+f03XwVndC3CkPwdmwR3VsnVld7WPsLaTemUDKefZ6yTR0OVYZ +fN86bw5Ao9ZzjyH079fAvL2BAQ27I64WJz2480dJX9xgp9wzo91oYyy/ox6cA+ls +lPxUIvV/Se9mV0NhtkkdEgC0DidXyKZM79PsDTxnQUYiqF8p6YGv+bDbvBgyaoTJ +uJQ5MyEDKEl6PcTWq06mHWGF7iMCAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCLJtr15jan+ipoJ4MHQ8hHuj5SmMB8G +A1UdIwQYMBaAFImx3vMM84mQETGNTEnxspVKDakdMA0GCSqGSIb3DQEBCwUAA4IB +AQAhoPNnfpwuqjZClvtMNGZz4He2g09XO7RgE6uFBPG4DKZyJoRwNkrUrdqpkG13 +2D1lV8IUUzgRBqzP4JmVjmm6Vb7106V6epaZXQPar9PaM9pS/whf6G+5lXTVbn0G +skjMAuDeqZLIPh+547hvdI3WguJ+Lt+iqtlpVpqjPqySgG21H7CFMEZ7pKz7bKgc +Q7qzlvVNJc/XDE8nkuxfdBDZODQD3HEQXOVVGx/1x82HXy/EsTS6nBZnO9gOyj5H +gd/+uN2S5oIuxs1a1b1b54MuWZaFz9OGaqGr3CAjSZO1imIfgT4Ftfoa0FAFrSdO +pXpfpZLT5iOYwG+WxcYu1fW5 +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/node.key b/tests/integration/test_reload_ca_certificate/certs/node.key new file mode 100644 index 0000000000000..61143a040b98e --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/node.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe8Y2U9ivaHE5l +4Aik4EVO9K07w61vZabqTgAElJvqcFo7cLWLOgMiNXjpTOXnGZTBwuc4YJ4MR5UN +TCqjJrxXjCZj2ZaRuMT4sF0F8aBb20PU+p0b3J8YQdkf6cEsQb1cMF9k3+4fn9N1 +8FZ3QtwpD8HZsEd1bJ1ZXe1j7C2k3plAynn2esk0dDlWGXzfOm8OQKPWc48h9O/X +wLy9gQENuyOuFic9uPNHSV/cYKfcM6PdaGMsv6MenAPpbJT8VCL1f0nvZldDYbZJ +HRIAtA4nV8imTO/T7A08Z0FGIqhfKemBr/mw27wYMmqEybiUOTMhAyhJej3E1qtO +ph1hhe4jAgMBAAECggEAOTrp3OTqseVVTLqbjXOS5ydRNwfOxEtkcz5Nq99YPPDV +gO+4csKUHlp6rO0UEWSUNr8pKuRGfiF8BjtYsKQXciPkkPpAuCylx69CWe3CfAIH +4irpXMcgQhJZQeN4Nruzd/Bk9Ji1YIHfPyXQlHHh4VqNqSui1GZq6A+ACogM2Ybe +Qu2tEKGYC7LIYATfcnWMF8QpdF+FDCQmjL8yZTz6WhCDTk+h58y/lDfFwzhCKEwx +Y3g69S16lgIRlzxqoXU9UR31ieGsG+op6hmgrdO57chTUF/+g3qS7fGoFN6g6gsm +GeanWfM2nHpM4A73+TF7aI8xf4IHIgkXYa/KEXXh4QKBgQDz+Kj7OEQvv2NIs+lY +RkYTl0ljof8IzFPp8qZEo9z0PpD/Nl9sphfoV/ZYj79Ijnb8bkPcK2aqn2f9Ortx +I28XtzZog/p4XVSLw0o6hRRENsZEpsxYebaGp6kighZM9JHPm67T4neQ5PcR0r4g +Kz6DpoN/bM+uiWiULIMBKeHlwwKBgQDp73x77kvFXGb/Xc67qjfT7H8gPdBcAajd +SGeQHP+AbvsxZaEp0hMSXHYoJRq4eSsA5h8BOHDjM39LZ4uvlcphQDpuCHC/cbUI +bt5PmuddY/YTDkvEi/WIZi9+HSuAZ+RnVgbZowuTV0/t776Fj8R/MefDuWSuiDvF +Bt2z1XZwIQKBgBGZxtcY4BJxxD/ietsbdsLDD1BYx4Vi+ErQbp5VFAOq39sJmSjF +csQYVHVfKXWakYr0iYDAwM9eYKosKomm/MTBOvOfUdqNISRUGm7OWv/w06zwO53G +ahycy97pc6JpontPx/URSX7yhcCLa5v2grQMtz/iIbl9wEWwUGMtGlbxAoGANQD3 +IplWf6w1Bg06JxklNxYxo5t91yrlGOYr2OJJHc+HiKSvRGt9uL5MY0Is8Lk7fiOl +yMACC+iCIhKe+rSkuy4zTvUInsfjrbp5Em5Vl7pradvmXO0dP79vaVKwpZJklOlP ++gXQPJ0e1hlpAJgXfH5RNe6OmmDxse2hU/q8sCECgYBmK1MbPoX0SbuzPmAT6iu8 +4spHZp/AsiWOAnXU42VgEJCRfH+086CvZdf8N7Q6cJDfxofO95syDIMkC/7bkTGT +WT+R+1s0ouZFBtcX+6uPf9eR0/5W3SCzV1YkQtw6iXhoFociRdXDGO8JON80+x8e +BWtbo3oEMlwc6/rNAELLbQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper1.xml b/tests/integration/test_reload_ca_certificate/configs/keeper1.xml new file mode 100644 index 0000000000000..427a4b8f6872c --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper1.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper2.xml b/tests/integration/test_reload_ca_certificate/configs/keeper2.xml new file mode 100644 index 0000000000000..5dac68a6af56d --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper2.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper3.xml b/tests/integration/test_reload_ca_certificate/configs/keeper3.xml new file mode 100644 index 0000000000000..28a155e926314 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper3.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/ssl.xml b/tests/integration/test_reload_ca_certificate/configs/ssl.xml new file mode 100644 index 0000000000000..3b93b0a7552ab --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/ssl.xml @@ -0,0 +1,31 @@ + + 8443 + + + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + false + + relaxed + false + sslv2,sslv3 + true + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + false + relaxed + false + sslv2,sslv3 + true + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml b/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml new file mode 100644 index 0000000000000..6a1077ce40b97 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml @@ -0,0 +1,29 @@ + + 8443 + + + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + + relaxed + false + sslv2,sslv3 + true + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + relaxed + false + sslv2,sslv3 + true + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_reload_ca_certificate/test.py b/tests/integration/test_reload_ca_certificate/test.py new file mode 100644 index 0000000000000..473a2e850e2cb --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/test.py @@ -0,0 +1,275 @@ +""" +Hot reload of the trusted CA certificates (`openSSL.*.caConfig`). + +The certificates in certs/ are produced by certs/generate_certs.sh: +two unrelated root CAs (ca1, ca2) and one leaf certificate issued by each of them (cert1, cert2). +Every instance starts with ca.crt = ca1 as the trusted CA and node.crt/node.key = cert1 as its own certificate, +and the tests overwrite these files in the container to rotate them without restarting anything. +""" + +import time +import uuid + +import pytest + +import helpers.keeper_utils as ku +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +CONFIG_DIR = "/etc/clickhouse-server/config.d" +CERT_FILES = [ + "certs/ca.crt", + "certs/ca1.crt", + "certs/ca2.crt", + "certs/node.crt", + "certs/node.key", + "certs/cert1.crt", + "certs/cert1.key", + "certs/cert2.crt", + "certs/cert2.key", +] + +# Serves HTTPS and acts as a TLS client towards itself. +node = cluster.add_instance("node", main_configs=["configs/ssl.xml"] + CERT_FILES) + +# Does not set `loadDefaultCAFile`, and OpenSSL's default CA file is ca2 for this instance. +node_with_default_cas = cluster.add_instance( + "node_with_default_cas", + main_configs=["configs/ssl_with_default_cas.xml"] + CERT_FILES, + env_variables={"SSL_CERT_FILE": f"{CONFIG_DIR}/ca2.crt"}, +) + +# Three nodes with embedded Keeper talking Raft over TLS. `loadDefaultCAFile` is not set for them: Keeper assumes `false` for +# the Raft connections then, unlike everything else, and their CA certificates have to be reloaded all the same. +keeper_nodes = [ + cluster.add_instance(f"node{i}", main_configs=[f"configs/keeper{i}.xml", "configs/ssl_with_default_cas.xml"] + CERT_FILES) for i in (1, 2, 3) +] + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def set_trusted_cas(instance, *cas): + """Overwrite ca.crt (the configured `caConfig`) with the given CA certificates.""" + sources = " ".join(f"{CONFIG_DIR}/{ca}.crt" for ca in cas) + instance.exec_in_container(["bash", "-c", f"cat {sources} > {CONFIG_DIR}/ca.crt.tmp && mv {CONFIG_DIR}/ca.crt.tmp {CONFIG_DIR}/ca.crt"]) + + +def set_own_certificate(instance, cert): + """Overwrite node.crt/node.key (the configured `certificateFile`/`privateKeyFile`) with the given leaf certificate.""" + instance.exec_in_container( + [ + "bash", + "-c", + f"cp {CONFIG_DIR}/{cert}.crt {CONFIG_DIR}/node.crt.tmp && mv {CONFIG_DIR}/node.crt.tmp {CONFIG_DIR}/node.crt && " + f"cp {CONFIG_DIR}/{cert}.key {CONFIG_DIR}/node.key.tmp && mv {CONFIG_DIR}/node.key.tmp {CONFIG_DIR}/node.key", + ] + ) + + +@pytest.fixture(autouse=True) +def restore_certificates(started_cluster): + yield + for instance in [node, node_with_default_cas] + keeper_nodes: + set_trusted_cas(instance, "ca1") + set_own_certificate(instance, "cert1") + instance.query("SYSTEM RELOAD CONFIG") + for instance in keeper_nodes: + kill_raft_connections(instance) + ku.wait_nodes(cluster, keeper_nodes) + + +def https_request_with_client_certificate(cert, instance=node): + """Query the HTTPS port of `instance` presenting the given client certificate. Returns the response, or None if the TLS handshake failed.""" + result = instance.exec_in_container( + [ + "bash", + "-c", + f"curl --silent --show-error --cacert {CONFIG_DIR}/ca1.crt --cert {CONFIG_DIR}/{cert}.crt --key {CONFIG_DIR}/{cert}.key " + f"'https://localhost:8443/?query=SELECT%201' 2>&1 || echo CURL_FAILED", + ] + ) + return None if "CURL_FAILED" in result else result + + +def assert_eventually(predicate, description, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.5) + assert predicate(), description + + +def test_server_reloads_ca(started_cluster): + """The CAs used to verify client certificates follow the content of `caConfig` without a restart or an explicit reload.""" + assert https_request_with_client_certificate("cert1") == "1\n" + assert https_request_with_client_certificate("cert2") is None + + # Trust both CAs: the file change alone triggers the reload. + set_trusted_cas(node, "ca1", "ca2") + assert_eventually(lambda: https_request_with_client_certificate("cert2") == "1\n", "cert2 is accepted after ca2 was added") + assert https_request_with_client_certificate("cert1") == "1\n" + + # Drop the old CA: certificates issued by it are not accepted anymore. + set_trusted_cas(node, "ca2") + assert_eventually(lambda: https_request_with_client_certificate("cert1") is None, "cert1 is rejected after ca1 was removed") + assert https_request_with_client_certificate("cert2") == "1\n" + + assert node.contains_in_log("Reloaded CA certificates") + + +def test_client_reloads_ca(started_cluster): + """The CAs used to verify server certificates of outgoing connections follow the content of `caConfig`.""" + # `node` connects to its own HTTPS port, which serves cert1. `Connection: close` rules out reusing a pooled connection. + query = "SELECT * FROM url('https://localhost:8443/?query=SELECT%201', 'TSV', 'x UInt8', headers('Connection'='close'))" + + set_trusted_cas(node, "ca2") + node.query("SYSTEM RELOAD CONFIG") + error = node.query_and_get_error(query) + assert "certificate verify failed" in error, error + + set_trusted_cas(node, "ca1") + node.query("SYSTEM RELOAD CONFIG") + assert node.query(query) == "1\n" + + +def test_default_cas_are_kept(started_cluster): + """With `loadDefaultCAFile` left at its default, both the CAs from `caConfig` and the default ones are trusted, before and after a reload.""" + # cert1 is trusted through `caConfig` (ca.crt = ca1), cert2 through the default CA file (SSL_CERT_FILE = ca2). + assert https_request_with_client_certificate("cert1", node_with_default_cas) == "1\n" + assert https_request_with_client_certificate("cert2", node_with_default_cas) == "1\n" + + reloads = int(node_with_default_cas.count_in_log("Reloaded CA certificates").strip()) + set_trusted_cas(node_with_default_cas, "ca1") # same content, new modification time + node_with_default_cas.query("SYSTEM RELOAD CONFIG") + assert int(node_with_default_cas.count_in_log("Reloaded CA certificates").strip()) > reloads + + assert https_request_with_client_certificate("cert1", node_with_default_cas) == "1\n" + assert https_request_with_client_certificate("cert2", node_with_default_cas) == "1\n" + + +def test_system_certificates_follows_reload(started_cluster): + """`system.certificates` shows the CA certificates that are currently used, also after `caConfig` is changed to another file.""" + query = "SELECT path, subject LIKE '%Test Root CA {}%' FROM system.certificates WHERE NOT default" + assert node.query(query.format(1)) == f"{CONFIG_DIR}/ca.crt\t1\n" + + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca.crt", f"{CONFIG_DIR}/ca2.crt") + try: + node.query("SYSTEM RELOAD CONFIG") + assert node.query(query.format(2)) == f"{CONFIG_DIR}/ca2.crt\t1\n" + finally: + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca2.crt", f"{CONFIG_DIR}/ca.crt") + node.query("SYSTEM RELOAD CONFIG") + + +def test_ca_directory(started_cluster): + """`caConfig` can be a directory with certificates named by their subject hash. Replacing a certificate in it is noticed too.""" + ca_dir = f"{CONFIG_DIR}/ca_dir" + node.exec_in_container(["bash", "-c", f"mkdir -p {ca_dir} && cp {CONFIG_DIR}/ca1.crt {ca_dir}/$(openssl x509 -noout -subject_hash -in {CONFIG_DIR}/ca1.crt).0"]) + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca.crt", ca_dir) + try: + node.query("SYSTEM RELOAD CONFIG") + assert https_request_with_client_certificate("cert1") == "1\n" + assert https_request_with_client_certificate("cert2") is None + + # Overwrite the only file in place: the set of file names in the directory does not change. + node.exec_in_container(["bash", "-c", f"cat {CONFIG_DIR}/ca2.crt > {ca_dir}/$(openssl x509 -noout -subject_hash -in {CONFIG_DIR}/ca1.crt).0"]) + node.query("SYSTEM RELOAD CONFIG") + assert https_request_with_client_certificate("cert1") is None + finally: + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", ca_dir, f"{CONFIG_DIR}/ca.crt") + node.exec_in_container(["rm", "-rf", ca_dir]) + node.query("SYSTEM RELOAD CONFIG") + + +def kill_raft_connections(instance): + instance.exec_in_container( + ["bash", "-c", "ss --kill -tn state established '( dport = :9234 or sport = :9234 )' > /dev/null"], nothrow=True + ) + + +def raft_port_accepts_client_certificate(instance, target, cert): + """Whether the Raft port of `target` completes a TLS handshake with a client that presents `cert`.""" + # With TLS 1.2 the server verifies the client certificate before the handshake completes on the client side, + # so a rejected certificate reliably shows up as a failed handshake in s_client. + result = instance.exec_in_container( + [ + "bash", + "-c", + f"openssl s_client -brief -tls1_2 -connect {target.name}:9234 -cert {CONFIG_DIR}/{cert}.crt -key {CONFIG_DIR}/{cert}.key " + f"&1 || true", + ] + ) + return "CONNECTION ESTABLISHED" in result + + +def raft_port_certificate_issuer(instance, target): + return instance.exec_in_container( + [ + "bash", + "-c", + f"openssl s_client -connect {target.name}:9234 /dev/null | openssl x509 -noout -issuer 2>/dev/null || true", + ] + ).strip() + + +def check_keeper_cluster_works(path): + connections = [] + try: + for instance in keeper_nodes: + connections.append(ku.get_fake_zk(cluster, instance.name)) + connections[0].create(path, b"data") + for connection in connections: + connection.sync(path) + assert connection.get(path)[0] == b"data" + finally: + for connection in connections: + connection.stop() + connection.close() + + +def reload_and_reconnect_raft(): + for instance in keeper_nodes: + instance.query("SYSTEM RELOAD CONFIG") + for instance in keeper_nodes: + kill_raft_connections(instance) + ku.wait_nodes(cluster, keeper_nodes) + + +def test_keeper_raft_reloads_ca(started_cluster): + """Rotate the CA of the Raft connections between Keeper nodes without restarting them.""" + run = uuid.uuid4().hex + ku.wait_nodes(cluster, keeper_nodes) + check_keeper_cluster_works(f"/before_{run}") + assert "Test Root CA 1" in raft_port_certificate_issuer(keeper_nodes[0], keeper_nodes[1]) + + # 1. Trust the new CA in addition to the old one. + for instance in keeper_nodes: + set_trusted_cas(instance, "ca1", "ca2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/both_cas_{run}") + + # 2. Switch the nodes to certificates issued by the new CA. + for instance in keeper_nodes: + set_own_certificate(instance, "cert2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/new_certs_{run}") + for target in keeper_nodes[1:]: + assert "Test Root CA 2" in raft_port_certificate_issuer(keeper_nodes[0], target) + + # 3. Stop trusting the old CA. + for instance in keeper_nodes: + set_trusted_cas(instance, "ca2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/new_ca_{run}") + + assert raft_port_accepts_client_certificate(keeper_nodes[0], keeper_nodes[1], "cert2") + assert not raft_port_accepts_client_certificate(keeper_nodes[0], keeper_nodes[1], "cert1") From 36971ad11119f280dfec6bcfe25c1391ca784659 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 27 Sep 2026 22:52:09 +0000 Subject: [PATCH 113/296] Backport #122337 to 26.8: Fix Keeper crash after a failed changelog preallocation --- src/Common/FailPoint.cpp | 1 + src/Coordination/Changelog.cpp | 13 +++++--- .../tests/gtest_coordination_changelog.cpp | 31 +++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 4ea56fc9fb76e..60ae93c6ff4c2 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -349,6 +349,7 @@ static struct InitFiu PAUSEABLE(keeper_changelog_readahead_pre_drain) \ PAUSEABLE(object_storage_source_pause_before_virtual_columns) \ REGULAR(keeper_changelog_readahead_fill_exception) \ + ONCE(keeper_changelog_preallocate_no_space) \ REGULAR(distributed_plan_record_failure_while_starting_tasks) \ ONCE(zk_send_thread_request_window_throw) \ ONCE(zk_send_thread_operations_insert_throw) \ diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 4b358776b2c13..a5d204ea5bce5 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -92,6 +92,7 @@ namespace FailPoints extern const char keeper_changelog_readahead_park_armed[]; extern const char keeper_changelog_readahead_pre_drain[]; extern const char keeper_changelog_readahead_fill_exception[]; + extern const char keeper_changelog_preallocate_no_space[]; } namespace @@ -347,7 +348,8 @@ class ChangelogWriter bool appendRecord(ChangelogRecord && record) { const auto * file_buffer = tryGetFileBaseBuffer(); - chassert(file_buffer && current_file_description); + if (!file_buffer || !current_file_description) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Log writer wasn't initialized for any file"); chassert(record.header.index - getStartIndex() <= current_file_description->expectedEntriesCountInLog()); // check if log file reached the limit for amount of records it can contain @@ -590,6 +592,12 @@ class ChangelogWriter file_buffer->getFD(), FALLOC_FL_KEEP_SIZE, 0, log_file_settings.max_size + log_file_settings.overallocate_size); } while (res < 0 && errno == EINTR); + fiu_do_on(FailPoints::keeper_changelog_preallocate_no_space, + { + res = -1; + errno = ENOSPC; + }); + if (res != 0) { if (errno == ENOSPC) @@ -4154,9 +4162,6 @@ void Changelog::appendCompletionThread() bool append_ok = false; while (append_completion_queue.pop(append_ok)) { - if (!append_ok) - current_writer->finalize(); - // we shouldn't start the raft_server before sending it here if (auto raft_server_locked = raft_server.lock()) raft_server_locked->notify_log_append_completion(append_ok); diff --git a/src/Coordination/tests/gtest_coordination_changelog.cpp b/src/Coordination/tests/gtest_coordination_changelog.cpp index 74e6c2b1a2ffb..ef2a02920b17f 100644 --- a/src/Coordination/tests/gtest_coordination_changelog.cpp +++ b/src/Coordination/tests/gtest_coordination_changelog.cpp @@ -30,6 +30,7 @@ namespace FailPoints { extern const char keeper_changelog_read_plan_resolved[]; extern const char keeper_changelog_removed_from_disk_set[]; + extern const char keeper_changelog_preallocate_no_space[]; } namespace ErrorCodes @@ -346,6 +347,36 @@ TEST_P(CoordinationTestWithCompression, ChangelogTestFlushThrottling) EXPECT_GE(watch.elapsedMilliseconds(), 100); } +/// A failed preallocation (e.g. `ENOSPC`) fails the batch, but must leave the writer usable: +/// the next append retries the preallocation. Previously the append completion thread +/// finalized the writer without holding the writer lock, and the next append dereferenced +/// the destroyed file buffer. +TEST_P(CoordinationTestWithCompression, ChangelogTestAppendAfterPreallocationFailure) +{ + ChangelogDirTest test("./logs"); + this->setLogDirectory("./logs"); + + DB::KeeperLogStore changelog( + DB::LogFileSettings{ + .force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 1000, .max_size = 1024 * 1024}, + DB::FlushSettings(), + DB::ReadAheadSettings{}, + this->keeper_context); + changelog.init(0, 0); + + DB::FailPointInjection::enableFailPoint(DB::FailPoints::keeper_changelog_preallocate_no_space); + + auto entry = getLogEntry("hello world", 77); + changelog.append(entry); + EXPECT_FALSE(changelog.flush()); + + for (size_t i = 0; i < 10; ++i) + { + changelog.append(entry); + EXPECT_TRUE(changelog.flush()); + } +} + TEST_P(CoordinationTestWithCompression, ChangelogTestFile) { ChangelogDirTest test("./logs"); From 9b80c8730866a0f6d6c7b11e5d298a58bda8eda3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Sep 2026 10:10:38 +0000 Subject: [PATCH 114/296] Backport #121397 to 26.8: Do not materialize a lazily replicated array column in ARRAY JOIN --- src/Interpreters/ArrayJoinAction.cpp | 84 ++++++++++++------- src/Interpreters/ArrayJoinAction.h | 9 +- tests/performance/array_join.xml | 8 ++ ...join_replicated_array_stays_lazy.reference | 25 ++++++ ...array_join_replicated_array_stays_lazy.sql | 30 +++++++ 5 files changed, 127 insertions(+), 29 deletions(-) create mode 100644 tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.reference create mode 100644 tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.sql diff --git a/src/Interpreters/ArrayJoinAction.cpp b/src/Interpreters/ArrayJoinAction.cpp index ef87afc35c30f..1b5126a770704 100644 --- a/src/Interpreters/ArrayJoinAction.cpp +++ b/src/Interpreters/ArrayJoinAction.cpp @@ -42,6 +42,15 @@ static ColumnPtr getArrayJoinColumn(const ColumnPtr & column) return column; if (const auto * map = typeid_cast(column.get())) return map->getNestedColumnPtr(); + /// Keep replicated arrays lazy, only unwrap maps. + if (const auto * replicated = typeid_cast(column.get())) + { + const auto & nested = replicated->getNestedColumn(); + if (typeid_cast(nested.get())) + return column; + if (const auto * map = typeid_cast(nested.get())) + return ColumnReplicated::create(map->getNestedColumnPtr(), replicated->getIndexesColumn()); + } return nullptr; } @@ -151,11 +160,8 @@ ArrayJoinResultIterator::ArrayJoinResultIterator(const ArrayJoinAction * array_j const auto & function_array_resize = array_join->function_array_resize; const auto & function_builder = array_join->function_builder; - /// TODO: avoid convertToFullColumnIfReplicated - any_array_map_ptr = block.getByName(*columns.begin()).column->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); - any_array = getArrayJoinColumnRawPtr(any_array_map_ptr); - if (!any_array) - throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN requires array or map argument"); + any_array_map_ptr = block.getByName(*columns.begin()).column->convertToFullColumnIfConst(); + initAnyArray(); if (is_unaligned) { @@ -187,27 +193,60 @@ ArrayJoinResultIterator::ArrayJoinResultIterator(const ArrayJoinAction * array_j any_array_map_ptr = src_col.column->convertToFullColumnIfConst(); } - any_array = getArrayJoinColumnRawPtr(any_array_map_ptr); - if (!any_array) - throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN requires array or map argument"); + initAnyArray(); } else if (is_left) { for (const auto & name : columns) { const auto & src_col = block.getByName(name); - ColumnWithTypeAndName array_col = convertArrayJoinColumn(src_col); + /// emptyArrayToSingle is fine with a replicated input, no need to materialize it. + ColumnWithTypeAndName array_col{getArrayJoinColumn(src_col.column->convertToFullColumnIfConst()), getArrayJoinDataType(src_col.type), src_col.name}; ColumnsWithTypeAndName tmp_block{array_col}; non_empty_array_columns[name] = function_builder->build(tmp_block)->execute(tmp_block, array_col.type, array_col.column->size(), /* dry_run = */ false); } any_array_map_ptr = non_empty_array_columns.begin()->second->convertToFullColumnIfConst(); - any_array = getArrayJoinColumnRawPtr(any_array_map_ptr); - if (!any_array) - throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN requires array or map argument"); + initAnyArray(); } } +void ArrayJoinResultIterator::initAnyArray() +{ + any_array = getArrayJoinColumnRawPtr(any_array_map_ptr); + if (any_array) + return; + + /// Replicated arrays are materialized per window, here we only need the row sizes. + const auto * replicated = typeid_cast(any_array_map_ptr.get()); + const auto * nested_array = replicated ? getArrayJoinColumnRawPtr(replicated->getNestedColumn()) : nullptr; + if (!nested_array) + throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN requires array or map argument"); + + const auto & nested_offsets = nested_array->getOffsets(); + const auto & indexes = replicated->getIndexes(); + replicated_offsets.resize(replicated->size()); + size_t accumulated = 0; + for (size_t row = 0; row < replicated_offsets.size(); ++row) + { + size_t index = indexes.getIndexAt(row); + accumulated += nested_offsets[index] - nested_offsets[index - 1]; + replicated_offsets[row] = accumulated; + } +} + +const IColumn::Offsets & ArrayJoinResultIterator::anyOffsets() const +{ + return any_array ? any_array->getOffsets() : replicated_offsets; +} + +ColumnPtr ArrayJoinResultIterator::cutAnyArray(size_t start, size_t length) const +{ + if (any_array) + return any_array->cut(start, length); + return getArrayJoinColumn(any_array_map_ptr->cut(start, length)->convertToFullColumnIfReplicated()); +} + bool ArrayJoinResultIterator::hasNext() const { return total_rows != 0 && current_row < total_rows; @@ -220,7 +259,7 @@ Block ArrayJoinResultIterator::next() throw Exception(ErrorCodes::LOGICAL_ERROR, "No more elements in ArrayJoinResultIterator."); size_t max_block_size = array_join->max_block_size; - const auto & offsets = any_array->getOffsets(); + const auto & offsets = anyOffsets(); /// Make sure output block rows do not exceed max_block_size. size_t next_row = current_row; @@ -237,7 +276,7 @@ Block ArrayJoinResultIterator::next() const auto & columns = array_join->columns; bool is_unaligned = array_join->is_unaligned; bool is_left = array_join->is_left; - auto cut_any_col = any_array->cut(current_row, next_row - current_row); + auto cut_any_col = cutAnyArray(current_row, next_row - current_row); const auto * cut_any_array = typeid_cast(cut_any_col.get()); ColumnPtr indexes_for_lazy_replication; @@ -258,20 +297,9 @@ Block ArrayJoinResultIterator::next() { if (const auto & type = getArrayJoinDataType(current.type)) { - ColumnPtr array_ptr; - if (typeid_cast(current.type.get())) - { - array_ptr = (is_left && !is_unaligned) ? non_empty_array_columns[current.name]->cut(current_row, next_row - current_row) - : current.column; - array_ptr = array_ptr->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); - } - else - { - ColumnPtr map_ptr = current.column->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); - const ColumnMap & map = typeid_cast(*map_ptr); - array_ptr = (is_left && !is_unaligned) ? non_empty_array_columns[current.name]->cut(current_row, next_row - current_row) - : map.getNestedColumnPtr(); - } + ColumnPtr array_ptr = (is_left && !is_unaligned) ? non_empty_array_columns[current.name]->cut(current_row, next_row - current_row) + : getArrayJoinColumn(current.column->convertToFullColumnIfConst()->convertToFullColumnIfReplicated()); + array_ptr = array_ptr->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); const ColumnArray & array = typeid_cast(*array_ptr); if (!is_unaligned && !array.hasEqualOffsets(*cut_any_array)) diff --git a/src/Interpreters/ArrayJoinAction.h b/src/Interpreters/ArrayJoinAction.h index aac5aac7b9db7..141c9e440d5bc 100644 --- a/src/Interpreters/ArrayJoinAction.h +++ b/src/Interpreters/ArrayJoinAction.h @@ -2,6 +2,7 @@ #include #include +#include #include @@ -58,12 +59,18 @@ class ArrayJoinResultIterator bool hasNext() const; private: + void initAnyArray(); + const PaddedPODArray & anyOffsets() const; + ColumnPtr cutAnyArray(size_t start, size_t length) const; + const ArrayJoinAction * array_join; Block block; bool enable_lazy_columns_replication; ColumnPtr any_array_map_ptr; - const ColumnArray * any_array; + /// Null if the joined column is replicated, then replicated_offsets is used instead. + const ColumnArray * any_array = nullptr; + PaddedPODArray replicated_offsets; /// If LEFT ARRAY JOIN, then we create columns in which empty arrays are replaced by arrays with one element - the default value. std::map non_empty_array_columns; diff --git a/tests/performance/array_join.xml b/tests/performance/array_join.xml index b3fc24d316652..af586967907b0 100644 --- a/tests/performance/array_join.xml +++ b/tests/performance/array_join.xml @@ -1,4 +1,6 @@ + CREATE TABLE array_join_wide_chain (A Array(String), B Array(String), C Array(String)) ENGINE = MergeTree ORDER BY tuple() + INSERT INTO array_join_wide_chain SELECT arrayMap(i -> concat('a', toString(number), '_', toString(i)), range(100)), arrayMap(i -> concat('b', toString(number), '_', toString(i)), range(100)), arrayMap(i -> concat('c', toString(number), '_', toString(i)), range(100)) FROM numbers(100000) @@ -10,4 +12,10 @@ SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 with 'clickhouse' as str select arrayJoin(range(number % 10)), materialize(str) from numbers(10000000) + + + SELECT a, b, c FROM array_join_wide_chain ARRAY JOIN A AS a ARRAY JOIN B AS b ARRAY JOIN C AS c LIMIT 31 SETTINGS max_threads = 1 FORMAT Null + SELECT a, b, c FROM array_join_wide_chain LEFT ARRAY JOIN A AS a LEFT ARRAY JOIN B AS b LEFT ARRAY JOIN C AS c LIMIT 31 SETTINGS max_threads = 1 FORMAT Null + + DROP TABLE IF EXISTS array_join_wide_chain diff --git a/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.reference b/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.reference new file mode 100644 index 0000000000000..6072b25b79680 --- /dev/null +++ b/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.reference @@ -0,0 +1,25 @@ +1 a1_0 0 ('k',1) +1 a1_0 0 ('kk',2) +2 a2_0 0 ('k',2) +2 a2_0 0 ('kk',4) +2 a2_0 1 ('k',2) +2 a2_0 1 ('kk',4) +2 a2_1 0 ('k',2) +2 a2_1 0 ('kk',4) +2 a2_1 1 ('k',2) +2 a2_1 1 ('kk',4) +5 a5_0 0 ('k',5) +5 a5_0 0 ('kk',10) +5 a5_1 0 ('k',5) +5 a5_1 0 ('kk',10) +14 7208362719481167226 +14 7208362719481167226 +28 17533497658690827015 +28 17533497658690827015 +7 17387321633734044581 +7 17387321633734044581 +5 11221959250764763722 +5 11221959250764763722 +10 +10 +10 diff --git a/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.sql b/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.sql new file mode 100644 index 0000000000000..6155f4f853ef5 --- /dev/null +++ b/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.sql @@ -0,0 +1,30 @@ +-- The second and third joins get their arrays lazily replicated from the previous one. +SET enable_lazy_columns_replication = 1; + +DROP TABLE IF EXISTS t_lazy_arrays; +CREATE TABLE t_lazy_arrays (id UInt32, a Array(String), b Array(UInt32), c Array(String), m Map(String, UInt32)) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_lazy_arrays SELECT number, arrayMap(i -> concat('a', toString(number), '_', toString(i)), range(number % 3)), range(number % 4), arrayMap(i -> concat('c', toString(i)), range(number % 4)), map('k', number, 'kk', number * 2) FROM numbers(7); + +-- Results must not depend on lazy replication. +SELECT id, x, y, kv FROM t_lazy_arrays ARRAY JOIN a AS x ARRAY JOIN b AS y ARRAY JOIN m AS kv ORDER BY ALL SETTINGS max_block_size = 3; +SELECT count(), sum(cityHash64(id, x, y, kv)) FROM t_lazy_arrays ARRAY JOIN a AS x ARRAY JOIN b AS y ARRAY JOIN m AS kv SETTINGS max_block_size = 3; +SELECT count(), sum(cityHash64(id, x, y, kv)) FROM t_lazy_arrays ARRAY JOIN a AS x ARRAY JOIN b AS y ARRAY JOIN m AS kv SETTINGS max_block_size = 3, enable_lazy_columns_replication = 0; +SELECT count(), sum(cityHash64(id, x, y, kv)) FROM t_lazy_arrays LEFT ARRAY JOIN a AS x LEFT ARRAY JOIN b AS y LEFT ARRAY JOIN m AS kv SETTINGS max_block_size = 3; +SELECT count(), sum(cityHash64(id, x, y, kv)) FROM t_lazy_arrays LEFT ARRAY JOIN a AS x LEFT ARRAY JOIN b AS y LEFT ARRAY JOIN m AS kv SETTINGS max_block_size = 3, enable_lazy_columns_replication = 0; +SELECT count(), sum(cityHash64(id, x, y, z)) FROM t_lazy_arrays ARRAY JOIN a AS x ARRAY JOIN b AS y, c AS z SETTINGS max_block_size = 3; +SELECT count(), sum(cityHash64(id, x, y, z)) FROM t_lazy_arrays ARRAY JOIN a AS x ARRAY JOIN b AS y, c AS z SETTINGS max_block_size = 3, enable_lazy_columns_replication = 0; +SELECT count(), sum(cityHash64(id, x, y)) FROM t_lazy_arrays ARRAY JOIN a AS x ARRAY JOIN b AS y WHERE y % 2 = 0 SETTINGS max_block_size = 3; +SELECT count(), sum(cityHash64(id, x, y)) FROM t_lazy_arrays ARRAY JOIN a AS x ARRAY JOIN b AS y WHERE y % 2 = 0 SETTINGS max_block_size = 3, enable_lazy_columns_replication = 0; + +DROP TABLE t_lazy_arrays; + +DROP TABLE IF EXISTS t_lazy_arrays_wide; +CREATE TABLE t_lazy_arrays_wide (id UInt32, a Array(String), b Array(String), c Array(String)) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_lazy_arrays_wide SELECT number, arrayMap(i -> concat('a', toString(number), '_', toString(i), 'xxxxxxxxxx'), range(100)), arrayMap(i -> concat('b', toString(number), '_', toString(i), 'xxxxxxxxxx'), range(100)), arrayMap(i -> concat('c', toString(number), '_', toString(i), 'xxxxxxxxxx'), range(100)) FROM numbers(1000); + +-- Used to materialize the whole block of arrays in every join, way above this limit. +SELECT count() FROM (SELECT x, y, z FROM t_lazy_arrays_wide ARRAY JOIN a AS x ARRAY JOIN b AS y ARRAY JOIN c AS z LIMIT 10) SETTINGS max_block_size = 65409, max_threads = 1, enable_lazy_columns_replication = 1, max_memory_usage = 100000000; +SELECT count() FROM (SELECT x, y, z FROM t_lazy_arrays_wide LEFT ARRAY JOIN a AS x LEFT ARRAY JOIN b AS y LEFT ARRAY JOIN c AS z LIMIT 10) SETTINGS max_block_size = 65409, max_threads = 1, enable_lazy_columns_replication = 1, max_memory_usage = 100000000; +SELECT count() FROM (SELECT arrayJoin(a) AS x, arrayJoin(b) AS y, arrayJoin(c) AS z FROM t_lazy_arrays_wide WHERE id >= 0 LIMIT 10) SETTINGS max_block_size = 65409, max_threads = 1, enable_lazy_columns_replication = 1, max_memory_usage = 100000000, query_plan_lower_array_join_function = 1; + +DROP TABLE t_lazy_arrays_wide; From afada3c3f66c41148663c05f7347fe63278ccdf0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Sep 2026 10:43:53 +0000 Subject: [PATCH 115/296] Backport #119097 to 26.8: Bump `zxc` from 0.13.1 to 0.14.0 --- .gitmodules | 2 +- contrib/zxc | 2 +- contrib/zxc-cmake/CMakeLists.txt | 38 +++++++++++++++++++------------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/.gitmodules b/.gitmodules index 2f3dd2930a0a0..db76496d84cb0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -416,7 +416,7 @@ url = https://github.com/ClickHouse/geometry.hpp [submodule "contrib/zxc"] path = contrib/zxc - url = https://github.com/ClickHouse/zxc + url = https://github.com/hellobertrand/zxc [submodule "contrib/libucontext"] path = contrib/libucontext url = https://github.com/kaniini/libucontext diff --git a/contrib/zxc b/contrib/zxc index b9890cfe3466b..e568d1967309f 160000 --- a/contrib/zxc +++ b/contrib/zxc @@ -1 +1 @@ -Subproject commit b9890cfe3466b19d0eb5005874ce81728efb9565 +Subproject commit e568d1967309fd5d160f45f2fdabcbff6ab3f058 diff --git a/contrib/zxc-cmake/CMakeLists.txt b/contrib/zxc-cmake/CMakeLists.txt index 98c52cb1111bb..70451492f30dc 100644 --- a/contrib/zxc-cmake/CMakeLists.txt +++ b/contrib/zxc-cmake/CMakeLists.txt @@ -33,9 +33,14 @@ set(ZXC_CORE_SOURCES # emitted symbols (e.g. zxc_compress_chunk_wrapper_avx2). zxc_dispatch.c declares # these suffixed symbols unconditionally per architecture, so the exact set of # variants below must match what the dispatcher expects for each target: -# x86-64 -> _default, _sse2, _avx2, _avx512 -# aarch64 -> _default, _neon +# x86-64 -> _default, _avx2, _avx512 +# aarch64 -> _default only (NEON is the AArch64 baseline) # other -> _default only +# SSE2 is the x86-64 baseline and NEON the AArch64 one, so `_default` already +# compiles those code paths; zxc dropped the dedicated `_sse2` and `_neon` +# variants in v0.14.0 and the dispatcher no longer references them. The +# remaining `_neon32` variant is for 32-bit ARM, which ClickHouse does not +# target. set(ZXC_VARIANT_OBJECTS "") macro(zxc_add_variant suffix) @@ -54,11 +59,14 @@ macro(zxc_add_variant suffix) endmacro() if (OS_DARWIN AND ARCH_AMD64) - # zxc's x86-64 runtime dispatch calls `__builtin_cpu_init` / + # Historically zxc's x86-64 runtime dispatch called `__builtin_cpu_init` / # `__builtin_cpu_supports`, which reference compiler-rt's `__cpu_model`. That - # symbol is not linked into ClickHouse's macOS cross-build, so linking fails - # with "undefined ___cpu_model". macOS builds are for local development only, - # so restrict zxc to its portable scalar core there. + # symbol is not linked into ClickHouse's macOS cross-build, so linking failed + # with "undefined ___cpu_model". Since v0.14.0 the dispatcher issues `CPUID` + # and `XGETBV` directly and no longer needs `__cpu_model`, so this is now a + # conservative restriction rather than a required one: macOS builds are for + # local development only and are never test-run in CI, so there is nothing + # to gain from the x86 SIMD variants there. Lifting it is a separate change. set(ZXC_FORCE_SCALAR TRUE) elseif (SANITIZE STREQUAL "memory") # clang's MemorySanitizer has no precise shadow model for the x86 byte-shuffle @@ -67,7 +75,7 @@ elseif (SANITIZE STREQUAL "memory") # that the shuffle discards still poisons the logical result. zxc's SIMD # kernels deliberately do speculative 16/32-byte loads whose tail lanes fall # into malloc'ed scratch buffers or not-yet-written output and are then - # discarded by such shuffles (e.g. `zxc_decode_copy_overlap_run`, + # discarded by such shuffles (e.g. `zxc_decode_copy_overlap_run32`, # `zxc_pivco_merge`), which triggers false `use-of-uninitialized-value` # reports. The scalar core branches only on initialized bytes, so build only # that under MSan. (AArch64 NEON `tbl` gets a precise shadow, but keep all @@ -90,16 +98,16 @@ if (ZXC_FORCE_SCALAR) target_compile_definitions(${_tgt} PRIVATE ZXC_DISABLE_SIMD) endforeach() elseif (ARCH_AMD64) - # SSE2/AVX2 flags are redundant with the x86-64-v3 baseline but harmless; + # The AVX2 flags are redundant with the x86-64-v3 baseline but harmless; # AVX512 is NOT in the baseline, so those flags are required for that variant - # (only ever entered at runtime on AVX-512 capable CPUs). - zxc_add_variant(_sse2 -msse2) - zxc_add_variant(_avx2 -mavx2 -mfma -mbmi -mbmi2 -mlzcnt) + # (only ever entered at runtime on AVX-512 capable CPUs). The flag sets must + # stay in sync with `zxc_detect_cpu_features`, which admits a CPU to the AVX2 + # and AVX-512 tiers only after proving BMI1, BMI2 and LZCNT as well. Note + # that it does not probe FMA, so `-mfma` must not be added here: the + # compiler would be free to emit FMA instructions into a variant that runs + # on any AVX2 CPU. + zxc_add_variant(_avx2 -mavx2 -mbmi -mbmi2 -mlzcnt) zxc_add_variant(_avx512 -mavx512f -mavx512bw -mavx512vbmi -mavx512vbmi2 -mbmi -mbmi2 -mlzcnt) -elseif (ARCH_AARCH64) - # NEON is guaranteed by ClickHouse's armv8.2-a+simd baseline; no extra flags - # (adding -march=armv8-a would downgrade below the baseline). - zxc_add_variant(_neon) endif () add_library(_zxc ${ZXC_CORE_SOURCES} ${ZXC_VARIANT_OBJECTS}) From b2dd7951beafc18f4596a4e2695793e25e1c4cfc Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 28 Sep 2026 12:54:20 +0000 Subject: [PATCH 116/296] Drop the query needing a master-only setting from 05234 `query_plan_lower_array_join_function` does not exist on 26.8, so the last query of `05234_array_join_replicated_array_stays_lazy` fails with UNKNOWN_SETTING. Without the setting the `arrayJoin` function form is not lowered to ARRAY JOIN, so the query would not exercise this change anyway. This was meant to be part of the cherry-pick resolution and is already in the 26.7, 26.3 and the three private ones; on this branch alone the edit was left unstaged, so the commit kept the auto-merged test. Both files are taken verbatim from the 26.7 cherry-pick (03c6b226231). https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=122650&sha=9b80c8730866a0f6d6c7b11e5d298a58bda8eda3&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20distributed%20plan%2C%20parallel%29 Related: https://github.com/ClickHouse/ClickHouse/pull/122650 Co-Authored-By: Claude Opus 5 (1M context) --- .../05234_array_join_replicated_array_stays_lazy.reference | 1 - .../0_stateless/05234_array_join_replicated_array_stays_lazy.sql | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.reference b/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.reference index 6072b25b79680..8ed3d59980230 100644 --- a/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.reference +++ b/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.reference @@ -22,4 +22,3 @@ 5 11221959250764763722 10 10 -10 diff --git a/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.sql b/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.sql index 6155f4f853ef5..d733b18575219 100644 --- a/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.sql +++ b/tests/queries/0_stateless/05234_array_join_replicated_array_stays_lazy.sql @@ -25,6 +25,5 @@ INSERT INTO t_lazy_arrays_wide SELECT number, arrayMap(i -> concat('a', toString -- Used to materialize the whole block of arrays in every join, way above this limit. SELECT count() FROM (SELECT x, y, z FROM t_lazy_arrays_wide ARRAY JOIN a AS x ARRAY JOIN b AS y ARRAY JOIN c AS z LIMIT 10) SETTINGS max_block_size = 65409, max_threads = 1, enable_lazy_columns_replication = 1, max_memory_usage = 100000000; SELECT count() FROM (SELECT x, y, z FROM t_lazy_arrays_wide LEFT ARRAY JOIN a AS x LEFT ARRAY JOIN b AS y LEFT ARRAY JOIN c AS z LIMIT 10) SETTINGS max_block_size = 65409, max_threads = 1, enable_lazy_columns_replication = 1, max_memory_usage = 100000000; -SELECT count() FROM (SELECT arrayJoin(a) AS x, arrayJoin(b) AS y, arrayJoin(c) AS z FROM t_lazy_arrays_wide WHERE id >= 0 LIMIT 10) SETTINGS max_block_size = 65409, max_threads = 1, enable_lazy_columns_replication = 1, max_memory_usage = 100000000, query_plan_lower_array_join_function = 1; DROP TABLE t_lazy_arrays_wide; From 452cc3d8b5d6ead3f6c803cc10963b5bf560b2a5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Sep 2026 16:10:33 +0000 Subject: [PATCH 117/296] Update autogenerated version to 26.8.14.3 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index af630df25c06c..4e3df68592efd 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54526) +SET(VERSION_REVISION 54527) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 14) -SET(VERSION_GITHASH 3eac80eef9b1f81e88b581a6c107c59f21d9e15c) -SET(VERSION_DESCRIBE v26.8.14.1-lts) -SET(VERSION_STRING 26.8.14.1) +SET(VERSION_PATCH 15) +SET(VERSION_GITHASH f1d4a0d36450556c9e3c4f8b3244770b20345516) +SET(VERSION_DESCRIBE v26.8.15.1-lts) +SET(VERSION_STRING 26.8.15.1) # end of autochange From 0993565cf3b13c885892331beafdaad019d25d3a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Sep 2026 17:55:11 +0000 Subject: [PATCH 118/296] Backport #122035 to 26.8: Disable statistics pruning in case of row level policy --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 10 +++- ...atistics_part_pruning_row_policy.reference | 16 ++++++ ...255_statistics_part_pruning_row_policy.sql | 56 +++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05255_statistics_part_pruning_row_policy.reference create mode 100644 tests/queries/0_stateless/05255_statistics_part_pruning_row_policy.sql diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 30930da722f4b..03feebb56429d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -821,10 +822,17 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByStatistics( /// 3. There are on-the-fly mutations or patch parts (statistics only reflects original data) /// 4. A masking policy applies: it rewrites values at read time, so the statistics (like /// the on-the-fly mutations above) no longer describe the values the query sees. + /// 5. A row policy applies: the statistics describe all rows of a part, including the ones the + /// policy hides, so the number of rows left to read after pruning by the query's predicate, + /// which is reported to the client, reveals the values of the hidden rows. The policy is + /// either pushed into this read (possibly from a wrapper such as `Alias`) or belongs to + /// this table and is applied above the read (e.g. for a child of `Merge`). if (!settings[Setting::use_statistics_for_part_pruning] || query_info.isFinal() || (mutations_snapshot && (mutations_snapshot->hasDataMutations() || mutations_snapshot->hasPatchParts())) - || (!parts.empty() && parts.front().data_part->storage.hasEnabledMaskingPolicies(context))) + || (!parts.empty() && parts.front().data_part->storage.hasEnabledMaskingPolicies(context)) + || query_info.row_level_filter + || (!parts.empty() && getEffectiveRowPolicyFilter(parts.front().data_part->storage, context))) { return parts; } diff --git a/tests/queries/0_stateless/05255_statistics_part_pruning_row_policy.reference b/tests/queries/0_stateless/05255_statistics_part_pruning_row_policy.reference new file mode 100644 index 0000000000000..39b64abfe2048 --- /dev/null +++ b/tests/queries/0_stateless/05255_statistics_part_pruning_row_policy.reference @@ -0,0 +1,16 @@ +1 +0 +0 +0 +0 +0 +0 +0 +row_policy_probe_1_no_policy_1998 1000 +row_policy_probe_2_no_policy_1999 0 +row_policy_probe_3_table_policy_1998 1000 +row_policy_probe_4_table_policy_1999 1000 +row_policy_probe_5_merge_child_policy_1998 1000 +row_policy_probe_6_merge_child_policy_1999 1000 +row_policy_probe_7_alias_policy_1998 1000 +row_policy_probe_8_alias_policy_1999 1000 diff --git a/tests/queries/0_stateless/05255_statistics_part_pruning_row_policy.sql b/tests/queries/0_stateless/05255_statistics_part_pruning_row_policy.sql new file mode 100644 index 0000000000000..da69c5c4a485e --- /dev/null +++ b/tests/queries/0_stateless/05255_statistics_part_pruning_row_policy.sql @@ -0,0 +1,56 @@ +-- Statistics describe all rows of a part, including the ones a row policy hides. Pruning parts by them +-- would make the number of rows read an oracle over the values of the hidden rows, so statistics +-- pruning is disabled when a row policy applies, whether the policy belongs to the table, to a child +-- of `Merge`, or to a wrapper such as `Alias`. + +DROP ROW POLICY IF EXISTS payroll_policy ON payroll; +DROP ROW POLICY IF EXISTS payroll_alias_policy ON payroll_alias; +DROP TABLE IF EXISTS payroll_merge; +DROP TABLE IF EXISTS payroll_alias; +DROP TABLE IF EXISTS payroll; + +SET use_statistics_for_part_pruning = 1; +SET materialize_statistics_on_insert = 1; +SET use_query_condition_cache = 0; +SET enable_parallel_replicas = 0; + +CREATE TABLE payroll (id UInt64, dept String, salary UInt64) +ENGINE = MergeTree ORDER BY id +SETTINGS auto_statistics_types = 'basic'; + +-- Visible rows have `salary` below 150, hidden rows have `salary` up to 1999. +INSERT INTO payroll SELECT number, if(number % 2 = 0, 'public', 'exec'), if(number % 2 = 0, 100 + number % 50, 1000 + number) FROM numbers(1000); + +CREATE TABLE payroll_merge (id UInt64, dept String, salary UInt64) ENGINE = Merge(currentDatabase(), '^payroll$'); +CREATE TABLE payroll_alias ENGINE = Alias('payroll'); + +-- Without a row policy the part is pruned when the predicate is out of the range of the statistics. +SELECT count() FROM payroll WHERE salary > 1998 SETTINGS log_comment = 'row_policy_probe_1_no_policy_1998'; +SELECT count() FROM payroll WHERE salary > 1999 SETTINGS log_comment = 'row_policy_probe_2_no_policy_1999'; + +CREATE ROW POLICY payroll_policy ON payroll FOR SELECT USING dept = 'public' TO CURRENT_USER; + +SELECT count() FROM payroll WHERE salary > 1998 SETTINGS log_comment = 'row_policy_probe_3_table_policy_1998'; +SELECT count() FROM payroll WHERE salary > 1999 SETTINGS log_comment = 'row_policy_probe_4_table_policy_1999'; + +SELECT count() FROM payroll_merge WHERE salary > 1998 SETTINGS log_comment = 'row_policy_probe_5_merge_child_policy_1998'; +SELECT count() FROM payroll_merge WHERE salary > 1999 SETTINGS log_comment = 'row_policy_probe_6_merge_child_policy_1999'; + +DROP ROW POLICY payroll_policy ON payroll; +CREATE ROW POLICY payroll_alias_policy ON payroll_alias FOR SELECT USING dept = 'public' TO CURRENT_USER; + +SELECT count() FROM payroll_alias WHERE salary > 1998 SETTINGS log_comment = 'row_policy_probe_7_alias_policy_1998', enable_analyzer = 1; +SELECT count() FROM payroll_alias WHERE salary > 1999 SETTINGS log_comment = 'row_policy_probe_8_alias_policy_1999', enable_analyzer = 1; + +SYSTEM FLUSH LOGS query_log; + +-- The number of rows read must not depend on the values of the hidden rows. +SELECT log_comment, read_rows +FROM system.query_log +WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment LIKE 'row\_policy\_probe\_%' +ORDER BY log_comment; + +DROP ROW POLICY payroll_alias_policy ON payroll_alias; +DROP TABLE payroll_merge; +DROP TABLE payroll_alias; +DROP TABLE payroll; From 5e64e9f373428efcc02c4e6c5d033a38530ddff2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Sep 2026 19:17:13 +0000 Subject: [PATCH 119/296] Backport #122668 to 26.8: Link with `--icf=safe`: `--icf=all` merges address-taken functions and crashes gRPC compression --- CMakeLists.txt | 5 ++-- ...05291_grpc_transport_compression.reference | 1 + .../05291_grpc_transport_compression.sh | 28 +++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05291_grpc_transport_compression.reference create mode 100755 tests/queries/0_stateless/05291_grpc_transport_compression.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 7860b6cd2ed8e..68b194ad629c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,7 +173,8 @@ if (OS_LINUX) if (LINKER_NAME MATCHES "lld" AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT WITH_COVERAGE) # Fold identical code and data sections to reduce binary size. - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--icf=all") + # `safe` keeps address-taken functions distinct, as C and C++ require. + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--icf=safe") endif () # In Release builds, strip residual debug sections that may come from contrib @@ -345,7 +346,7 @@ if (NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT WITH_COVERAGE) set(COMPILER_FLAGS "${COMPILER_FLAGS} -ffunction-sections -fdata-sections") # -faddrsig emits an address-significance table that lets lld's ICF # distinguish address-taken symbols from non-address-taken ones. - # Only useful on Linux where we use lld with --icf=all; Darwin's ld64 + # Only useful on Linux where we use lld with --icf=safe; Darwin's ld64 # does not consume these sections. if (OS_LINUX) set(COMPILER_FLAGS "${COMPILER_FLAGS} -faddrsig") diff --git a/tests/queries/0_stateless/05291_grpc_transport_compression.reference b/tests/queries/0_stateless/05291_grpc_transport_compression.reference new file mode 100644 index 0000000000000..08839f6bb296e --- /dev/null +++ b/tests/queries/0_stateless/05291_grpc_transport_compression.reference @@ -0,0 +1 @@ +200 diff --git a/tests/queries/0_stateless/05291_grpc_transport_compression.sh b/tests/queries/0_stateless/05291_grpc_transport_compression.sh new file mode 100755 index 0000000000000..b42c789d9bd1e --- /dev/null +++ b/tests/queries/0_stateless/05291_grpc_transport_compression.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: In fasttest, ENABLE_LIBRARIES=0, so the grpc library is not built + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# gRPC queries with gzip and deflate compression of both the request and the result. +# gRPC sends a message uncompressed if compression does not shrink it, so the query carries a long comment. +python3 - "$CURDIR/../../../utils/grpc-client/pb2" <<'EOF' +import sys +sys.path.insert(0, sys.argv[1]) +import grpc +import clickhouse_grpc_pb2 +import clickhouse_grpc_pb2_grpc + +padding = " -- " + "x" * 1000 +ok = 0 +for algorithm, compression in (("gzip", grpc.Compression.Gzip), ("deflate", grpc.Compression.Deflate)): + with grpc.insecure_channel("localhost:9100", compression=compression) as channel: + stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(channel) + for i in range(100): + result = stub.ExecuteQuery(clickhouse_grpc_pb2.QueryInfo( + query=f"SELECT {i}{padding}", transport_compression_type=algorithm, transport_compression_level=3)) + ok += result.output == f"{i}\n".encode() and not result.HasField("exception") +print(ok) +EOF From a5aa9b7e9c17c1185aea17f3d7c23cfb9d08d31a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 06:10:04 +0000 Subject: [PATCH 120/296] Backport #122451 to 26.8: Fix array and interval `plus`/`minus` producing a constant inside an `Array` or a `Tuple` --- src/Functions/FunctionBinaryArithmetic.h | 3 ++- src/Functions/vectorFunctions.cpp | 12 ++++++----- ...nstant_element_in_array_or_tuple.reference | 20 +++++++++++++++++++ ...tic_constant_element_in_array_or_tuple.sql | 15 ++++++++++++++ 4 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05258_arithmetic_constant_element_in_array_or_tuple.reference create mode 100644 tests/queries/0_stateless/05258_arithmetic_constant_element_in_array_or_tuple.sql diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 96adfe2a3e82f..206717bab661a 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -1867,7 +1867,8 @@ class FunctionBinaryArithmetic : public IFunction, WithContext ? array_element_function->executeImpl(new_arguments, result_array_type, rows_count) : executeImpl(new_arguments, result_array_type, rows_count); - return ColumnArray::create(res, typeid_cast(arguments[0].column.get())->getOffsetsPtr()); + /// The element-wise result can be a constant (for example a NULL), the data of an array cannot. + return ColumnArray::create(res->convertToFullColumnIfConst(), typeid_cast(arguments[0].column.get())->getOffsetsPtr()); } ColumnPtr executeArrayWithNumericImpl(const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp index 1df1d5757abc0..ff44f7aea8607 100644 --- a/src/Functions/vectorFunctions.cpp +++ b/src/Functions/vectorFunctions.cpp @@ -630,7 +630,7 @@ struct FunctionTupleOperationInterval final : public ITupleFunction else tuple_columns.resize(2); - tuple_columns[0] = arguments[0].column->convertToFullColumnIfConst(); + tuple_columns[0] = arguments[0].column; } else if (first_tuple) { @@ -675,15 +675,13 @@ struct FunctionTupleOperationInterval final : public ITupleFunction { auto minus = FunctionFactory::instance().get("minus", context); auto elem_minus = minus->build({left, arguments[1]}); - last_column = elem_minus->execute({left, arguments[1]}, arguments[1].type, input_rows_count, /* dry_run = */ false) - ->convertToFullColumnIfConst(); + last_column = elem_minus->execute({left, arguments[1]}, arguments[1].type, input_rows_count, /* dry_run = */ false); } else { auto plus = FunctionFactory::instance().get("plus", context); auto elem_plus = plus->build({left, arguments[1]}); - last_column = elem_plus->execute({left, arguments[1]}, arguments[1].type, input_rows_count, /* dry_run = */ false) - ->convertToFullColumnIfConst(); + last_column = elem_plus->execute({left, arguments[1]}, arguments[1].type, input_rows_count, /* dry_run = */ false); } } else @@ -700,6 +698,10 @@ struct FunctionTupleOperationInterval final : public ITupleFunction } } + /// Either operand can be a constant, which a tuple cannot hold. + for (auto & column : tuple_columns) + column = column->convertToFullColumnIfConst(); + return ColumnTuple::create(tuple_columns); } }; diff --git a/tests/queries/0_stateless/05258_arithmetic_constant_element_in_array_or_tuple.reference b/tests/queries/0_stateless/05258_arithmetic_constant_element_in_array_or_tuple.reference new file mode 100644 index 0000000000000..acea8230f1720 --- /dev/null +++ b/tests/queries/0_stateless/05258_arithmetic_constant_element_in_array_or_tuple.reference @@ -0,0 +1,20 @@ +[NULL] +[NULL] [null] +[NULL] [null] +[NULL,NULL] [] +[NULL,NULL] [] +[NULL] [NULL] +[NULL] [NULL] +[[NULL]] +[[NULL]] +[NULL] +[NULL] +[NULL] + +[] +(0,1) (0,-1) +(1,1) (1,-1) +(1,1) (1,1,0) +(1,2) (1,1,1) +2020-01-01 01:00:00 +2020-01-02 01:00:00 diff --git a/tests/queries/0_stateless/05258_arithmetic_constant_element_in_array_or_tuple.sql b/tests/queries/0_stateless/05258_arithmetic_constant_element_in_array_or_tuple.sql new file mode 100644 index 0000000000000..28fa181272a74 --- /dev/null +++ b/tests/queries/0_stateless/05258_arithmetic_constant_element_in_array_or_tuple.sql @@ -0,0 +1,15 @@ +-- Arithmetic must not produce a constant inside an array or a tuple: an array operation whose element-wise +-- result is a constant (a date plus a tuple holding NULL is NULL), and a constant interval added to a +-- non-constant interval of another unit. + +SELECT toString([toDateTime64(1, 3)] + [(2, NULL)]); +SELECT toString([toDateTime64(number, 3)] + [(2, NULL)]), toJSONString([toDateTime64(number, 3)] + [(2, NULL)]) FROM numbers(2); +SELECT arrayPushBack([toDateTime64(number, 3)] + [(2, NULL)], NULL), arrayDistinct([toDateTime64(number, 3)] + [(2, NULL)]) FROM numbers(2); +SELECT toString([toDate('2020-01-01') + number] - [(1, NULL)]), toString([(1, NULL)] + [toDateTime(number)]) FROM numbers(2); +SELECT toString([[toDateTime64(number, 3)]] + [[(2, NULL)]]) FROM numbers(2); +SELECT [toDateTime64(number, 3)] + [(2, NULL)] FROM numbers(1) UNION ALL SELECT materialize([NULL]); +SELECT arrayPartialShuffle([toDateTime64(1, 3)] + [(2, NULL)]) AS a GROUP BY a WITH TOTALS; + +SELECT toIntervalDay(number) + INTERVAL 1 HOUR, toIntervalDay(number) - INTERVAL 1 HOUR FROM numbers(2); +SELECT (INTERVAL 1 DAY, INTERVAL 1 HOUR) + toIntervalHour(number), (INTERVAL 1 DAY, INTERVAL 1 MONTH) + toIntervalHour(number) FROM numbers(2); +SELECT toDateTime('2020-01-01 00:00:00', 'UTC') + (toIntervalDay(number) + INTERVAL 1 HOUR) FROM numbers(2); From 5354187eea442a65ea90e6b8ae8e00d6d3e1543f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 07:11:31 +0000 Subject: [PATCH 121/296] Backport #114629 to 26.8: Fix DPsub join reordering silently dropping single-table ON-clause filters --- .../QueryPlan/Optimizations/joinOrder.cpp | 17 ++-- ...ub_single_table_filter_placement.reference | 12 +++ ...69_dpsub_single_table_filter_placement.sql | 86 +++++++++++++++++++ 3 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference create mode 100644 tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql diff --git a/src/Processors/QueryPlan/Optimizations/joinOrder.cpp b/src/Processors/QueryPlan/Optimizations/joinOrder.cpp index a7f654ac37ed7..0083567295b95 100644 --- a/src/Processors/QueryPlan/Optimizations/joinOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/joinOrder.cpp @@ -842,9 +842,10 @@ const std::vector & JoinOrderOptimizer::collectJoinEdgesMask(UI if (dpsub_data.edge_pinned[i] && (dpsub_data.edge_pin_mask[i] & ~joined)) continue; - /// Relations that must all be present before the predicate is applicable: the relations it - /// references (`sources`) plus any relations it is pinned to. For a plain equi-predicate the - /// pin is empty, so this is just `sources`. For a single-table conjunct of an outer join's ON + /// Works much like Extended Eligibility List (EEL) in case of outerjoins: + /// encoding relations that must be present for the predicate to be applicable (in `pin` mask) + /// For innerjoins its just the sources of the predicate, i.e., NEL, here pin is empty. + /// For a single-table conjunct of an outer join's ON /// clause (e.g. `t2.value = 'x'` in `... LEFT JOIN t3 ON t2.id = t3.id AND t2.value = 'x'`), /// `sources` is only `{t2}` but the pin is `{t3}`: the predicate belongs to the ON condition of /// the join that brings in `t3`, not to `t2` as a base-table filter. Placing it by `sources` @@ -856,17 +857,17 @@ const std::vector & JoinOrderOptimizer::collectJoinEdgesMask(UI if (std::popcount(applicable) <= 1) { - /// Base-relation filter or constant predicate: it becomes applicable as soon as its single - /// relation is present, so attach it at the earliest (two-relation) join to filter as low - /// as possible. - if (two_relations && (edge.fromLeft() || edge.fromRight() || edge.fromNone())) + /// Base-relation filter or constant predicate (the edge references at most one relation). + const bool relation_introduced = applicable != 0 && (left_mask == applicable || right_mask == applicable); + const bool constant_at_earliest_join = applicable == 0 && two_relations; + if (relation_introduced || constant_at_earliest_join) out.push_back(&edge); } else if ((applicable & ~left_mask) && (applicable & ~right_mask)) { /// The predicate spans the split (a connecting equi-predicate, or a single-table ON-clause /// conjunct pinned to the opposite side): neither side alone contains all the relations it - /// needs. This join is the lowest one that makes it applicable, so attach it here — into the + /// needs. This join is the lowest one that makes it applicable, so attach it here: into the /// correct join's ON condition. out.push_back(&edge); } diff --git a/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference new file mode 100644 index 0000000000000..3350bf0bdf334 --- /dev/null +++ b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference @@ -0,0 +1,12 @@ +reported greedy: +0 Join_1_Value_0 0 Join_2_Value_0 0 Join_3_Value_0 +reported dpsub: +0 Join_1_Value_0 0 Join_2_Value_0 0 Join_3_Value_0 +filter deep relation greedy: +0 Join_2_Value_0 0 Join_3_Value_0 0 Join_1_Value_0 +filter deep relation dpsub: +0 Join_2_Value_0 0 Join_3_Value_0 0 Join_1_Value_0 +four way filter last greedy: +0 0 0 0 Join_4_Value_0 +four way filter last dpsub: +0 0 0 0 Join_4_Value_0 diff --git a/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql new file mode 100644 index 0000000000000..b0657008aedb2 --- /dev/null +++ b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql @@ -0,0 +1,86 @@ +-- Tests that the DPsub join-order algorithm keeps single-table filter predicates that live in a +-- join's ON clause (e.g. `... JOIN t ON t.a = u.a AND t.b = 'x'`). DPsub attaches such a predicate +-- at the join that introduces its relation. Two earlier placement bugs silently dropped the +-- predicate, letting extra rows through: +-- 1. attaching only at two-relation joins dropped it when the relation was introduced against an +-- already-multi-relation subplan (e.g. `t1 JOIN (t2 JOIN t3)`); +-- 2. gating on fromLeft()/fromRight() dropped it for any filter on a relation whose id is >= 2, +-- because those helpers test relation ids 0 and 1 specifically. +-- For each shape we print the result with the 'greedy' algorithm (the reference, which places the +-- predicate correctly) and with 'dpsub'; the two must be identical row-for-row. +-- +-- `query_plan_enable_optimizations = 0` is required to expose the bug and does NOT disable join +-- reordering (that is controlled by `query_plan_optimize_join_order_algorithm`). It disables the +-- general plan-optimization passes, notably filter push-down; with push-down enabled the +-- single-table filter is applied independently of the join and masks the dropped ON-clause +-- conjunct, so the wrong-result would not surface. + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS t4; + +CREATE TABLE t1 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t2 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t3 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t4 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t1 VALUES (0, 'Join_1_Value_0'), (1, 'Join_1_Value_1'), (2, 'Join_1_Value_2'); +INSERT INTO t2 VALUES (0, 'Join_2_Value_0'), (1, 'Join_2_Value_1'), (3, 'Join_2_Value_3'); +INSERT INTO t3 VALUES (0, 'Join_3_Value_0'), (1, 'Join_3_Value_1'), (4, 'Join_3_Value_4'); +INSERT INTO t4 VALUES (0, 'Join_4_Value_0'), (1, 'Join_4_Value_1'), (5, 'Join_4_Value_5'); + +SET enable_analyzer = 1; +SET query_plan_optimize_join_order_limit = 10; + +-- The reported case: filter on the first relation, INNER then LEFT join. +SELECT 'reported greedy:'; +SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' +LEFT JOIN t3 ON t2.id = t3.id ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'reported dpsub:'; +SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' +LEFT JOIN t3 ON t2.id = t3.id ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +-- Filter on a relation that is introduced last, against an already-joined subplan (relation id >= 2). +SELECT 'filter deep relation greedy:'; +SELECT t2.id, t2.value, t3.id, t3.value, t1.id, t1.value +FROM t2 INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t1 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'filter deep relation dpsub:'; +SELECT t2.id, t2.value, t3.id, t3.value, t1.id, t1.value +FROM t2 INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t1 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +-- Four-way all-inner chain with the filter on the last relation (id = 3). +SELECT 'four way filter last greedy:'; +SELECT t1.id, t2.id, t3.id, t4.id, t4.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id +INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t4 ON t3.id = t4.id AND t4.value = 'Join_4_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'four way filter last dpsub:'; +SELECT t1.id, t2.id, t3.id, t4.id, t4.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id +INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t4 ON t3.id = t4.id AND t4.value = 'Join_4_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; From a126632cbde5ab200a22bf4c8bac315dc1a736d8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 07:21:06 +0000 Subject: [PATCH 122/296] Backport #117590 to 26.8: Fix wrong results from JIT-compiled float to integer and `Decimal` conversions --- src/DataTypes/Native.cpp | 2 + src/Functions/FunctionIfBase.h | 2 +- src/Functions/FunctionsConversion.cpp | 64 +++++++------- .../04072_jit_decimal_expressions.sql | 1 + .../04143_if_decimal_int_literal_jit.sql | 2 +- ...04205_jit_decimal_cast_branches_parity.sql | 8 +- ...40_jit_float_to_big_int_libcalls.reference | 16 +--- .../04240_jit_float_to_big_int_libcalls.sql | 41 +++++---- .../05055_jit_float_to_integer_cast.reference | 8 ++ .../05055_jit_float_to_integer_cast.sql | 88 +++++++++++++++++++ 10 files changed, 162 insertions(+), 70 deletions(-) create mode 100644 tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference create mode 100644 tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql diff --git a/src/DataTypes/Native.cpp b/src/DataTypes/Native.cpp index 3784b161efefd..6c670eafe12b5 100644 --- a/src/DataTypes/Native.cpp +++ b/src/DataTypes/Native.cpp @@ -311,6 +311,8 @@ llvm::Value * nativeCastWithDecimalScale( } if (from_w.isFloat32() || from_w.isFloat64()) { + /// A float source must not reach here: `fptosi` has no defined result outside the destination range. + chassert(false, "Float to Decimal must not be JIT-compiled"); /// Float → `Decimal`: multiply by `10^to_scale` in floating point first, /// then truncate to the target integer storage type. if (to_scale == 0) diff --git a/src/Functions/FunctionIfBase.h b/src/Functions/FunctionIfBase.h index e80cd3b756943..6ec99c2097604 100644 --- a/src/Functions/FunctionIfBase.h +++ b/src/Functions/FunctionIfBase.h @@ -64,7 +64,7 @@ class FunctionIfBase : public IFunction b.CreateCondBr(nativeBoolCast(b, cond), then, next); b.SetInsertPoint(then); - /// Use `nativeCastWithDecimalScale` to correctly lift integer/float branches to a + /// Use `nativeCastWithDecimalScale` to correctly lift integer branches to a /// `Decimal` `result_type` (and to convert between `Decimal` types of different scales). /// Plain `nativeCast` reinterprets the integer bits without applying the `10^scale` /// factor, which silently produces wrong values when the analyzer leaves a non-`Decimal` diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 726781d7ea8c4..d20712d3f7383 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -3351,6 +3351,34 @@ bool castBothTypes(const IDataType * left, const IDataType * right, F && f) return castType(left, [&](const auto & left_) { return castType(right, [&](const auto & right_) { return f(left_, right_); }); }); } +/// Whether a numeric conversion `from` -> `to` can be JIT-compiled. A float source is refused for an +/// integer or `Decimal` destination, because `fptosi` / `fptoui` have no defined result outside the +/// destination range. A `Bool` destination stays allowed, it is compiled through `nativeBoolCast`. +static bool isCompilableNumericConversion(const IDataType * from, const IDataType * to) +{ + return castBothTypes(from, to, [](const auto & left, const auto & right) + { + using LeftDataType = std::decay_t; + using RightDataType = std::decay_t; + + if constexpr (IsDataTypeDecimalOrNumber && IsDataTypeDecimalOrNumber) + { + if constexpr (IsDataTypeNumber && IsDataTypeNumber) + { + if constexpr (is_floating_point + && !is_floating_point) + return isBool(right.getPtr()); + return true; + } + else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) + return !is_floating_point; + else if constexpr (IsDataTypeDecimal && IsDataTypeNumber) + return true; + } + return false; + }); +} + bool convertIsCompilableImpl(const DataTypes & types, const DataTypePtr & result_type) { if (types.empty()) @@ -3359,25 +3387,7 @@ bool convertIsCompilableImpl(const DataTypes & types, const DataTypePtr & result if (!canBeNativeType(types[0]) || !canBeNativeType(result_type)) return false; - return castBothTypes( - types[0].get(), - result_type.get(), - [](const auto & left, const auto & right) - { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - - if constexpr (IsDataTypeDecimalOrNumber && IsDataTypeDecimalOrNumber) - { - if constexpr (IsDataTypeNumber && IsDataTypeNumber) - return true; - else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) - return true; - else if constexpr (IsDataTypeDecimal && IsDataTypeNumber) - return true; - } - return false; - }); + return isCompilableNumericConversion(types[0].get(), result_type.get()); } llvm::Value * convertCompileImpl(llvm::IRBuilderBase & builder, const ValuesWithType & arguments, const DataTypePtr & result_type) @@ -3479,21 +3489,7 @@ bool FunctionCast::isCompilable() const if (!canBeNativeType(denull_input_type) || !canBeNativeType(denull_result_type)) return false; - return castBothTypes(denull_input_type.get(), denull_result_type.get(), [](const auto & left, const auto & right) - { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - if constexpr (IsDataTypeDecimalOrNumber && IsDataTypeDecimalOrNumber) - { - if constexpr (IsDataTypeNumber && IsDataTypeNumber) - return true; - else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) - return true; - else if constexpr (IsDataTypeDecimal && IsDataTypeNumber) - return true; - } - return false; - }); + return isCompilableNumericConversion(denull_input_type.get(), denull_result_type.get()); } llvm::Value * FunctionCast::compile(llvm::IRBuilderBase & builder, const ValuesWithType & arguments) const diff --git a/tests/queries/0_stateless/04072_jit_decimal_expressions.sql b/tests/queries/0_stateless/04072_jit_decimal_expressions.sql index e735f68165d7b..eda70438d6ae3 100644 --- a/tests/queries/0_stateless/04072_jit_decimal_expressions.sql +++ b/tests/queries/0_stateless/04072_jit_decimal_expressions.sql @@ -34,6 +34,7 @@ SELECT toFloat32(d32), toFloat64(d64) FROM test_jit_dec_expr ORDER BY d32; SELECT 'Test integer to Decimal conversions'; SELECT toDecimal32(i64, 2), toDecimal64(i64, 4) FROM test_jit_dec_expr ORDER BY i64; +-- The `Float ->` direction is evaluated by the interpreter: see #117442. SELECT 'Test float to Decimal conversions'; SELECT toDecimal64(f64, 4) FROM test_jit_dec_expr ORDER BY f64; diff --git a/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql b/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql index 0900b1e0533e2..d54a37585d5a1 100644 --- a/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql +++ b/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql @@ -45,7 +45,7 @@ WITH materialize(3::Decimal(38, 30)) AS r, materialize(2) AS k SELECT sum(multiI WITH materialize(1::Decimal(76, 60)) AS r, materialize(1) AS k SELECT sum(if(k != 1, r, 1)); WITH materialize(1::Decimal(76, 73)) AS r, materialize(1) AS k SELECT sum(if(k != 1, r, 1)); --- `Float` -> `Decimal` branch lift (the analyzer promotes the result to `Decimal`). +-- A float branch is never lifted: the result is `Variant(Decimal(18, 4), Float64)`, which is not compiled. WITH materialize(toDecimal64(1.5, 4)) AS r, materialize(1) AS k SELECT if(k != 1, r, 2.5::Float64); -- Sanity check: the non-JIT path produces the same answers. diff --git a/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql b/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql index ca1d5926f8309..ac79ea0cb10b5 100644 --- a/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql +++ b/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql @@ -40,11 +40,9 @@ -- the analyzer promotes the result type to `Variant` because -- `use_variant_as_common_type = 1` by default, and `canBeNativeType` -- excludes `Variant`. `FunctionIfBase::isCompilableImpl` returns `false` --- and the helper is never called. The helper still has correct code paths --- for these (the `pow10_fp_const` helper avoids 64-bit narrowing via --- `APFloat::convertFromAPInt` so high-bit-width `Decimal128`/`Decimal256` --- factors stay accurate) so it is forward-compatible if the analyzer ever --- stops promoting to `Variant`. +-- and the helper is never called. `Float` -> `Decimal` is asserted against +-- rather than supported: an out-of-range float to integer conversion has no +-- defined result once compiled. -- -- * `Decimal` -> `Decimal` with different scales (scale increase OR decrease): -- `FunctionIf::executeImpl` rejects this with `NOT_IMPLEMENTED: diff --git a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference index 67a5306eb758e..102257aae8b59 100644 --- a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference +++ b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference @@ -1,18 +1,5 @@ --- JIT --- ---- Float -> 128-bit / 256-bit integers --- -2 -2 -2 -2 -2 -2 -2 -2 ---- 128-bit / 256-bit integers -> Float --- -2 -2 -2 -2 +--- 128-bit integers -> Float --- 2 2 2 @@ -36,3 +23,4 @@ 2 2 2 +1 diff --git a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql index 54923031e8c2c..71acd02410deb 100644 --- a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql +++ b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql @@ -6,25 +6,15 @@ SELECT '--- JIT ---'; SET compile_expressions = 1, min_count_to_compile_expression = 0; -SELECT '--- Float -> 128-bit / 256-bit integers ---'; -SELECT toInt128 (materialize(1.5) + materialize(0.5)); -SELECT toUInt128(materialize(1.5) + materialize(0.5)); -SELECT toInt256 (materialize(1.5) + materialize(0.5)); -SELECT toUInt256(materialize(1.5) + materialize(0.5)); -SELECT toInt128 (materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toUInt128(materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toInt256 (materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toUInt256(materialize(1.5)::Float32 + materialize(0.5)::Float32); - -SELECT '--- 128-bit / 256-bit integers -> Float ---'; +-- Only the 128-bit integer to float direction is compiled, and it reaches `__floattisf` and +-- `__floattidf`. A float source with an integer destination is declined, and a 256-bit value is not +-- a native JIT type, so both have an interpreted arm only: the first is covered by +-- `05055_jit_float_to_integer_cast`, the second by the block below. +SELECT '--- 128-bit integers -> Float ---'; SELECT toFloat32(materialize(2::Int128) + materialize(0::Int128)); SELECT toFloat64(materialize(2::Int128) + materialize(0::Int128)); SELECT toFloat32(materialize(2::UInt128) + materialize(0::UInt128)); SELECT toFloat64(materialize(2::UInt128) + materialize(0::UInt128)); -SELECT toFloat32(materialize(2::Int256) + materialize(0::Int256)); -SELECT toFloat64(materialize(2::Int256) + materialize(0::Int256)); -SELECT toFloat32(materialize(2::UInt256) + materialize(0::UInt256)); -SELECT toFloat64(materialize(2::UInt256) + materialize(0::UInt256)); SELECT '--- no JIT ---'; SET compile_expressions = 0; @@ -48,3 +38,24 @@ SELECT toFloat32(materialize(2::Int256) + materialize(0::Int256)); SELECT toFloat64(materialize(2::Int256) + materialize(0::Int256)); SELECT toFloat32(materialize(2::UInt256) + materialize(0::UInt256)); SELECT toFloat64(materialize(2::UInt256) + materialize(0::UInt256)); + +-- Every row above compares a compiled value against an interpreted one, so all of them would still +-- pass if the JIT arm stopped compiling. This pins that the 128-bit conversion is compiled wherever +-- the control is, which also holds in a build without the embedded compiler. The control is plain +-- arithmetic, so it stays at 1 even if every conversion stops being compilable. +SELECT toFloat64(materialize(2::Int128) + materialize(0::Int128)) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '04240_int128' FORMAT Null; +SELECT materialize(2.0) + materialize(0.0) + materialize(1.0) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '04240_control' FORMAT Null; + +SYSTEM FLUSH LOGS query_log; + +WITH shapes AS +( + SELECT log_comment, argMax(ProfileEvents['CompiledFunctionExecute'] > 0, event_time_microseconds) AS compiled + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment LIKE '04240_%' + GROUP BY log_comment +) +SELECT (SELECT compiled FROM shapes WHERE log_comment = '04240_int128') + = (SELECT compiled FROM shapes WHERE log_comment = '04240_control'); diff --git a/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference new file mode 100644 index 0000000000000..1c4357e61ffd8 --- /dev/null +++ b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference @@ -0,0 +1,8 @@ +230 1 230 +250 1 250 +true true +true true +true true +true true +0 +1 1 1 diff --git a/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql new file mode 100644 index 0000000000000..26e66cc4e70c8 --- /dev/null +++ b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql @@ -0,0 +1,88 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/117442 +SET compile_expressions = 1; +SET min_count_to_compile_expression = 0; + +DROP TABLE IF EXISTS t_jit_float_cast; +CREATE TABLE t_jit_float_cast (c0 UInt8) ENGINE = Memory; +INSERT INTO t_jit_float_cast VALUES (230), (250); + +-- In-range conversions are exact and identical compiled or interpreted. +SELECT c0, + toFloat64(c0) / 10 <= CAST(toFloat64(c0) AS UInt8) AS in_range_by_cast, + CAST(toFloat64(c0) AS Decimal32(2)) AS in_range_decimal +FROM t_jit_float_cast +ORDER BY c0; + +-- A value the destination cannot hold raises. The non-finite one is built from the column so that +-- it is not constant folded before execution. +SELECT CAST(-toFloat64(c0) * 1e9 AS Decimal32(2)) FROM t_jit_float_cast; -- { serverError DECIMAL_OVERFLOW } +SELECT toDecimal32(-toFloat64(c0) * 1e9, 2) FROM t_jit_float_cast; -- { serverError DECIMAL_OVERFLOW } +SELECT toUInt8(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0))) FROM t_jit_float_cast; -- { serverError CANNOT_CONVERT_TYPE } +SELECT CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS UInt8) FROM t_jit_float_cast; -- { serverError CANNOT_CONVERT_TYPE } + +-- `Bool` is the one float to integer destination that stays compiled, because it is lowered as +-- `value != 0` rather than as a conversion, which is exact for every value. +SELECT CAST(-toFloat64(c0) * 1e30 AS Bool), + CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS Bool) +FROM t_jit_float_cast +ORDER BY c0; + +SET compile_expressions = 0; +SELECT CAST(-toFloat64(c0) * 1e30 AS Bool), + CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS Bool) +FROM t_jit_float_cast +ORDER BY c0; +SET compile_expressions = 1, min_count_to_compile_expression = 0; + +-- The value an out-of-range float converts to is not defined by the language, so compare the compiled +-- and the interpreted evaluation of the same expression instead of pinning a literal. +CREATE TABLE t_jit_float_cast_arms (c0 UInt8, lte UInt8) ENGINE = Memory; + +INSERT INTO t_jit_float_cast_arms +SELECT c0, toFloat64(c0) / 10 <= CAST(-toFloat64(c0) AS UInt8) FROM t_jit_float_cast; + +SET compile_expressions = 0; +INSERT INTO t_jit_float_cast_arms +SELECT c0, toFloat64(c0) / 10 <= CAST(-toFloat64(c0) AS UInt8) FROM t_jit_float_cast; +SET compile_expressions = 1, min_count_to_compile_expression = 0; + +SELECT count() FROM (SELECT c0 FROM t_jit_float_cast_arms GROUP BY c0 HAVING uniqExact(lte) > 1); + +DROP TABLE t_jit_float_cast_arms; + +-- Every row above is a value oracle, so all of them would still pass if the conversions silently +-- stopped or started being compiled. The shapes below pin which of them compiles. Each has one +-- compilable child, so once its `CAST` is declined nothing is left to compile: a declined shape +-- reaches zero even where the control compiles. `CompiledFunctionExecute` counts executions of an +-- already-compiled node, so a warm compiled cache does not change any of them. +SELECT CAST(toFloat64(number) AS Bool) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_bool' FORMAT Null; +SELECT CAST(toFloat64(number) AS UInt8) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_declined' FORMAT Null; +SELECT toFloat64(number) + 1 FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_control' FORMAT Null; +-- The reported shape: a comparison whose right side is a declined conversion. The conversion becomes +-- an input to the compiled expression, so the comparison around it must still compile. Neither side +-- can compile on its own, so the counter here belongs to the comparison and to nothing else. +SELECT toFloat64(number) <= CAST(toFloat64(number) AS UInt8) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_parent' FORMAT Null; + +SYSTEM FLUSH LOGS query_log; + +WITH shapes AS +( + SELECT log_comment, argMax(ProfileEvents['CompiledFunctionExecute'] > 0, event_time_microseconds) AS compiled + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment LIKE '05055_%' + GROUP BY log_comment +) +-- The control keeps the first column honest in a build without the embedded compiler, where every +-- shape is interpreted and an absolute assertion would go green on nothing being compiled. +SELECT + (SELECT compiled FROM shapes WHERE log_comment = '05055_bool') + = (SELECT compiled FROM shapes WHERE log_comment = '05055_control'), + (SELECT compiled FROM shapes WHERE log_comment = '05055_declined') = 0, + (SELECT compiled FROM shapes WHERE log_comment = '05055_parent') + = (SELECT compiled FROM shapes WHERE log_comment = '05055_control'); + +DROP TABLE t_jit_float_cast; From 801e55472b925a983ff727089ec166f2da2a34ac Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 07:21:44 +0000 Subject: [PATCH 123/296] Backport #114521 to 26.8: Fix Iceberg query failure after MODIFY COLUMN to Nullable --- .../DataLakes/Iceberg/SchemaProcessor.cpp | 10 +- ..._prewhere_modify_column_nullable.reference | 39 +++++ ...iceberg_prewhere_modify_column_nullable.sh | 153 ++++++++++++++++++ 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference create mode 100755 tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index 51583a11987fd..5eaaaaf079755 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -813,7 +813,15 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( /// a whitespace-only difference is the same type and needs only a rename, not a cast. if (canonicalizeTypeSpacing(old_type) == canonicalizeTypeSpacing(new_type)) { - if (old_json->getValue(f_name) != name) + /// Nullability is carried by the separate `required` key, so equal type strings + /// can still resolve to different types. Only relaxing required to optional is + /// legal evolution; the reverse keeps the plain passthrough. + const bool old_required = old_json->getValue(f_required); + if (old_required && !required && !old_node->result_type->equals(*type)) + { + node = &dag->addCast(*old_node, type, name, nullptr); + } + else if (old_json->getValue(f_name) != name) { node = &dag->addAlias(*old_node, name); } diff --git a/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference new file mode 100644 index 0000000000000..1a595af2079a5 --- /dev/null +++ b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference @@ -0,0 +1,39 @@ +--- WHERE on the evolved column --- +4 4 +5 five +--- PREWHERE on the evolved column --- +4 4 +5 five +--- PREWHERE IS NULL / IS NOT NULL --- +none +6 +--- PREWHERE on an untouched column --- +4 4 +--- declared type and full scan --- +7 2 Nullable(Int64) +0 0 +1 1 +2 2 +3 3 +4 4 +5 five +\N none +--- MODIFY COLUMN to Nullable plus RENAME COLUMN --- +4 4 Nullable(Int64) +5 five Nullable(Int64) +--- widening plus nullability (int required to long optional) --- +4 Nullable(Int64) +--- optional to required is still rejected --- +Iceberg spec doesn't allow change type from nullable to non-nullable +--- String column made Nullable --- +4 Nullable(String) +--- externally authored optional to required stays a passthrough --- +id Int64 +s String +none +one +three +none +one +three +three diff --git a/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh new file mode 100755 index 0000000000000..064a0c290e549 --- /dev/null +++ b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel-replicas +# `no-parallel-replicas`: see comment in `04071_iceberg_orc_prewhere_crash.sh`. +# `StorageObjectStorageCluster` (used when `parallel_replicas_for_cluster_engines = 1`, +# default) does not delegate `supportsPrewhere` to its underlying configuration. +# +# Regression test for issue #85029: filtering a column that `ALTER TABLE ... MODIFY COLUMN` +# made `Nullable` fails on the Iceberg data files written before the `ALTER`. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# `optimize_move_to_prewhere=1` + `query_plan_optimize_prewhere=1` are pinned on every +# discriminating statement: the failure only appears when the predicate is pushed into the +# reader, the runner randomizes both, and with either one off the pre-fix result is already +# correct, so the test would stop exercising the fix. +PREWHERE_SETTINGS="--optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1" + +TABLE="t_null_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_REN="t_ren_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_WID="t_wid_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_REJ="t_rej_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_STR="t_str_${CLICKHOUSE_DATABASE}_${RANDOM}" + +drop_table() { + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $1" + rm -rf "${USER_FILES_PATH}/$1/" +} + +# Rows 0..4 are written while `id` is still required, rows 5 and NULL after the `ALTER`, so the +# table mixes pre- and post-evolution data files. Only the pre-evolution ones carry the defect. +create_mixed_nullability_table() { + local table="$1" + local table_path="${USER_FILES_PATH}/${table}/" + rm -rf "${table_path}" + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${table}" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${table} (id Int64, s String) ENGINE = IcebergLocal('${table_path}', 'Parquet')" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${table} SELECT number, toString(number) FROM numbers(5)" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${table} MODIFY COLUMN id Nullable(Int64)" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${table} SELECT * FROM values('id Nullable(Int64), s String', (5, 'five'), (NULL, 'none'))" +} + +create_mixed_nullability_table "${TABLE}" + +echo "--- WHERE on the evolved column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} WHERE id > 3 ORDER BY id" + +# Explicit PREWHERE probes the reader-side path directly, without depending on the +# WHERE->PREWHERE mover. +echo "--- PREWHERE on the evolved column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} PREWHERE id > 3 ORDER BY id" + +echo "--- PREWHERE IS NULL / IS NOT NULL ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT s FROM ${TABLE} PREWHERE id IS NULL ORDER BY s" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT count() FROM ${TABLE} PREWHERE id IS NOT NULL" + +echo "--- PREWHERE on an untouched column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} PREWHERE s = '4' ORDER BY id" + +echo "--- declared type and full scan ---" +${CLICKHOUSE_CLIENT} --query "SELECT count(), countIf(id > 3), toTypeName(any(id)) FROM ${TABLE}" +${CLICKHOUSE_CLIENT} --query "SELECT id, s FROM ${TABLE} ORDER BY id, s" + +# Composition with a rename: the transform must apply the new name and the new nullability in one +# node. Before the fix the rename branch renamed the column and dropped the type change. +echo "--- MODIFY COLUMN to Nullable plus RENAME COLUMN ---" +create_mixed_nullability_table "${TABLE_REN}" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_REN} RENAME COLUMN id TO idx" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT idx, s, toTypeName(idx) FROM ${TABLE_REN} PREWHERE idx > 3 ORDER BY idx" + +# Widening and nullability at once already took the type-conversion branch and was already +# correct; assert it stays correct. +echo "--- widening plus nullability (int required to long optional) ---" +rm -rf "${USER_FILES_PATH}/${TABLE_WID}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_WID} (id Int32, s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_WID}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_WID} SELECT toInt32(number), toString(number) FROM numbers(5)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_WID} MODIFY COLUMN id Nullable(Int64)" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, toTypeName(id) FROM ${TABLE_WID} PREWHERE id > 3 ORDER BY id" + +# The reverse direction is not legal evolution and must keep being rejected. +echo "--- optional to required is still rejected ---" +rm -rf "${USER_FILES_PATH}/${TABLE_REJ}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_REJ} (id Nullable(Int64), s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_REJ}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_REJ} SELECT number, toString(number) FROM numbers(3)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_REJ} MODIFY COLUMN id Int64" 2>&1 \ + | grep -oF "Iceberg spec doesn't allow change type from nullable to non-nullable" | head -1 + +# A non-numeric type reaches the same branch through a different comparison function. +echo "--- String column made Nullable ---" +rm -rf "${USER_FILES_PATH}/${TABLE_STR}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_STR} (v String, s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_STR}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_STR} SELECT toString(number), toString(number) FROM numbers(5)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_STR} MODIFY COLUMN v Nullable(String)" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT v, toTypeName(v) FROM ${TABLE_STR} PREWHERE v = '4' ORDER BY v" + +# The reverse direction (optional -> required) cannot be produced by `ALTER`, which rejects it +# above, so the passthrough branch is reached only through metadata written by another engine. +# Appending a schema leaves schema 0 byte-identical, so the schema-id immutability check still +# passes, and the read selects the pair with `iceberg_metadata_file_path`. +echo "--- externally authored optional to required stays a passthrough ---" +TABLE_REV="t_rev_${CLICKHOUSE_DATABASE}_${RANDOM}" +REV_PATH="${USER_FILES_PATH}/${TABLE_REV}/" +rm -rf "${REV_PATH}" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_REV} (id Nullable(Int64), s String) ENGINE = IcebergLocal('${REV_PATH}', 'Parquet')" +# The NULL row is what makes the assertion non-vacuous: a cast to the required type only +# misbehaves when a NULL actually has to pass through it. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_REV} SELECT * FROM values('id Nullable(Int64), s String', (1, 'one'), (NULL, 'none'), (3, 'three'))" + +LATEST_REV=$(ls "${REV_PATH}metadata/" | grep -E '^v[0-9]+\.metadata\.json$' | sort -t v -k2 -n | tail -1) +REV_META=$(python3 - "${REV_PATH}metadata" "${LATEST_REV}" <<'PYEOF' +import copy, json, os, re, sys + +metadata_dir, latest_file = sys.argv[1], sys.argv[2] + +with open(os.path.join(metadata_dir, latest_file)) as fh: + metadata = json.load(fh) + +current = next(s for s in metadata["schemas"] if s["schema-id"] == metadata["current-schema-id"]) +tightened = copy.deepcopy(current) +tightened["schema-id"] = max(s["schema-id"] for s in metadata["schemas"]) + 1 +for field in tightened["fields"]: + if field["name"] == "id": + field["required"] = True + +metadata["schemas"].append(tightened) +metadata["current-schema-id"] = tightened["schema-id"] +metadata["last-updated-ms"] = metadata.get("last-updated-ms", 0) + 60000 + +version = int(re.match(r"v(\d+)\.metadata\.json", latest_file).group(1)) + 1 +tmp_file = os.path.join(metadata_dir, ".tmp_next") +with open(tmp_file, "w") as fh: + json.dump(metadata, fh) +os.rename(tmp_file, os.path.join(metadata_dir, f"v{version}.metadata.json")) +print(f"metadata/v{version}.metadata.json") +PYEOF +) + +REV_TF="icebergLocal('${REV_PATH}', 'Parquet', SETTINGS iceberg_metadata_file_path = '${REV_META}')" +# The tightened schema is what the reader resolves against. +${CLICKHOUSE_CLIENT} --query "DESCRIBE ${REV_TF}" | cut -f1,2 +# Each of these three shapes fails if the transform casts the old optional column to the new +# required type instead of passing it through. +${CLICKHOUSE_CLIENT} --query "SELECT s FROM ${REV_TF} ORDER BY s" +${CLICKHOUSE_CLIENT} --query "SELECT s FROM ${REV_TF} WHERE id IS NOT NULL ORDER BY s" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT s FROM ${REV_TF} PREWHERE id > 1 ORDER BY s" + +drop_table "${TABLE}" +drop_table "${TABLE_REN}" +drop_table "${TABLE_WID}" +drop_table "${TABLE_REJ}" +drop_table "${TABLE_STR}" +drop_table "${TABLE_REV}" From 99207640e4ae5fad2c59a6597d6a15dc3f08856f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 09:07:13 +0000 Subject: [PATCH 124/296] Backport #121132 to 26.8: Add the `http_x_clickhouse_format_overrides_output_format` compatibility setting --- docs/concepts/features/interfaces/http.mdx | 4 +- src/Access/SettingsConstraints.cpp | 4 ++ src/Core/Settings.cpp | 11 ++++ src/Core/SettingsChangesHistory.cpp | 1 + src/Server/HTTPHandler.cpp | 43 +++++++++---- ..._http_x_clickhouse_format_compat.reference | 34 +++++++++++ .../05233_http_x_clickhouse_format_compat.sh | 61 +++++++++++++++++++ 7 files changed, 144 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/05233_http_x_clickhouse_format_compat.reference create mode 100755 tests/queries/0_stateless/05233_http_x_clickhouse_format_compat.sh diff --git a/docs/concepts/features/interfaces/http.mdx b/docs/concepts/features/interfaces/http.mdx index ff580401ef9c2..dbadd8c4a7c99 100644 --- a/docs/concepts/features/interfaces/http.mdx +++ b/docs/concepts/features/interfaces/http.mdx @@ -190,7 +190,7 @@ wget -nv -O- 'http://localhost:8123/?query=SELECT 1, 2, 3 FORMAT JSON' } ``` -You can use the `default_format` URL parameter to specify a default format other than `TabSeparated`. The `X-ClickHouse-Format` header selects the format of the response explicitly: it is an alias for the `output_format` setting, so it also overrides a `FORMAT` clause in the query. It never changes how the request body of an `INSERT` is parsed — use `input_format` or `format` for that. +You can use the `default_format` URL parameter to specify a default format other than `TabSeparated`. The `X-ClickHouse-Format` header selects the format of the response explicitly: it is an alias for the `output_format` setting, so it also overrides a `FORMAT` clause in the query. It never changes how the request body of an `INSERT` is parsed — use `input_format` or `format` for that. To restore the behavior of versions before 26.8, where the header was an alias for `default_format` and did not override a `FORMAT` clause, disable the `http_x_clickhouse_format_overrides_output_format` setting (in a user profile or as a URL parameter; like the format settings themselves, it can be changed in read-only mode). ```bash $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @- @@ -316,7 +316,7 @@ INSERT INTO t SELECT number FROM numbers(10) SETTINGS limit = 2; | Setting | Effect | |---------|--------| -| `output_format` | Overrides the output format. It takes precedence over the query's `FORMAT` clause, the path extension, `format`, and `default_format`. It can also be set using the `X-ClickHouse-Format` header. | +| `output_format` | Overrides the output format. It takes precedence over the query's `FORMAT` clause, the path extension, `format`, and `default_format`. It can also be set using the `X-ClickHouse-Format` header. With `http_x_clickhouse_format_overrides_output_format = 0`, the header sets `default_format` instead, as in versions before 26.8. | | `input_format` | Overrides the input format for `INSERT`. It takes precedence over the query's `FORMAT` clause and `format`. | | `format` | Overrides the format in both directions unless a direction-specific setting is provided. | | `default_format` | Sets the output format when the query has no `FORMAT` clause, no path extension, and no other format override. | diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index fd68f0c6da25e..fcf380632efef 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -77,6 +77,10 @@ bool isAlwaysChangeableInReadonly(std::string_view name) /// HTTP routing / session. if (name == "database" || name == "default_format") return true; + /// Selects which of `output_format` / `default_format` the `X-ClickHouse-Format` header aliases; + /// both targets are changeable here, so the switch between them must be too. + if (name == "http_x_clickhouse_format_overrides_output_format") + return true; /// Output format selection and response compression. if (name == "format" || name == "input_format" || name == "output_format" || name == "compression") return true; diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 9b4f663805082..fa401792157dc 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7070,6 +7070,17 @@ If enabled, any URL parameter not recognized as a known parameter, setting, or ` - A plain `name=value` becomes the equality `` `name` = 'value' `` (the identifier is back-quoted, the value is quoted as a string literal). - A comparison operator (`!=`, `>`, `<`, `>=`, `<=`, `<>`) makes it a comparison: either split across the parameter (`?a!=2`, `?a>=2`) or written inline when the URL has no `=` to split on (`?a<>2`, `?f(x)>3`), in which case the reassembled `name[=value]` is parsed as a full SQL expression. +)", 0) \ + DECLARE(Bool, http_x_clickhouse_format_overrides_output_format, true, R"( +Controls which setting the `X-ClickHouse-Format` HTTP header maps to. + +If enabled (the default), the header is an alias for the `output_format` setting: it is an explicit override of the response format that wins over the `FORMAT` clause in the query and over the file extension in the URL path. + +If disabled, the header is an alias for the `default_format` setting, as it was before version 26.8: it only selects the format used when the query has no `FORMAT` clause and no other format override is applied. + +In both cases the header overrides the URL parameter of the same name (`output_format` or `default_format`, respectively) and never changes how the request body of an `INSERT` is parsed. + +This is a compatibility setting for the HTTP interface: the header is consumed before the query is parsed, so it must be supplied via a URL parameter or a user profile, not via an in-query `SETTINGS` clause. Like `output_format` and `default_format` themselves, it can always be changed in read-only mode (`readonly = 1`), so a read-only user can pass it as a URL parameter. )", 0) \ \ DECLARE(UInt64, function_range_max_elements_in_block, 500000000, R"( diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 54044bf6c4413..7ae4b8ec11d4f 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -129,6 +129,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"http_allow_table_as_file", false, false, "New setting to recognize a table name in the URL path of HTTP requests, with optional format/compression extensions."}, {"http_allow_filters_as_path", false, false, "New setting to recognize hive-style `name=value` filters in the URL path of HTTP requests."}, {"http_allow_filters_as_unrecognized_url_parameters", false, false, "New setting to treat unrecognized URL parameters as filter expressions in HTTP requests."}, + {"http_x_clickhouse_format_overrides_output_format", false, true, "Controls whether the `X-ClickHouse-Format` HTTP header is an alias for `output_format` (overriding the query's `FORMAT` clause), as since 26.8, or for `default_format`, as before 26.8. The entry sits in the 26.8 block because that is the release whose behavior it restores, so `compatibility` with a version before 26.8 brings back the old header behavior."}, {"ignore_on_cluster_for_replicated_handler_queries", false, false, "New setting to ignore the ON CLUSTER clause for handler management queries when handlers are backed by replicated (Keeper) storage."}, {"materialized_views_populate_atomically", false, true, "New setting that makes plain `CREATE MATERIALIZED VIEW ... POPULATE` locally atomic: existing data is snapshotted and the view is subscribed to new inserts together, under a brief exclusive lock on the source, so rows inserted through the same server are neither missed nor duplicated. The guarantee covers the local insert path only - inserts arriving on another replica or through a distributed write are outside the cut - and it requires a source that can provide a pinned snapshot (the `MergeTree` family and `Memory`); other sources, as well as `CREATE OR REPLACE` / `REPLACE`, keep the legacy non-atomic population. Set to `false` for the legacy non-atomic behavior everywhere."}, {"input_format_json_max_object_size", 512 * 1024 * 1024, 512 * 1024 * 1024, "New setting to limit the maximum size of a single JSON object in bytes"}, diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 3637fe61d4f3a..df830316d51b3 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -94,6 +94,7 @@ namespace Setting extern const SettingsBool http_allow_table_as_file; extern const SettingsBool http_allow_filters_as_path; extern const SettingsBool http_allow_filters_as_unrecognized_url_parameters; + extern const SettingsBool http_x_clickhouse_format_overrides_output_format; extern const SettingsString compression; extern const SettingsString filter; extern const SettingsString format; @@ -393,20 +394,15 @@ void HTTPHandler::processQuery( deferred_unrecognized_params.emplace_back(key, value); } - /// The `X-ClickHouse-Database` header is an alias for the `database` setting, and - /// `X-ClickHouse-Format` is an alias for the `output_format` setting. They override any matching - /// URL parameter (preserving the historical precedence). - /// - /// `X-ClickHouse-Format` maps to `output_format` rather than to `default_format`: sending this - /// header means the client definitely wants the response in that format, so it is an explicit - /// override (winning over the query's `FORMAT` clause and the path extension), not a fallback - /// used only when nothing else selects a format. It maps to `output_format` and not to the - /// bidirectional `format`, because the header has always described the response only: the same - /// header on `INSERT INTO t FORMAT JSONEachRow …` must not reinterpret the request body. + /// The `X-ClickHouse-Database` header is an alias for the `database` setting. It overrides a + /// matching URL parameter (preserving the historical precedence). if (auto header_value = request.get("X-ClickHouse-Database", ""); !header_value.empty()) settings_changes.setSetting("database", header_value); - if (auto header_value = request.get("X-ClickHouse-Format", ""); !header_value.empty()) - settings_changes.setSetting("output_format", header_value); + + /// The `X-ClickHouse-Format` header is applied below, once the settings from the URL and the user + /// profile are in effect: which setting it aliases depends on + /// `http_x_clickhouse_format_overrides_output_format`. + const String format_header_value = request.get("X-ClickHouse-Format", ""); ContextMutablePtr context; { @@ -480,6 +476,29 @@ void HTTPHandler::processQuery( context->checkSettingsConstraints(settings_changes, SettingSource::QUERY); context->applySettingsChanges(settings_changes); + /// The `X-ClickHouse-Format` header is an alias for the `output_format` setting, or - when + /// `http_x_clickhouse_format_overrides_output_format` is disabled - for the `default_format` + /// setting, which is what it meant before 26.8. Either way it overrides the URL parameter of the + /// same name (preserving the historical precedence). The choice is read from the context after + /// the URL parameters and the user profile have been applied, so the compatibility setting can + /// come from either of them. + /// + /// By default `X-ClickHouse-Format` maps to `output_format` rather than to `default_format`: + /// sending this header means the client definitely wants the response in that format, so it is + /// an explicit override (winning over the query's `FORMAT` clause and the path extension), not a + /// fallback used only when nothing else selects a format. It maps to `output_format` and not to + /// the bidirectional `format`, because the header has always described the response only: the + /// same header on `INSERT INTO t FORMAT JSONEachRow …` must not reinterpret the request body. + if (!format_header_value.empty()) + { + SettingsChanges format_header_changes; + format_header_changes.setSetting( + context->getSettingsRef()[Setting::http_x_clickhouse_format_overrides_output_format] ? "output_format" : "default_format", + format_header_value); + context->checkSettingsConstraints(format_header_changes, SettingSource::QUERY); + context->applySettingsChanges(format_header_changes); + } + const auto & settings = context->getSettingsRef(); /// === URL path parsing happens after settings are applied === diff --git a/tests/queries/0_stateless/05233_http_x_clickhouse_format_compat.reference b/tests/queries/0_stateless/05233_http_x_clickhouse_format_compat.reference new file mode 100644 index 0000000000000..91530eb01b1c5 --- /dev/null +++ b/tests/queries/0_stateless/05233_http_x_clickhouse_format_compat.reference @@ -0,0 +1,34 @@ +-- default: the header overrides the FORMAT clause of the query +{"x":1} +-- default: the header applies when the query has no FORMAT clause +{"x":1} +-- default: the header overrides the output_format URL parameter +{"x":1} +-- default: the response header reports the effective format +X-ClickHouse-Format: JSONEachRow +-- disabled: the header only sets default_format, so the FORMAT clause wins +1 +-- disabled: the header applies when the query has no FORMAT clause +{"x":1} +-- disabled: the header overrides the default_format URL parameter +{"x":1} +-- disabled: an explicit output_format URL parameter wins over the header +1 +-- disabled: the response header reports the effective format +X-ClickHouse-Format: CSV +-- disabled: works on a read-only GET request too +1 +-- disabled: the header does not change the input format of an INSERT body +{"s":"a","n":1} +-- disabled via the session (as a profile would): the FORMAT clause wins +1 +-- disabled as a URL parameter in a readonly = 1 session: the setting is always changeable in read-only mode +1 +{"x":1} +-- ... while an ordinary setting is rejected in the same session +Cannot modify 'max_rows_to_read' setting in readonly mode +-- compatibility with a version before 26.8 restores the old header behavior +1 +-- compatibility with 26.8 or 26.9 keeps the header an alias for output_format, as in those releases +{"x":1} +{"x":1} diff --git a/tests/queries/0_stateless/05233_http_x_clickhouse_format_compat.sh b/tests/queries/0_stateless/05233_http_x_clickhouse_format_compat.sh new file mode 100755 index 0000000000000..ff816400aedd5 --- /dev/null +++ b/tests/queries/0_stateless/05233_http_x_clickhouse_format_compat.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `X-ClickHouse-Format` is an alias for `output_format` by default (since 26.8), and for `default_format` +# when `http_x_clickhouse_format_overrides_output_format` is disabled (the behavior of earlier versions). + +URL_COMPAT="${CLICKHOUSE_URL}&http_x_clickhouse_format_overrides_output_format=0" + +echo "-- default: the header overrides the FORMAT clause of the query" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' +echo "-- default: the header applies when the query has no FORMAT clause" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x' +echo "-- default: the header overrides the output_format URL parameter" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&output_format=CSV" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x' +echo "-- default: the response header reports the effective format" +${CLICKHOUSE_CURL} -sS -i "${CLICKHOUSE_URL}" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' | grep -i '^X-ClickHouse-Format' | tr -d '\r' + +echo "-- disabled: the header only sets default_format, so the FORMAT clause wins" +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' +echo "-- disabled: the header applies when the query has no FORMAT clause" +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x' +echo "-- disabled: the header overrides the default_format URL parameter" +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}&default_format=CSV" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x' +echo "-- disabled: an explicit output_format URL parameter wins over the header" +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}&output_format=CSV" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x' +echo "-- disabled: the response header reports the effective format" +${CLICKHOUSE_CURL} -sS -i "${URL_COMPAT}" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' | grep -i '^X-ClickHouse-Format' | tr -d '\r' +echo "-- disabled: works on a read-only GET request too" +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}&query=SELECT+1+AS+x+FORMAT+CSV" -H 'X-ClickHouse-Format: JSONEachRow' +echo "-- disabled: the header does not change the input format of an INSERT body" +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}" -d 'CREATE TABLE t_05233 (s String, n UInt8) ENGINE = Memory' +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}" -H 'X-ClickHouse-Format: JSONEachRow' -d 'INSERT INTO t_05233 FORMAT CSV +"a",1' +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}" -d 'SELECT * FROM t_05233 FORMAT JSONEachRow' +${CLICKHOUSE_CURL} -sS "${URL_COMPAT}" -d 'DROP TABLE t_05233' + +echo "-- disabled via the session (as a profile would): the FORMAT clause wins" +SESSION_ID="${CLICKHOUSE_DATABASE}_05233" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID}" -d 'SET http_x_clickhouse_format_overrides_output_format = 0' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID}" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID}&close_session=1" -d 'SELECT 1 AS x FORMAT Null' + +echo "-- disabled as a URL parameter in a readonly = 1 session: the setting is always changeable in read-only mode" +# A bare URL: the randomized settings the harness puts into CLICKHOUSE_URL are not changeable under readonly = 1. +SESSION_ID_RO="${CLICKHOUSE_DATABASE}_05233_ro" +URL_RO="${CLICKHOUSE_URL%%\?*}?database=${CLICKHOUSE_DATABASE}&session_id=${SESSION_ID_RO}" +${CLICKHOUSE_CURL} -sS "${URL_RO}" -d 'SET readonly = 1' +${CLICKHOUSE_CURL} -sS "${URL_RO}&http_x_clickhouse_format_overrides_output_format=0" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' +${CLICKHOUSE_CURL} -sS "${URL_RO}&http_x_clickhouse_format_overrides_output_format=1" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' +echo "-- ... while an ordinary setting is rejected in the same session" +${CLICKHOUSE_CURL} -sS "${URL_RO}&max_rows_to_read=1" -d 'SELECT 1 AS x FORMAT CSV' | grep -o "Cannot modify 'max_rows_to_read' setting in readonly mode" +${CLICKHOUSE_CURL} -sS "${URL_RO}&close_session=1" -d 'SELECT 1 AS x FORMAT Null' + +echo "-- compatibility with a version before 26.8 restores the old header behavior" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&compatibility=26.7" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' +echo "-- compatibility with 26.8 or 26.9 keeps the header an alias for output_format, as in those releases" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&compatibility=26.8" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&compatibility=26.9" -H 'X-ClickHouse-Format: JSONEachRow' -d 'SELECT 1 AS x FORMAT CSV' From 3291a11dee8ae6a021b8d3547cfe151ac9dc0f2c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 09:21:03 +0000 Subject: [PATCH 125/296] Backport #115065 to 26.8: Do not pop hints on a line displayed programmatically --- .gitmodules | 2 +- contrib/CMakeLists.txt | 6 +- contrib/ai-sdk-cpp-cmake/CMakeLists.txt | 23 +- contrib/nlohmann-json | 2 +- contrib/nlohmann-json-cmake/CMakeLists.txt | 6 +- contrib/replxx | 2 +- src/Client/ReplxxLineReader.cpp | 121 +++++- src/Client/ReplxxLineReader.h | 26 ++ ...909_client_hints_history_navigation.python | 392 ++++++++++++++++++ ..._client_hints_history_navigation.reference | 11 + .../04909_client_hints_history_navigation.sh | 13 + ...025_client_hints_prepopulated_query.python | 338 +++++++++++++++ ..._client_hints_prepopulated_query.reference | 2 + .../05025_client_hints_prepopulated_query.sh | 14 + 14 files changed, 931 insertions(+), 27 deletions(-) create mode 100644 tests/queries/0_stateless/04909_client_hints_history_navigation.python create mode 100644 tests/queries/0_stateless/04909_client_hints_history_navigation.reference create mode 100755 tests/queries/0_stateless/04909_client_hints_history_navigation.sh create mode 100644 tests/queries/0_stateless/05025_client_hints_prepopulated_query.python create mode 100644 tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference create mode 100755 tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh diff --git a/.gitmodules b/.gitmodules index 2f3dd2930a0a0..f110ecccc5f76 100644 --- a/.gitmodules +++ b/.gitmodules @@ -365,7 +365,7 @@ url = https://github.com/ClickHouse/ai-sdk-cpp [submodule "contrib/nlohmann-json"] path = contrib/nlohmann-json - url = https://github.com/nlohmann/json.git + url = https://github.com/ClickHouse/json [submodule "contrib/crc32c"] path = contrib/crc32c url = https://github.com/ClickHouse/crc32c diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 4253bf5fad424..537a9f057984d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -266,8 +266,12 @@ option(ENABLE_GOOGLE_CLOUD_CPP "Enable Google Cloud Cpp" ${ENABLE_GOOGLE_CLOUD_C # `crc32c` is a small portable library used by both `google-cloud-cpp` and the snappy framing format # in `SnappyWriteBuffer`/`SnappyFramedReadBuffer`, so it is always built when libraries are enabled. add_contrib (crc32c-cmake crc32c) +# `nlohmann::json` is header-only and is used by both `google-cloud-cpp` and `ai-sdk-cpp`, which are +# enabled independently of each other. It is added for everyone so that there is one copy of it in +# the build: two copies of the same version define the same symbols with different layouts, which +# the linker resolves silently - see `contrib/ai-sdk-cpp-cmake/CMakeLists.txt`. +add_contrib (nlohmann-json-cmake nlohmann-json) if(ENABLE_GOOGLE_CLOUD_CPP) - add_contrib (nlohmann-json-cmake nlohmann-json) add_contrib (google-cloud-cpp-cmake google-cloud-cpp) # requires grpc, protobuf, absl, nlohmann's json, crc32c else() message(STATUS "Not using Google Cloud Cpp") diff --git a/contrib/ai-sdk-cpp-cmake/CMakeLists.txt b/contrib/ai-sdk-cpp-cmake/CMakeLists.txt index 6bcf697023abf..72230124d7e1f 100644 --- a/contrib/ai-sdk-cpp-cmake/CMakeLists.txt +++ b/contrib/ai-sdk-cpp-cmake/CMakeLists.txt @@ -38,13 +38,16 @@ set(AI_SDK_ANTHROPIC_SOURCES "${AI_SDK_SOURCE_DIR}/src/providers/anthropic/anthropic_factory.cpp" ) -# Add nlohmann_json from submodule -set(NLOHMANN_JSON_SOURCE_DIR "${AI_SDK_THIRD_PARTY_DIR}/nlohmann_json_patched") -add_library(_ai_sdk_nlohmann_json INTERFACE) -target_include_directories(_ai_sdk_nlohmann_json SYSTEM INTERFACE - "${NLOHMANN_JSON_SOURCE_DIR}/include" -) -add_library(ai_sdk_nlohmann_json::ai_sdk_nlohmann_json ALIAS _ai_sdk_nlohmann_json) +# `nlohmann::json` comes from `contrib/nlohmann-json`, not from the `third_party/nlohmann_json_patched` +# copy this library vendors. Both are version 3.12.0, so both define the same +# `nlohmann::json_abi_v3_12_0` symbols - and the two definitions differ: the `contrib` copy is the +# ClickHouse fork that reads numbers without consulting the locale, whose `lexer` holds a +# `number_buffer` member that the vendored copy has no field for. Linking both is an ODR violation +# that the linker resolves by keeping one definition of each symbol, so a `parser` laid out by one +# header ends up in a `parse` compiled from the other, which reads and destroys members past the +# end of it: `AddressSanitizer` reports a stack-buffer-overflow and the client aborts on the first +# response of the model. The vendored copy exists to keep the number parsing away from the locale, +# which the fork does properly, so there is nothing left to keep it for. # Add httplib from submodule (header-only) set(HTTPLIB_SOURCE_DIR "${AI_SDK_THIRD_PARTY_DIR}/httplib-header-only") @@ -79,7 +82,7 @@ target_include_directories(_ai-sdk-cpp-core SYSTEM target_link_libraries(_ai-sdk-cpp-core PUBLIC - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue @@ -105,7 +108,7 @@ target_include_directories(_ai-sdk-cpp-openai SYSTEM target_link_libraries(_ai-sdk-cpp-openai PUBLIC _ai-sdk-cpp-core - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue @@ -133,7 +136,7 @@ target_include_directories(_ai-sdk-cpp-anthropic SYSTEM target_link_libraries(_ai-sdk-cpp-anthropic PUBLIC _ai-sdk-cpp-core - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue diff --git a/contrib/nlohmann-json b/contrib/nlohmann-json index 55f93686c0152..4438a2a0cda77 160000 --- a/contrib/nlohmann-json +++ b/contrib/nlohmann-json @@ -1 +1 @@ -Subproject commit 55f93686c01528224f448c19128836e7df245f72 +Subproject commit 4438a2a0cda77bf4cd74652423e03363e003a0cc diff --git a/contrib/nlohmann-json-cmake/CMakeLists.txt b/contrib/nlohmann-json-cmake/CMakeLists.txt index 752b845bc819e..863c35363cd12 100644 --- a/contrib/nlohmann-json-cmake/CMakeLists.txt +++ b/contrib/nlohmann-json-cmake/CMakeLists.txt @@ -1,8 +1,4 @@ set(JSON_DIR ${ClickHouse_SOURCE_DIR}/contrib/nlohmann-json) add_library(_nlohmann_json INTERFACE) -set_property( - TARGET _nlohmann_json - APPEND - PROPERTY INTERFACE_INCLUDE_DIRECTORIES - ${JSON_DIR}/single_include) +target_include_directories(_nlohmann_json SYSTEM INTERFACE ${JSON_DIR}/single_include) add_library(ch_contrib::nlohmann_json ALIAS _nlohmann_json) diff --git a/contrib/replxx b/contrib/replxx index c2de583a3cd41..4ab25d428dd0b 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit c2de583a3cd41f7b476cb625e954c191b6fcc448 +Subproject commit 4ab25d428dd0bfc0590cac0d15e1fb872794dd62 diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 37af946759846..6a564e5e19650 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -438,6 +438,19 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) hint_completions_context.clear(); hint_completions_context_size = 0; + /// A line that was just displayed programmatically (recalled from history, found by a + /// history search, pasted, brought back from the editor) must not pop hints by itself: + /// with hints visible, the next Up/Down press would navigate the hints instead of the + /// history. The display armed the one-shot and pinned the displayed text (the same + /// display can regenerate the hints once more when replxx replays a throttled + /// refresh); the first run for an edited text unpins and shows the hints again. + if (suppress_hints_once || (!suppress_hints_for_text.empty() && suppress_hints_for_text == rx.get_state().text())) + { + suppress_hints_once = false; + return replxx::Replxx::hints_t{}; + } + suppress_hints_for_text.clear(); + /// Mirror `set_complete_on_empty(false)` *before* matching: an empty last word matches /// every suggestion, and this callback runs on every zero-delay repaint, so we must not /// fold and stable-sort the whole dictionary only to drop the result here. @@ -488,12 +501,43 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) /// The modify callback runs on every dispatched action, so reset the mirror here to track /// replxx; the hint-navigation keys re-set it *after* invoking, so a real navigation stays. rx.set_modify_callback([this] (std::string &, int &) { hint_selection = -1; }); + + /// A pasted query is also a whole new line displayed at once, so it does not pop hints + /// either (replxx's default binding for the paste marker just invokes the same action; + /// the action reads the whole paste, so the buffer holds the pasted text afterwards). + rx.bind_key(Replxx::KEY::PASTE_START, [this](char32_t code) + { + suppress_hints_once = true; + auto result = rx.invoke(Replxx::ACTION::BRACKETED_PASTE, code); + /// The paste action fills the buffer directly, without invalidating replxx's hint + /// cache, which is keyed by the buffer text and lives across prompts. Pasting the + /// exact text that carried a visible hint on an earlier prompt would therefore + /// redisplay the cached hints without ever asking our hint callback, and the + /// suppression below would have nothing to suppress. Re-setting the state is what + /// invalidates that cache (see openEditor). + rx.set_state(rx.get_state()); + suppressHintsForDisplayedLine(); + return result; + }); } /// By default C-p/C-n bound to COMPLETE_NEXT/COMPLETE_PREV, /// bind C-p/C-n to history-previous/history-next like readline. - rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); }); - rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); }); + rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_NEXT, code); }); + rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_PREVIOUS, code); }); + rx.bind_key(Replxx::KEY::meta(Replxx::KEY::DOWN), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_NEXT, code); }); + rx.bind_key(Replxx::KEY::meta(Replxx::KEY::UP), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_PREVIOUS, code); }); + rx.bind_key(Replxx::KEY::meta('p'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_COMMON_PREFIX_SEARCH, code); }); + rx.bind_key(Replxx::KEY::meta('n'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_COMMON_PREFIX_SEARCH, code); }); + rx.bind_key(Replxx::KEY::meta('<'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_FIRST, code); }); + rx.bind_key(Replxx::KEY::PAGE_UP, [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_FIRST, code); }); + rx.bind_key(Replxx::KEY::meta('>'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_LAST, code); }); + rx.bind_key(Replxx::KEY::PAGE_DOWN, [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_LAST, code); }); + rx.bind_key(Replxx::KEY::control('G'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_RESTORE_CURRENT, code); }); + rx.bind_key(Replxx::KEY::meta('g'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_RESTORE, code); }); + rx.bind_key(Replxx::KEY::control('R'), [this](char32_t code) { return historySearch(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, code); }); + rx.bind_key(Replxx::KEY::control('S'), [this](char32_t code) { return historySearch(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, code); }); + rx.bind_key(Replxx::KEY::meta('r'), [this](char32_t code) { return historySearch(Replxx::ACTION::HISTORY_SEEDED_INCREMENTAL_SEARCH, code); }); /// We don't want the default, "suspend" behavior, it confuses people. if (options.ignore_shell_suspend) @@ -562,7 +606,7 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) hint_selection = next; return result; } - return rx.invoke(Replxx::ACTION::LINE_NEXT, code); + return historyNavigate(Replxx::ACTION::LINE_NEXT, code); }; /// Up navigates the hints only once a hint is selected; before that it keeps recalling /// command history, so the hints do not shadow it. @@ -575,7 +619,7 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) hint_selection = next; return result; } - return rx.invoke(Replxx::ACTION::LINE_PREVIOUS, code); + return historyNavigate(Replxx::ACTION::LINE_PREVIOUS, code); }; rx.bind_key(Replxx::KEY::DOWN, hint_next); rx.bind_key(Replxx::KEY::UP, hint_previous); @@ -590,7 +634,7 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) hint_selection = next; return result; } - return rx.invoke(Replxx::ACTION::LINE_PREVIOUS, code); + return historyNavigate(Replxx::ACTION::LINE_PREVIOUS, code); }); /// Right accepts the chosen hint (the single one shown, or the one selected by navigating); @@ -670,14 +714,23 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) /// REPAINT before to avoid prompt overlap by the query rx.invoke(Replxx::ACTION::REPAINT, code); - if (!new_query.empty()) + const bool selected_query = !new_query.empty(); + if (selected_query) + { + /// The picked query is a whole new line displayed at once - do not pop hints on it + /// (see historyNavigate). + suppress_hints_once = true; rx.set_state(replxx::Replxx::State(new_query.c_str(), static_cast(new_query.size()))); + } if (bracketed_paste_enabled) enableBracketedPaste(); rx.invoke(Replxx::ACTION::CLEAR_SELF, code); - return rx.invoke(Replxx::ACTION::REPAINT, code); + auto result = rx.invoke(Replxx::ACTION::REPAINT, code); + if (selected_query) + suppressHintsForDisplayedLine(); + return result; }; rx.bind_key(Replxx::KEY::control(key_fuzzy), interactive_history_search); @@ -692,7 +745,7 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) { /// Reverse search is detected by C-R. uint32_t reverse_search = Replxx::KEY::control('R'); - return rx.invoke(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, reverse_search); + return historySearch(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, reverse_search); }); /// Change cursor style for overwrite mode to blinking (see console_codes(5)) @@ -736,6 +789,41 @@ bool ReplxxLineReader::hintChosen() return hintPopupActive() && (hint_selection >= 0 || hint_count == 1); } +replxx::Replxx::ACTION_RESULT ReplxxLineReader::historyNavigate(replxx::Replxx::ACTION action, char32_t code) +{ + /// The recalled entry is displayed (and its hints regenerated) inside the action, so the + /// suppression must be armed before it; the pin below keeps later regenerations of the + /// recalled text hintless (the refresh inside the action may be throttled and replayed after + /// this returns) and is cleared by the first edit. + suppress_hints_once = true; + auto result = rx.invoke(action, code); + if (rx.history_recalled()) + suppressHintsForDisplayedLine(); + else + suppress_hints_once = false; + return result; +} + +replxx::Replxx::ACTION_RESULT ReplxxLineReader::historySearch(replxx::Replxx::ACTION action, char32_t code) +{ + /// The selected entry is displayed (and its hints regenerated) inside the search action, so + /// the suppression must be armed before it. C-R, C-S, Meta-R, and the ClickHouse regular + /// history-search binding all use this wrapper. + suppress_hints_once = true; + auto result = rx.invoke(action, code); + if (rx.history_recalled()) + suppressHintsForDisplayedLine(); + else + suppress_hints_once = false; + return result; +} + +void ReplxxLineReader::suppressHintsForDisplayedLine() +{ + suppress_hints_once = false; + suppress_hints_for_text = rx.get_state().text(); +} + ReplxxLineReader::~ReplxxLineReader() { /// `Replxx::print` may fail with `std::runtime_error("write failed")` when e.g. the pty of the embedded @@ -841,8 +929,19 @@ void ReplxxLineReader::openEditor(bool format_query) rx.print("\n"); } + /// The repaint below displays the whole buffer at once on every return path - the edited + /// query, or the original one brought back when the editor exited unsuccessfully or the + /// round trip threw. All of them are programmatic displays, so none of them may pop hints + /// (see historyNavigate); otherwise the hints left over from before the editor was opened + /// would stay live and the next Down would navigate them instead of the history. + /// replxx caches the hints by the buffer text, which is unchanged unless the edited query was + /// accepted, so re-setting the state is what makes it ask the hint callback again (and get an + /// empty list) instead of redisplaying the stale cached ones. + rx.set_state(rx.get_state()); + suppress_hints_once = true; rx.invoke(replxx::Replxx::ACTION::CLEAR_SELF, 0); rx.invoke(replxx::Replxx::ACTION::REPAINT, 0); + suppressHintsForDisplayedLine(); if (bracketed_paste_enabled) enableBracketedPaste(); @@ -866,6 +965,12 @@ void ReplxxLineReader::setInitialText(const String & text) if (!text.empty()) { rx.set_preload_buffer(text); + /// The preloaded query is displayed at once - do not pop hints on it (see + /// historyNavigate). The one-shot is consumed at the first render of the line inside + /// input(); the pin is set to the raw text (replxx may normalize whitespace in the + /// preload, in which case it just stays inert). + suppress_hints_once = true; + suppress_hints_for_text = text; } } diff --git a/src/Client/ReplxxLineReader.h b/src/Client/ReplxxLineReader.h index 81958ae8ca350..f7813514374da 100644 --- a/src/Client/ReplxxLineReader.h +++ b/src/Client/ReplxxLineReader.h @@ -52,6 +52,18 @@ class ReplxxLineReader : public LineReader int executeEditor(const std::string & path); void openEditor(bool format_query); + /// Run a history-navigation action with the hint suppression armed (see + /// `suppress_hints_once`): the entry it recalls must not pop hints by itself. + replxx::Replxx::ACTION_RESULT historyNavigate(replxx::Replxx::ACTION action, char32_t code); + + /// Run a history-search action with the hint suppression armed (see + /// `suppress_hints_once`): a selected entry must not pop hints by itself. + replxx::Replxx::ACTION_RESULT historySearch(replxx::Replxx::ACTION action, char32_t code); + + /// After a line was displayed programmatically, pin its text so that any hint regeneration + /// for it shows nothing (see `suppress_hints_for_text`). + void suppressHintsForDisplayedLine(); + /// Whether the text cursor is at the very end of the input (where as-you-type hints render). bool isCursorAtEndOfInput(); /// Whether the as-you-type hint "popup" is currently navigable here: hints are shown and the @@ -89,6 +101,20 @@ class ReplxxLineReader : public LineReader int hint_count = 0; int hint_selection = -1; + /// Suppression of the as-you-type hints for a line that is displayed programmatically - + /// recalled from history, found by a history search, pasted, brought back from the editor. + /// Such a display must not pop hints by itself: with hints visible, the next Up/Down press + /// would navigate the hints instead of the history. An edit shows the hints again. + /// `suppress_hints_once` is armed before the action that displays the line (the action + /// repaints, and regenerates the hints, inside itself) and consumed by the next run of the + /// hint callback. `suppress_hints_for_text` then pins the displayed text after the action, + /// because the same display can regenerate the hints again later - replxx replays a + /// throttled refresh after the key handler returns (its "rapid refresh" of e.g. a held-down + /// Up key) - so any later callback run for exactly this text shows no hints either; the + /// first run for an edited text clears the pin. + bool suppress_hints_once = false; + std::string suppress_hints_for_text; + /// Snapshot of the completion words computed when the hints were last displayed, plus the /// context (prefix and its length) they were computed for. The completion callback reuses it /// so that accepting a hint inserts exactly the word that was shown: replxx accepts a hint by diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.python b/tests/queries/0_stateless/04909_client_hints_history_navigation.python new file mode 100644 index 0000000000000..54cd43293f00c --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.python @@ -0,0 +1,392 @@ +import multiprocessing +import os +import pty +import re +import select +import shlex +import sys +import time + +TIMEOUT_SECONDS = 30 + +# How long to keep reading after the last keystroke while waiting for the expected output. The +# committed-query case only sees the result once the query has executed, which can take longer +# than the inter-keystroke drain on slow (sanitizer) builds. +FINAL_WAIT_SECONDS = 20 + +# How long to wait for the client to print the next `:) ` prompt after a committed query. This is +# the time a query of this test may take, and it is generous because the query does not have to be +# fast for the test to mean anything: in an MSan build a trivial `SELECT` that fails to resolve an +# identifier has been seen taking 36 seconds. +PROMPT_WAIT_SECONDS = 120 + +# Idle window after every keystroke, long enough for `replxx` to repaint the line. +DRAIN_IDLE_SECONDS = 0.4 + +# Extra slack for the supervisor on top of everything the worker itself may legitimately wait for. +SUPERVISOR_SLACK_SECONDS = 15 + +# A pseudo-keystroke: wait for the next prompt instead of writing anything. Sending the keys of the +# next line while the previous query is still running would let the terminal driver echo them +# instead of `replxx` interpreting them - a bracketed paste would then appear verbatim as +# `^[[200~...^[[201~` in the output and never reach the line editor. +WAIT_FOR_PROMPT = object() + +# `clickhouse-test` dumps this file into the failure report, but only under the name it derives +# from the testcase (the `.sh` wrapper), so take the path from there. +DEBUG_LOG = os.environ["CLICKHOUSE_TEST_DEBUG_LOG"] + +# The gray SGR sequence replxx uses for the as-you-type hint (Color::GRAY -> "0;90"). +HINT_COLOR = "\x1b[0;90m" + + +def make_history_file(name, entries): + """Write a replxx-format history file with the given entries, oldest first.""" + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + + f".{name}.history", + ) + with open(path, "w") as f: + for i, entry in enumerate(entries): + f.write(f"### 2020-01-01 00:00:{i:02}.000\n{entry}\n") + return path + + +def read_until(master, debug_log_fd, predicate, timeout): + output = "" + deadline = time.time() + timeout + while time.time() < deadline: + r, _, _ = select.select([master], [], [], 0.3) + if not r: + if predicate(output): + return output + continue + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + if predicate(output): + return output + return output + + +def drain(master, debug_log_fd, idle): + output = "" + while True: + r, _, _ = select.select([master], [], [], idle) + if not r: + break + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + return output + + +def report_failure(name, shell_pid, output, debug_log_fd): + """A PTY case that fails prints just `FAIL`, which says nothing about why. Record what the + terminal actually showed and whether the client is still alive - the client dying (the read + loop hitting EOF) looks exactly like the expected text never being printed.""" + try: + reaped_pid, wait_status = os.waitpid(shell_pid, os.WNOHANG) + except ChildProcessError: + reaped_pid, wait_status = shell_pid, -1 + running = reaped_pid == 0 + banner = ( + f"=== {name}: FAIL ===\n" + f"client still running: {running}, wait status: {wait_status}\n" + f"terminal output: {output[-8000:]!r}\n" + ) + debug_log_fd.write(banner) + debug_log_fd.flush() + sys.stderr.write(banner) + sys.stderr.flush() + + +def run_case(program, argv, name, keystrokes, check, state=None): + """Fork a PTY, wait for the prompt, send keystrokes (with a small pause between + them so replxx repaints), then run `check` over the collected output.""" + shell_pid, master = pty.fork() + if shell_pid == 0: + os.environ["TERM"] = "xterm" + os.execv(program, argv) + return + + debug_log_fd = open(DEBUG_LOG, "a") + try: + read_until(master, debug_log_fd, lambda o: ":)" in o, TIMEOUT_SECONDS) + + output = "" + # Only what arrived after the last keystroke - a prompt seen earlier in `output` says + # nothing about the prompt this step is waiting for. + since_last_keystroke = "" + for keys in keystrokes: + if keys is WAIT_FOR_PROMPT: + extra = read_until( + master, + debug_log_fd, + lambda more, seen=since_last_keystroke: ":)" in seen + more, + PROMPT_WAIT_SECONDS, + ) + output += extra + since_last_keystroke += extra + # `read_until` returns on its deadline as well as on a match, and going on from a + # deadline would write the next keys into a terminal that is still running the + # previous query: the driver echoes them instead of `replxx` interpreting them, so + # a bracketed paste ends up in the output verbatim and the case fails on its + # content, saying nothing about the prompt that never came. Stop here instead. + if ":)" not in since_last_keystroke: + print(f"{name}: FAIL") + report_failure(name, shell_pid, output, debug_log_fd) + state.value = 1 + return + continue + os.write(master, keys) + since_last_keystroke = drain(master, debug_log_fd, DRAIN_IDLE_SECONDS) + output += since_last_keystroke + + # The expected output may arrive only after the committed query executes, which can + # outlast the inter-keystroke drain on slow builds. Keep reading until the check passes + # instead of giving up after a single drain window. + if not check(output): + output += read_until( + master, + debug_log_fd, + lambda extra: check(output + extra), + FINAL_WAIT_SECONDS, + ) + + ok = check(output) + print(f"{name}: {'OK' if ok else 'FAIL'}") + if not ok: + report_failure(name, shell_pid, output, debug_log_fd) + state.value = 0 if ok else 1 + finally: + os.close(master) + debug_log_fd.close() + + +def worker_budget(keystrokes): + """Everything `run_case` may legitimately spend: the wait for the first prompt, the idle window + after every keystroke, every explicit prompt wait, and the final wait for the expected output. + The supervisor must not expire before that, or ordinary slowness looks like a failure.""" + prompt_waits = sum(1 for keys in keystrokes if keys is WAIT_FOR_PROMPT) + return ( + TIMEOUT_SECONDS + + PROMPT_WAIT_SECONDS * prompt_waits + + DRAIN_IDLE_SECONDS * len(keystrokes) + + FINAL_WAIT_SECONDS + + SUPERVISOR_SLACK_SECONDS + ) + + +def run_with_timeout(program, argv, name, history, keystrokes, check): + """Run a case with a history file holding the `history` entries. The file is written anew for + every attempt: an attempt killed after its final Enter may already have appended the query to + it, and a retry starting from that tail would recall a different entry than the case intends + while still printing the expected result.""" + for attempt in range(5): + history_file = make_history_file(name, history) + state = multiprocessing.Value("i", -1) + process = multiprocessing.Process( + target=run_case, + args=(program, argv + [f"--history_file={history_file}"], name, keystrokes, check), + kwargs={"state": state}, + ) + process.start() + process.join(worker_budget(keystrokes)) + if process.is_alive(): + process.terminate() + if state.value in (0, 1): + return + # transient timeout on a loaded machine - retry + print(f"{name}: FAIL") + sys.stderr.write(f"=== {name}: FAIL === the case timed out on every attempt\n") + + +def main(): + program = os.environ["CLICKHOUSE_LOCAL"] + base = shlex.split(program) + args = base + ["--wait_for_suggestions_to_load", "--hints", "1"] + + # A line displayed programmatically (recalled from history, pasted) must not pop the + # as-you-type hints by itself: with hints visible, the next Up/Down press would navigate + # the hints instead of the history. An edit shows the hints again. + + # 1. Up recalls an entry; no hint appears at its end ("ive" would be the ghost suffix of + # "concatAssumeInjective"). + run_with_timeout( + base[0], + args, + "up_recall_no_hints", + ["SELECT concatAssumeInject"], + [b"\x1b[A"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # Meta-Up is replxx's direct history-previous binding. It must use the same suppression as + # plain Up, rather than leaving the default action to regenerate hints. + run_with_timeout( + base[0], + args, + "meta_up_recall_no_hints", + ["SELECT concatAssumeInject"], + [b"\x1b\x1b[A"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # Ctrl-G restores the currently recalled history entry after an edit. It is another whole + # line display, so Down must return to the scratch line rather than navigate the hint list. + run_with_timeout( + base[0], + args, + "restore_current_down_navigates_history", + ["SELECT concatAssumeInjec"], + [b"SELECT 111 AS rand", b"\x1b[A", b"t", b"\x07", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + + # 2. Up, Up, Down walks the history (the entries end in "rand", which has many hint + # matches - Down must not step into the hint list); Enter then runs the recalled entry, + # so its result (222) is printed. The result can use either Pretty or TabSeparated output + # depending on the client's default format. + run_with_timeout( + base[0], + args, + "down_navigates_history", + ["SELECT 111 AS rand", "SELECT 222 AS rand"], + [b"\x1b[A", b"\x1b[A", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*222\s*│|\x1b\[\?25h222\r?\n)", o) is not None, + ) + + # 3. Recalling a history entry with the same text as the scratch line is still a programmatic + # display. Down must return from that entry to the scratch line instead of selecting a hint. + run_with_timeout( + base[0], + args, + "same_text_recall_down_navigates_history", + ["SELECT 222 AS rand", "SELECT 111 AS rand"], + [b"SELECT 111 AS rand", b"\x1b[A", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + + # 3b. An incremental history search (Ctrl-T, the ClickHouse binding for the regular non-fuzzy + # reverse search) that accepts an entry is another whole-line display: it goes through a + # different `replxx` code path than the plain history recall, so it needs its own proof that + # Down keeps walking the history. Ctrl-T, then the search text, then Ctrl-E leaves the search + # keeping the found line ("SELECT 111 AS rand") and puts the cursor at its end, where the hints + # would render. The accepted entry is committed as the most recent recall, so the first Down + # re-displays it (`replxx` emulates the Windows down-arrow there) and the second one moves on + # to "SELECT 222 AS rand", which Enter then runs. + run_with_timeout( + base[0], + args, + "incremental_search_down_navigates_history", + ["SELECT 111 AS rand", "SELECT 222 AS rand"], + [b"\x14", b"111", b"\x05", b"\x1b[B", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*222\s*│|\x1b\[\?25h222\r?\n)", o) is not None, + ) + + # 4. Editing a recalled entry shows the hints again: after Up, typing the "t" completes the + # prefix "concatAssumeInject" and the ghost "ive" appears. + run_with_timeout( + base[0], + args, + "edit_after_recall_shows_hints", + ["SELECT concatAssumeInjec"], + [b"\x1b[A", b"t"], + lambda o: (HINT_COLOR + "ive") in o, + ) + + # 5. A (bracketed) paste is also a whole new line displayed at once - no hint at its end. + run_with_timeout( + base[0], + args, + "paste_no_hints", + [], + [b"\x1b[200~SELECT concatAssumeInject\x1b[201~"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # 5b. Pasting the exact text that carried a visible hint on an earlier prompt: replxx caches + # the hints by the buffer text and the cache outlives the prompt, so without invalidating it + # the paste would redisplay the stale ghost without ever asking our hint callback. The first + # line is typed (so the hints for it are generated and cached) and committed - it is not a + # valid query, which is irrelevant here, only the cache state matters - and then the very same + # text is pasted on the next prompt, where no hint may appear. Only the output after the last + # prompt is examined: the first, typed line legitimately shows the ghost. + run_with_timeout( + base[0], + args, + "same_text_paste_no_hints", + [], + [ + b"SELECT concatAssumeInject", + b"\r", + WAIT_FOR_PROMPT, + b"\x1b[200~SELECT concatAssumeInject\x1b[201~", + ], + lambda o: "concatAssumeInject" in o.rsplit(":)", 1)[-1] + and (HINT_COLOR + "ive") not in o.rsplit(":)", 1)[-1], + ) + + # 6. An unchanged editor round-trip redisplays the line programmatically. It must invalidate + # replxx's cached hints, so Down remains history navigation. + editor = os.path.join(os.environ["CLICKHOUSE_TMP"], "client_hints_unchanged_editor.sh") + with open(editor, "w") as f: + f.write("#!/bin/sh\nexit 0\n") + os.chmod(editor, 0o755) + previous_editor = os.environ.get("EDITOR") + os.environ["EDITOR"] = editor + try: + run_with_timeout( + base[0], + args, + "unchanged_editor_down_navigates_history", + [], + [b"SELECT 111 AS rand", b"\x1bE", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + finally: + if previous_editor is None: + del os.environ["EDITOR"] + else: + os.environ["EDITOR"] = previous_editor + + # 7. An editor that exits unsuccessfully brings the original line back - also a whole-line + # programmatic display, and one that happens while the hints of the line typed before the + # editor was opened are still live. Down must remain history navigation there too. + editor = os.path.join(os.environ["CLICKHOUSE_TMP"], "client_hints_failing_editor.sh") + with open(editor, "w") as f: + f.write("#!/bin/sh\nexit 1\n") + os.chmod(editor, 0o755) + previous_editor = os.environ.get("EDITOR") + os.environ["EDITOR"] = editor + try: + run_with_timeout( + base[0], + args, + "failing_editor_down_navigates_history", + [], + [b"SELECT 111 AS rand", b"\x1bE", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + finally: + if previous_editor is None: + del os.environ["EDITOR"] + else: + os.environ["EDITOR"] = previous_editor + + +if __name__ == "__main__": + # The check lambdas are not picklable, so the subprocesses must be forked (not spawned, + # which became the default on newer Python). + multiprocessing.set_start_method("fork") + main() diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.reference b/tests/queries/0_stateless/04909_client_hints_history_navigation.reference new file mode 100644 index 0000000000000..bcd45174864a0 --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.reference @@ -0,0 +1,11 @@ +up_recall_no_hints: OK +meta_up_recall_no_hints: OK +restore_current_down_navigates_history: OK +down_navigates_history: OK +same_text_recall_down_navigates_history: OK +incremental_search_down_navigates_history: OK +edit_after_recall_shows_hints: OK +paste_no_hints: OK +same_text_paste_no_hints: OK +unchanged_editor_down_navigates_history: OK +failing_editor_down_navigates_history: OK diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.sh b/tests/queries/0_stateless/04909_client_hints_history_navigation.sh new file mode 100755 index 0000000000000..04878aed03896 --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: long, no-debug + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `clickhouse-test` dumps `$CLICKHOUSE_TMP/.debuglog` into the failure report, and the +# testcase name is the name of this shell script - the Python part cannot derive it from its own +# file name, so pass the path down. +export CLICKHOUSE_TEST_DEBUG_LOG="$CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog" + +python3 "$CUR_DIR"/04909_client_hints_history_navigation.python diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python new file mode 100644 index 0000000000000..072594e6baef4 --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python @@ -0,0 +1,338 @@ +"""A query prepopulated into the next input line (the `??` AI SQL generation flow) is displayed +programmatically, so - like a recalled or pasted line - it must not pop the as-you-type hints: +with hints visible, the next Up/Down press would navigate the hints instead of the history. + +The AI provider is a local HTTP server speaking just enough of the OpenAI chat completion protocol +to answer with a canned query, so the test needs no network and no API key. +""" + +import http.server +import json +import multiprocessing +import os +import pty +import select +import shlex +import sys +import threading +import time + +TIMEOUT_SECONDS = 60 + +# How long to keep reading after the last keystroke while waiting for the expected output. +FINAL_WAIT_SECONDS = 30 + +# How long an explicit `wait_for` of a keystroke may take. It is generous because it covers the +# wait for the next `:) ` prompt after a committed query, and a query does not have to be fast for +# this test to mean anything: in an MSan build a trivial `SELECT` that fails to resolve an +# identifier has been seen taking 36 seconds. +STEP_WAIT_SECONDS = 120 + +# Idle window after a keystroke that has no explicit `wait_for`, long enough for `replxx` to +# repaint the line. +DRAIN_IDLE_SECONDS = 0.4 + +# Extra slack for the supervisor on top of everything the worker itself may legitimately wait for. +SUPERVISOR_SLACK_SECONDS = 15 + +# `clickhouse-test` dumps this file into the failure report, but only under the name it derives +# from the testcase (the `.sh` wrapper), so take the path from there. +DEBUG_LOG = os.environ["CLICKHOUSE_TEST_DEBUG_LOG"] + +# The gray SGR sequence replxx uses for the as-you-type hint (Color::GRAY -> "0;90"). +HINT_COLOR = "\x1b[0;90m" + +# The query the fake provider "generates". Its last word is a prefix of `concatAssumeInjective`, +# so an unsuppressed hint would render the gray ghost suffix "ive" right after it. +GENERATED_QUERY = "SELECT concatAssumeInject" + +# The prepopulated line is syntax-highlighted, so only its last word survives as a plain substring +# of the terminal output. +GENERATED_QUERY_LAST_WORD = "concatAssumeInject" + + +class CompletionHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + self.rfile.read(length) + body = json.dumps( + { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": f"{GENERATED_QUERY}", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + +def write_config(port): + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + ".config.xml", + ) + with open(path, "w") as f: + f.write( + "\n" + " \n" + " openai\n" + " test\n" + f" http://127.0.0.1:{port}\n" + " test-model\n" + " false\n" + " \n" + "\n" + ) + return path + + +def make_history_file(): + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + ".history", + ) + with open(path, "w") as f: + pass + return path + + +def read_until(master, debug_log_fd, predicate, timeout): + """Read until the predicate holds, the deadline passes, or the client is gone. Returns whether + the predicate matched along with what was read: a deadline is not a match, and treating it as + one lets a case go on from a state it never reached.""" + output = "" + deadline = time.time() + timeout + while time.time() < deadline: + r, _, _ = select.select([master], [], [], 0.3) + if not r: + if predicate(output): + return True, output + continue + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + if predicate(output): + return True, output + return predicate(output), output + + +def drain(master, debug_log_fd, idle): + output = "" + while True: + r, _, _ = select.select([master], [], [], idle) + if not r: + break + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + return output + + +def report_failure(name, shell_pid, output, debug_log_fd): + """A PTY case that fails prints just `FAIL`, which says nothing about why. Record what the + terminal actually showed and whether the client is still alive - the client dying (the read + loop hitting EOF) looks exactly like the expected text never being printed.""" + try: + reaped_pid, wait_status = os.waitpid(shell_pid, os.WNOHANG) + except ChildProcessError: + reaped_pid, wait_status = shell_pid, -1 + running = reaped_pid == 0 + banner = ( + f"=== {name}: FAIL ===\n" + f"client still running: {running}, wait status: {wait_status}\n" + f"terminal output: {output[-8000:]!r}\n" + ) + debug_log_fd.write(banner) + debug_log_fd.flush() + sys.stderr.write(banner) + sys.stderr.flush() + + +def run_case(program, argv, name, keystrokes, check, state=None, check_from=0): + """Each keystroke is a pair of the bytes to send and an optional predicate to wait for before + sending the next one (in place of a fixed idle drain). Only the output produced from the + keystroke with index `check_from` on is passed to `check`, so the earlier keystrokes may + legitimately pop hints without failing a "no hints" check.""" + shell_pid, master = pty.fork() + if shell_pid == 0: + os.environ["TERM"] = "xterm" + os.execv(program, argv) + return + + debug_log_fd = open(DEBUG_LOG, "a") + try: + matched, transcript = read_until( + master, debug_log_fd, lambda o: ":)" in o, TIMEOUT_SECONDS + ) + if not matched: + print(f"{name}: FAIL") + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 1 + return + + # Everything the terminal showed, for the failure transcript - `output` holds only what + # the check is allowed to look at. + output = "" + for index, (keys, wait_for) in enumerate(keystrokes): + os.write(master, keys) + if wait_for is not None: + matched, segment = read_until( + master, debug_log_fd, wait_for, STEP_WAIT_SECONDS + ) + else: + matched, segment = True, drain(master, debug_log_fd, DRAIN_IDLE_SECONDS) + transcript += segment + if index >= check_from: + output += segment + # A `wait_for` that expired means the state the next keystroke assumes was never + # reached: the hint that arms the stale cache never appeared, or the prompt never came + # back and the next line would be typed into a still running query, where the terminal + # driver echoes the bytes instead of `replxx` interpreting them. Either way the case + # would prove nothing while still being able to pass, so end it here with its + # transcript. + if not matched: + print(f"{name}: FAIL") + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 1 + return + + # The prepopulated line only appears after the (faked) generation round trip, which can + # outlast the inter-keystroke drain on slow builds. + if not check(output): + _, extra = read_until( + master, + debug_log_fd, + lambda more: check(output + more), + FINAL_WAIT_SECONDS, + ) + output += extra + transcript += extra + + ok = check(output) + print(f"{name}: {'OK' if ok else 'FAIL'}") + if not ok: + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 0 if ok else 1 + finally: + os.close(master) + debug_log_fd.close() + + +def worker_budget(keystrokes): + """Everything `run_case` may legitimately spend: the wait for the first prompt, every explicit + `wait_for`, the idle window after every other keystroke, and the final wait for the expected + output. The supervisor must not expire before that, or ordinary slowness looks like a + failure.""" + waits = sum(1 for _, wait_for in keystrokes if wait_for is not None) + return ( + TIMEOUT_SECONDS + + STEP_WAIT_SECONDS * waits + + DRAIN_IDLE_SECONDS * (len(keystrokes) - waits) + + FINAL_WAIT_SECONDS + + SUPERVISOR_SLACK_SECONDS + ) + + +def run_with_timeout(program, argv, name, keystrokes, check, check_from=0): + for attempt in range(5): + state = multiprocessing.Value("i", -1) + process = multiprocessing.Process( + target=run_case, + args=(program, argv, name, keystrokes, check), + kwargs={"state": state, "check_from": check_from}, + ) + process.start() + process.join(worker_budget(keystrokes)) + if process.is_alive(): + process.terminate() + if state.value in (0, 1): + return + # transient timeout on a loaded machine - retry + print(f"{name}: FAIL") + sys.stderr.write(f"=== {name}: FAIL === the case timed out on every attempt\n") + + +def main(): + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), CompletionHandler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + + config = write_config(server.server_address[1]) + history = make_history_file() + + program = os.environ["CLICKHOUSE_LOCAL"] + base = shlex.split(program) + args = base + [ + "--wait_for_suggestions_to_load", + "--hints", + "1", + f"--config-file={config}", + f"--history_file={history}", + ] + + no_hints = lambda o: GENERATED_QUERY_LAST_WORD in o and (HINT_COLOR + "ive") not in o + + try: + run_with_timeout( + base[0], + args, + "prepopulated_query_no_hints", + [(b"?? show me a query\r", None)], + no_hints, + ) + # Same-text regression: replxx keeps the hint cache across prompts, so when the generated + # query is byte-for-byte equal to the previously displayed line, the hint callback may not + # even be called - the stale cached hint must not be reused either. Type the query manually + # first (and wait for the hint ghost, which poisons the cache), execute it, then have the + # AI "generate" the very same text. The `??` request is kept as short as possible so that + # replxx's rapid-refresh throttling (1 ms) can swallow its keystrokes without regenerating + # the hint seed - only then does the preloaded line hit the stale cache. Only the output + # after the `??` keystroke is checked, as the manually typed line legitimately shows the + # hint. + run_with_timeout( + base[0], + args, + "same_text_prepopulated_query_no_hints", + [ + (GENERATED_QUERY.encode(), lambda o: (HINT_COLOR + "ive") in o), + (b"\r", lambda o: ":)" in o), + (b"??q\r", None), + ], + no_hints, + check_from=2, + ) + finally: + server.shutdown() + + +if __name__ == "__main__": + # The check lambdas are not picklable, so the subprocesses must be forked (not spawned, + # which became the default on newer Python). + multiprocessing.set_start_method("fork") + main() diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference new file mode 100644 index 0000000000000..fccb0eff5b813 --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference @@ -0,0 +1,2 @@ +prepopulated_query_no_hints: OK +same_text_prepopulated_query_no_hints: OK diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh new file mode 100755 index 0000000000000..d48ace2e154ef --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Tags: long, no-debug, no-fasttest +# no-fasttest: needs the AI SQL generator (`ENABLE_CLIENT_AI`), which is not built in the fast test. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `clickhouse-test` dumps `$CLICKHOUSE_TMP/.debuglog` into the failure report, and the +# testcase name is the name of this shell script - the Python part cannot derive it from its own +# file name, so pass the path down. +export CLICKHOUSE_TEST_DEBUG_LOG="$CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog" + +python3 "$CUR_DIR"/05025_client_hints_prepopulated_query.python From 9308016b33cdc3bc79b785205fd72691cb770e10 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 11:58:03 +0000 Subject: [PATCH 126/296] Backport #122057 to 26.8: Mask credentials of an S3 URI in validation exceptions --- src/IO/S3/URI.cpp | 42 ++++++++++++++++--- ..._uri_exception_masks_credentials.reference | 9 ++++ ...256_s3_uri_exception_masks_credentials.sql | 33 +++++++++++++++ 3 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05256_s3_uri_exception_masks_credentials.reference create mode 100644 tests/queries/0_stateless/05256_s3_uri_exception_masks_credentials.sql diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index b671cb447531f..43d5c47f271fc 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -4,11 +4,13 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -33,6 +35,34 @@ namespace ErrorCodes namespace S3 { +namespace +{ + +/// `Poco::URI::toString` renders the userinfo (`user:password@`) and the query parameters of a presigned +/// URL verbatim. Exception messages reach `system.query_log` and the server log, which, unlike the query +/// text, are not masked, so a URI must not be put into them as is. +String maskedURIString(const Poco::URI & uri) +{ + String result = uri.toString(); + maskURIUserinfo(result); + /// With `compatibility_s3_presigned_url_query_in_path` the constructor folds the query of a presigned URL + /// into the path by percent-encoding its '?', which `toString` renders as `%3F`, so `maskPresignedURLParameters` + /// would not see the parameters. Put the '?' back: `toString` encodes '%' itself as `%25`, so a `%3F` can only + /// come from a '?'. + boost::replace_all(result, "%3F", "?"); + maskPresignedURLParameters(result); + return result; +} + +/// With `compatibility_s3_presigned_url_query_in_path` the query of a presigned URL ends up in the bucket or the key. +String maskedQuotedString(String value) +{ + maskPresignedURLParameters(value); + return quoteString(value); +} + +} + URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_presigned_query_parameters, S3UriStyle uri_style) { /// Case when AWS Private Link Interface is being used @@ -143,13 +173,13 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_pre case S3UriStyle::VIRTUAL_HOSTED: { if (!tryInitVirtualHostedStyle(is_using_aws_private_link_interface, false)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 virtual-hosted-style uri: {}", !uri.empty() ? uri.toString() : ""); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 virtual-hosted-style uri: {}", !uri.empty() ? maskedURIString(uri) : ""); break; } case S3UriStyle::PATH: { if (!tryInitPathStyle()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 path-style uri: {}", !uri.empty() ? uri.toString() : ""); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 path-style uri: {}", !uri.empty() ? maskedURIString(uri) : ""); break; } } @@ -229,8 +259,8 @@ void URI::validateBucket(const String & bucket, const Poco::URI & uri) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Bucket name length is out of bounds in virtual hosted style S3 URI: {}{}", - quoteString(bucket), - !uri.empty() ? " (" + uri.toString() + ")" : ""); + maskedQuotedString(bucket), + !uri.empty() ? " (" + maskedURIString(uri) + ")" : ""); } void URI::validateKey(const String & key, const Poco::URI & uri) @@ -240,8 +270,8 @@ void URI::validateKey(const String & key, const Poco::URI & uri) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Invalid S3 key: {}{}", - quoteString(key), - !uri.empty() ? " (" + uri.toString() + ")" : ""); + maskedQuotedString(key), + !uri.empty() ? " (" + maskedURIString(uri) + ")" : ""); }; diff --git a/tests/queries/0_stateless/05256_s3_uri_exception_masks_credentials.reference b/tests/queries/0_stateless/05256_s3_uri_exception_masks_credentials.reference new file mode 100644 index 0000000000000..d8c2eb8f0b46c --- /dev/null +++ b/tests/queries/0_stateless/05256_s3_uri_exception_masks_credentials.reference @@ -0,0 +1,9 @@ +ExceptionBeforeStart 0 Bucket name length is out of bounds in virtual hosted style S3 URI: \'b\' (http://[HIDDEN]@127.0.0.1:9/b/k.csv) +ExceptionBeforeStart 0 Invalid S3 key: \'/a//b.csv\' (http://[HIDDEN]@my-bucket-name.s3.amazonaws.com//a//b.csv) +ExceptionBeforeStart 0 Invalid S3 virtual-hosted-style uri: http://[HIDDEN]@127.0.0.1:9/bucketname/k.csv +ExceptionBeforeStart 0 Invalid S3 path-style uri: http://[HIDDEN]@127.0.0.1:9 +ExceptionBeforeStart 0 Bucket name length is out of bounds in virtual hosted style S3 URI: \'b\' (http://127.0.0.1:9/b/k.csv?X-Amz-Signature=[HIDDEN]) +ExceptionBeforeStart 0 Bucket name length is out of bounds in virtual hosted style S3 URI: \'b\' (http://127.0.0.1:9/b/k.csv?X-Amz-Signature=[HIDDEN]) +ExceptionBeforeStart 0 Invalid S3 key: \'/a//b.csv?X-Amz-Algorithm=[HIDDEN]&X-Amz-Signature=[HIDDEN]\' (http://my-bucket-name.s3.amazonaws.com//a//b.csv?X-Amz-Algorithm=[HIDDEN]&X-Amz-Signature=[HIDDEN]) +ExceptionBeforeStart 0 Bucket name length is out of bounds in virtual hosted style S3 URI: \'bucketname?X-Amz-Signature=[HIDDEN]\' (http://127.0.0.1:9/bucketname?X-Amz-Signature=[HIDDEN]) +ExceptionBeforeStart 0 Bucket name length is out of bounds in virtual hosted style S3 URI: \'b\' (http://[HIDDEN]@127.0.0.1:9/b/x.csv) diff --git a/tests/queries/0_stateless/05256_s3_uri_exception_masks_credentials.sql b/tests/queries/0_stateless/05256_s3_uri_exception_masks_credentials.sql new file mode 100644 index 0000000000000..cecd87c3589f7 --- /dev/null +++ b/tests/queries/0_stateless/05256_s3_uri_exception_masks_credentials.sql @@ -0,0 +1,33 @@ +-- Tags: no-fasttest +-- no-fasttest: needs the `s3` table function + +-- Exceptions thrown while validating an S3 URI must not reveal the credentials embedded in it: +-- the exception text is stored in `system.query_log`, which, unlike the query text, is not masked. + +SET log_queries = 1; + +-- Bucket name is too short. +SELECT count() FROM s3('http://s3user:SECRET_bucket@127.0.0.1:9/b/k.csv', 'CSV'); -- { serverError BAD_ARGUMENTS } +-- Key with consecutive slashes. +SELECT count() FROM s3('http://ku:SECRET_key@my-bucket-name.s3.amazonaws.com//a//b.csv', 'CSV'); -- { serverError BAD_ARGUMENTS } +-- Not a virtual-hosted-style URI. +SELECT count() FROM s3('http://vu:SECRET_virtual@127.0.0.1:9/bucketname/k.csv', 'CSV') SETTINGS s3_uri_style = 'virtual_hosted'; -- { serverError BAD_ARGUMENTS } +-- Not a path-style URI. +SELECT count() FROM s3('http://pu:SECRET_path@127.0.0.1:9', 'CSV') SETTINGS s3_uri_style = 'path'; -- { serverError BAD_ARGUMENTS } +-- The query parameters of a presigned URL. +SELECT count() FROM s3('http://127.0.0.1:9/b/k.csv?X-Amz-Signature=SECRET_signature', 'CSV'); -- { serverError BAD_ARGUMENTS } +-- With `compatibility_s3_presigned_url_query_in_path` the query of a presigned URL is folded into the path... +SELECT count() FROM s3('http://127.0.0.1:9/b/k.csv?X-Amz-Signature=SECRET_compat_uri', 'CSV') SETTINGS compatibility_s3_presigned_url_query_in_path = 1; -- { serverError BAD_ARGUMENTS } +-- ... so it ends up in the key... +SELECT count() FROM s3('http://my-bucket-name.s3.amazonaws.com//a//b.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=SECRET_compat_key', 'CSV') SETTINGS compatibility_s3_presigned_url_query_in_path = 1; -- { serverError BAD_ARGUMENTS } +-- ... or in the bucket. +SELECT count() FROM s3('http://127.0.0.1:9/bucketname?X-Amz-Signature=SECRET_compat_bucket_0123456789abcdef0123456789abcdef', 'CSV') SETTINGS compatibility_s3_presigned_url_query_in_path = 1; -- { serverError BAD_ARGUMENTS } +-- The same through a table engine. +CREATE TABLE s3_engine (x UInt8) ENGINE = S3('http://duser:SECRET_engine@127.0.0.1:9/b/x.csv', 'CSV'); -- { serverError BAD_ARGUMENTS } + +SYSTEM FLUSH LOGS query_log; + +SELECT type, position(exception, 'SECRET_') > 0 AS leaked, extract(exception, 'DB::Exception: (.*)\\. \\(BAD_ARGUMENTS\\)') AS message +FROM system.query_log +WHERE current_database = currentDatabase() AND type != 'QueryStart' AND exception_code = 36 +ORDER BY event_time_microseconds; From 26726010e8ca0e02f612dfdd9fe53ea08d6a03d1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 20:48:24 +0000 Subject: [PATCH 127/296] Backport #119244 to 26.8: Do not log kafka secrets --- contrib/librdkafka-cmake/CMakeLists.txt | 3 +- .../include/chrdkafka_conf_sensitive.h | 14 +++++ .../librdkafka-cmake/rdkafka_conf_sensitive.c | 31 +++++++++++ src/Storages/Kafka/KafkaConfigLoader.cpp | 41 ++++++++++++--- .../test_storage_kafka/test_batch_fast.py | 51 +++++++++++++++++++ 5 files changed, 131 insertions(+), 9 deletions(-) create mode 100644 contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h create mode 100644 contrib/librdkafka-cmake/rdkafka_conf_sensitive.c diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 57ccb00086b02..b2945e968d8d6 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -32,7 +32,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cert.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c" - "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" +# "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" # compiled via the rdkafka_conf_sensitive.c wrapper below "${RDKAFKA_SOURCE_DIR}/rdkafka_coord.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_error.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_event.c" @@ -84,6 +84,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_txnmgr.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_zstd.c" # WITH_ZSTD + "${CMAKE_CURRENT_SOURCE_DIR}/rdkafka_conf_sensitive.c" # ClickHouse wrapper compiling rdkafka_conf.c, see the comment in the file "${RDKAFKA_SOURCE_DIR}/rdlist.c" "${RDKAFKA_SOURCE_DIR}/rdlog.c" "${RDKAFKA_SOURCE_DIR}/rdmap.c" diff --git a/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h b/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h new file mode 100644 index 0000000000000..83b99dce955ca --- /dev/null +++ b/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h @@ -0,0 +1,14 @@ +/// See contrib/librdkafka-cmake/rdkafka_conf_sensitive.c +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/// The names of the configuration properties that librdkafka marks with the _RK_SENSITIVE flag, +/// i.e. whose values must not appear in logs, as a NULL-terminated array of static strings. +const char * const * chrd_kafka_conf_sensitive_properties(void); + +#ifdef __cplusplus +} +#endif diff --git a/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c b/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c new file mode 100644 index 0000000000000..740f444ad0a17 --- /dev/null +++ b/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c @@ -0,0 +1,31 @@ +/// Wrapper around librdkafka's rdkafka_conf.c: compiles the file (this is the only place it is +/// compiled, see the SRCS list in contrib/librdkafka-cmake/CMakeLists.txt) and adds an accessor +/// for the static rd_kafka_properties table, which is the source of truth for the _RK_SENSITIVE +/// flag. ClickHouse uses the flag to hide the values of sensitive properties (e.g. sasl.password) +/// in its logs. No public librdkafka API exposes the flag: the redacting dump +/// (rd_kafka_anyconf_dump with redact_sensitive) is static, and the public rd_kafka_conf_dump +/// does not redact. + +#include "rdkafka_conf.c" + +#include + +/* The table ends with a terminator entry, so the count is an upper bound and the + * array is always NULL-terminated. The names point into the static table. */ +static const char * chrd_sensitive_names[sizeof(rd_kafka_properties) / sizeof(*rd_kafka_properties)]; +static pthread_once_t chrd_sensitive_names_once = PTHREAD_ONCE_INIT; + +static void chrd_fill_sensitive_names(void) +{ + const struct rd_kafka_property * prop = NULL; + size_t n = 0; + for (prop = rd_kafka_properties; prop->name; prop++) + if (prop->scope & _RK_SENSITIVE) + chrd_sensitive_names[n++] = prop->name; +} + +const char * const * chrd_kafka_conf_sensitive_properties(void) +{ + pthread_once(&chrd_sensitive_names_once, chrd_fill_sensitive_names); + return chrd_sensitive_names; +} diff --git a/src/Storages/Kafka/KafkaConfigLoader.cpp b/src/Storages/Kafka/KafkaConfigLoader.cpp index ded88d65b90ab..cb26fad595048 100644 --- a/src/Storages/Kafka/KafkaConfigLoader.cpp +++ b/src/Storages/Kafka/KafkaConfigLoader.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -552,6 +553,36 @@ void updateConfigurationFromConfig( } +namespace +{ + +/// Sensitive properties must not be logged in cleartext: the log records can reach not only the +/// server log, but also clients that set `send_logs_level`. +bool isSensitiveProperty(std::string_view name) +{ + /// The properties librdkafka marks with the _RK_SENSITIVE flag, plus a substring safety net + /// for properties unknown to the vendored librdkafka version. + static const std::unordered_set sensitive_properties = [] + { + std::unordered_set res; + for (const char * const * prop_name = chrd_kafka_conf_sensitive_properties(); *prop_name; ++prop_name) + res.emplace(*prop_name); + return res; + }(); + return sensitive_properties.contains(name) || name.contains("password") || name.contains("secret"); +} + +/// Log all properties of a Kafka client configuration, replacing the values of sensitive +/// properties, e.g. `sasl.password` or `sasl.oauthbearer.client.secret`, with `[HIDDEN]`. +void logConfigProperties(const cppkafka::Configuration & conf, const LoggerPtr & log, std::string_view client_type) +{ + for (const auto & property : conf.get_all()) + LOG_TRACE(log, "{} set property {}:{}", client_type, property.first, + isSensitiveProperty(property.first) ? "[HIDDEN]" : property.second); +} + +} + template cppkafka::Configuration KafkaConfigLoader::getConsumerConfiguration(TKafkaStorage & storage, const ConsumerConfigParams & params, IKafkaExceptionInfoSinkPtr exception_info_sink_ptr) { @@ -582,12 +613,7 @@ cppkafka::Configuration KafkaConfigLoader::getConsumerConfiguration(TKafkaStorag conf.set("enable.auto.offset.store", "false"); // Update offset automatically - to commit them all at once. conf.set("enable.partition.eof", "false"); // Ignore EOF messages - for (auto & property : conf.get_all()) - { - if (property.first.contains("password")) - continue; - LOG_TRACE(params.log, "Consumer set property {}:{}", property.first, property.second); - } + logConfigProperties(conf, params.log, "Consumer"); return conf; } @@ -608,8 +634,7 @@ cppkafka::Configuration KafkaConfigLoader::getProducerConfiguration(TKafkaStorag updateConfigurationFromConfig(loadProducerConfig, conf, storage, params); - for (auto & property : conf.get_all()) - LOG_TRACE(params.log, "Producer set property {}:{}", property.first, property.second); + logConfigProperties(conf, params.log, "Producer"); /// compression.codec is a global and topic level property, however compression.level is only a topic level property. /// cppkafka::Configuration::get_all returns the global properties only, so we need to check compression.level separately. diff --git a/tests/integration/test_storage_kafka/test_batch_fast.py b/tests/integration/test_storage_kafka/test_batch_fast.py index c36d42a7b0dfe..f55249b5a7d63 100644 --- a/tests/integration/test_storage_kafka/test_batch_fast.py +++ b/tests/integration/test_storage_kafka/test_batch_fast.py @@ -1931,6 +1931,57 @@ def test_kafka_producer_consumer_separate_settings( assert property_in_log in kafka_producer_applied_properties +@pytest.mark.parametrize( + "create_query_generator", + [ + k.generate_old_create_table_query, + k.generate_new_create_table_query, + ], +) +def test_kafka_password_not_logged(kafka_cluster, create_query_generator): + suffix = k.random_string(6) + kafka_table = f"kafka_{suffix}" + username = f"kafka_user_{suffix}" + password = f"secret_kafka_password_{suffix}" + + instance.rotate_logs() + instance.query( + create_query_generator( + kafka_table, + "key UInt64", + topic_list="password_not_logged", + consumer_group="test", + settings={ + "kafka_sasl_username": username, + "kafka_sasl_password": password, + }, + ) + ) + + # Create an mv to initialize the librdkafka consumers + instance.query(f"CREATE MATERIALIZED VIEW test.{kafka_table}_view ENGINE=MergeTree ORDER BY tuple() AS SELECT * FROM test.{kafka_table}") + instance.wait_for_log_line(f"{kafka_table}.*Created #0 consumer") + instance.query(f"DROP TABLE test.{kafka_table}_view") + instance.query(f"INSERT INTO test.{kafka_table} VALUES (1)") + + assert instance.contains_in_log(f"{kafka_table}.*Kafka producer created") + + # The property-logging loops ran for both the consumer and the producer, + # but they hid the values of the sensitive properties. `sasl.username` is + # hidden because librdkafka marks it with the _RK_SENSITIVE flag, not + # because of the name, so it validates the generated blacklist. + for client_type in ["Consumer", "Producer"]: + for property_name in ["sasl.username", "sasl.password"]: + assert instance.contains_in_log( + f"{kafka_table}.*{client_type} set property {property_name}:\\[HIDDEN\\]" + ) + # The username still appears in the logged CREATE TABLE text (only + # kafka_sasl_password is masked there), so check only the password value. + assert not instance.contains_in_log(password) + + instance.query(f"DROP TABLE test.{kafka_table}") + + @pytest.mark.parametrize( "create_query_generator, log_line", [ From 0540a1e3da97ea7429f97f78a22a80a094fb9947 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 29 Sep 2026 20:57:46 +0000 Subject: [PATCH 128/296] Backport #122423 to 26.8: Mask 'role_session_name' in system.query_log --- .../accessing-s3-data-securely.mdx | 1 + src/Backups/BackupInfo.cpp | 31 ++++-- src/Databases/DataLake/DataLakeConstants.h | 3 + src/Parsers/FunctionSecretArgumentsFinder.h | 28 ++++-- .../test_backup_restore_s3_role_arn/test.py | 8 +- tests/integration/test_database_glue/test.py | 93 +++++++++++++++++ .../test_storage_s3_queue/test_sts_smoke.py | 6 +- ...3_explicit_url_named_secret_mask.reference | 2 +- ...4510_s3_explicit_url_named_secret_mask.sql | 4 +- ...ake_catalog_hide_aws_external_id.reference | 8 +- ...0_datalake_catalog_hide_aws_external_id.sh | 18 ++-- ...edentials_role_session_name_mask.reference | 38 +++++++ ...tra_credentials_role_session_name_mask.sql | 84 ++++++++++++++++ ...ackup_persists_role_session_name.reference | 20 ++++ ...91_s3_backup_persists_role_session_name.sh | 99 +++++++++++++++++++ 15 files changed, 402 insertions(+), 41 deletions(-) create mode 100644 tests/queries/0_stateless/05255_s3_extra_credentials_role_session_name_mask.reference create mode 100644 tests/queries/0_stateless/05255_s3_extra_credentials_role_session_name_mask.sql create mode 100644 tests/queries/0_stateless/05291_s3_backup_persists_role_session_name.reference create mode 100755 tests/queries/0_stateless/05291_s3_backup_persists_role_session_name.sh diff --git a/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx b/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx index d8002f31a5f2e..5c13a3cb8a13f 100644 --- a/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx +++ b/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx @@ -141,6 +141,7 @@ DESCRIBE TABLE s3('https://s3.amazonaws.com/BUCKETNAME/BUCKETOBJECT.csv','CSVWit Below is an example query that uses the `role_session_name` as a shared secret to query data from a bucket. If the `role_session_name` isn't correct, this operation will fail. +Because it can act as a shared secret, ClickHouse masks the value of `role_session_name` as `[HIDDEN]` in `system.query_log`, `SHOW CREATE TABLE`, and everywhere else the query text is shown, the same way as `external_id`. ```sql DESCRIBE TABLE s3('https://s3.amazonaws.com/BUCKETNAME/BUCKETOBJECT.csv','CSVWithNames',extra_credentials(role_arn = 'arn:aws:iam::111111111111:role/ClickHouseAccessRole-001', role_session_name = 'secret-role-name')) diff --git a/src/Backups/BackupInfo.cpp b/src/Backups/BackupInfo.cpp index 755027d190753..5b68b55a10e4f 100644 --- a/src/Backups/BackupInfo.cpp +++ b/src/Backups/BackupInfo.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -152,11 +151,22 @@ namespace return evaluated_literal->value.safeGet(); } - /// Rebuilds an `extra_credentials(...)` keeping only its non-secret keys, or `nullptr` if none are - /// left. `role_arn` and `role_session_name` only name the role to assume, which grants nothing - /// without the server's own identity and a matching trust policy, so `isNonSecretExtraCredentialsKey` - /// keeps them -- the same predicate that keeps them visible in a logged query. `external_id` is the - /// shared secret of the triple, and anything unclassifiable is dropped. + /// The `extra_credentials(...)` keys that the `` locator keeps in the `.backup` metadata: + /// the role identifiers `role_arn` and `role_session_name`. Every backup of a chain must be able to + /// reopen its base with the stored locator alone, and a trust policy may pin the session name through + /// `sts:RoleSessionName`, so dropping it would break such restores (see #116223). This deliberately + /// differs from the logged query text, where `role_session_name` is masked and only `role_arn` stays + /// visible (`FunctionSecretArgumentsFinder::isNonSecretExtraCredentialsKey`): the metadata file lives + /// in the user's own backup bucket, next to the data it protects. `external_id` is the shared secret + /// of the triple and is dropped, as is anything unclassifiable. + bool isBaseBackupRoleIdentifierKey(std::string_view key) + { + return key == "role_arn" || key == "role_session_name"; + } + + /// Rebuilds an `extra_credentials(...)` keeping only its role identifiers, or `nullptr` if none are + /// left. They only name the role to assume and its session, which grants nothing without the server's + /// own identity and a matching trust policy. ASTPtr withoutSecretExtraCredentials(const ASTPtr & function_arg, const ContextPtr & context) { const auto * func = function_arg->as(); @@ -168,7 +178,7 @@ namespace for (const auto & child : func->arguments->children) { auto key = getEffectiveKeyValueArgName(child, context); - if (!key || !FunctionSecretArgumentsFinder::isNonSecretExtraCredentialsKey(*key)) + if (!key || !isBaseBackupRoleIdentifierKey(*key)) continue; /// The value is only classified, not rewritten: an expression the open path resolves stays as /// it was written, so a locator that loses nothing serializes byte for byte. @@ -418,9 +428,10 @@ BackupInfo BackupInfo::withoutS3Credentials(ContextPtr context) const /// S3(collection, secret_access_key = '...') -> S3(collection) /// The keys are the `S3` authentication arguments consumed by `registerBackupEngineS3` - /// and `S3StorageParsedArguments::collectCredentials`, minus the non-secret role identifiers, which - /// stay so that a role-authenticated base backup remains openable. The key is resolved with the - /// context, so that an expression key (e.g. concat('secret_', 'access_key')) is recognized as well. + /// and `S3StorageParsedArguments::collectCredentials`, minus the role identifiers (`role_arn`, + /// `role_session_name`), which stay so that a role-authenticated base backup remains openable (see + /// isBaseBackupRoleIdentifierKey). The key is resolved with the context, so that an expression key + /// (e.g. concat('secret_', 'access_key')) is recognized as well. res.kv_args.erase( std::remove_if( res.kv_args.begin(), diff --git a/src/Databases/DataLake/DataLakeConstants.h b/src/Databases/DataLake/DataLakeConstants.h index bc5c96ac093f0..93156b46fd0e7 100644 --- a/src/Databases/DataLake/DataLakeConstants.h +++ b/src/Databases/DataLake/DataLakeConstants.h @@ -29,11 +29,14 @@ static inline std::unordered_map SETTINGS_TO_HIDE = {"aws_access_key_id", DEFAULT_MASKING_RULE}, {"aws_secret_access_key", DEFAULT_MASKING_RULE}, {"aws_external_id", DEFAULT_MASKING_RULE}, + /// A trust policy can require a specific session name (`sts:RoleSessionName`), so it is a secret too. + {"aws_role_session_name", DEFAULT_MASKING_RULE}, /// Legacy storage_* aliases (declared in DataLakeStorageSettings.h, originally for the Glue catalog) {"storage_catalog_credential", DEFAULT_MASKING_RULE}, {"storage_auth_header", DEFAULT_MASKING_RULE}, {"storage_aws_access_key_id", DEFAULT_MASKING_RULE}, {"storage_aws_secret_access_key", DEFAULT_MASKING_RULE}, + {"storage_aws_role_session_name", DEFAULT_MASKING_RULE}, /// OneLake credentials {"onelake_client_secret", DEFAULT_MASKING_RULE}, {"onelake_bearer_token", DEFAULT_MASKING_RULE}, diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index ee6b4922923b5..821c215939aaa 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -87,11 +87,17 @@ class FunctionSecretArgumentsFinder FunctionSecretArgumentsFinder::Result getResult() const { return result; } /// Whether a key of the `extra_credentials(..)` nested map carries a non-secret identifier whose - /// value stays visible when the map is masked (`role_arn` and `role_session_name`; the map's - /// secret is `external_id`). Any other key - unknown, malformed or an expression - fails closed. + /// value stays visible when the map is masked. Only `role_arn` qualifies: it names the role to + /// assume, like `access_key_id` names a key. The other two keys of the assume-role triple are + /// secrets: `external_id` is its shared secret, and `role_session_name` can be one too, because a + /// trust policy can require a specific value through the `sts:RoleSessionName` condition (the + /// ClickHouse Cloud guide documents exactly this use). Any other key - unknown, malformed or an + /// expression - fails closed. + /// The `.backup` metadata is a different matter: its `` locator keeps `role_session_name` + /// on purpose, so that a role-authenticated backup chain stays restorable (see `BackupInfo.cpp`). static bool isNonSecretExtraCredentialsKey(std::string_view key) { - return key == "role_arn" || key == "role_session_name"; + return key == "role_arn"; } protected: @@ -99,11 +105,12 @@ class FunctionSecretArgumentsFinder Result result; /// Named arguments carrying S3 secrets, shared by every S3 form (explicit-url and named-collection). - /// `external_id` is the shared secret of the assume-role triple; the other two (`role_arn`, - /// `role_session_name`) are non-secret identifiers passed inside `extra_credentials` and stay - /// visible (see isNonSecretExtraCredentialsKey). + /// `external_id` and `role_session_name` are the secrets of the assume-role triple; the third key + /// (`role_arn`) is a non-secret identifier passed inside `extra_credentials` and stays visible + /// (see isNonSecretExtraCredentialsKey). static constexpr std::string_view s3_secret_keys[] - = {"secret_access_key", "session_token", "google_adc_client_secret", "google_adc_refresh_token", "external_id"}; + = {"secret_access_key", "session_token", "google_adc_client_secret", "google_adc_refresh_token", "external_id", + "role_session_name"}; /// Named arguments carrying TLS credentials as the literal contents of a certificate or a key file, /// rather than as a path to it. They are secret and have to be hidden the same way a password is. @@ -120,9 +127,10 @@ class FunctionSecretArgumentsFinder void markSecretArgument(size_t index, bool argument_is_named = false); /// `headers(..)` and `extra_credentials(..)` are nested maps whose values are secret auth material - /// (`extra_credentials` carries the assume-role secret `external_id`; its non-secret identifiers - /// stay visible, see isNonSecretExtraCredentialsKey). The parsers accept them at any position, not - /// just at the tail. Record them so their values are hidden with the keys kept. + /// (`extra_credentials` carries the assume-role secrets `external_id` and `role_session_name`; its + /// non-secret identifier `role_arn` stays visible, see isNonSecretExtraCredentialsKey). The parsers + /// accept them at any position, not just at the tail. Record them so their values are hidden with + /// the keys kept. /// Idempotent: each map is recorded at most once. void maskNestedSecretMaps(); diff --git a/tests/integration/test_backup_restore_s3_role_arn/test.py b/tests/integration/test_backup_restore_s3_role_arn/test.py index a4ed1c43ca7dd..8c4126518e29a 100644 --- a/tests/integration/test_backup_restore_s3_role_arn/test.py +++ b/tests/integration/test_backup_restore_s3_role_arn/test.py @@ -9,9 +9,11 @@ from helpers.cluster import ClickHouseCluster from helpers.mock_servers import start_mock_servers -# `role_arn` and `role_session_name` are not secrets -- assuming the role still needs the server's -# own identity and a matching trust policy -- so they survive into the `` locator and -# let every hop of a chain reopen its base. The mock STS accepts the role and session name below. +# `role_arn` and `role_session_name` survive into the `` locator and let every hop of a +# chain reopen its base: assuming the role still needs the server's own identity and a matching trust +# policy. The session name is masked in logged query text, because a trust policy can pin it, but the +# stored locator keeps it on purpose (see `BackupInfo.cpp`). The mock STS accepts the role and +# session name below. # # Metadata written by a version that stripped the identifiers names no credentials and carries no # marker to reconstruct them from, so its base backup opens unauthenticated. Such a chain must still diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index 0254ff11fa8ad..50098dc9bae6d 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -1371,11 +1371,104 @@ def test_sts_smoke(started_cluster): result = node.query(f"SELECT sum(value) FROM {db_name_success}.`{root_namespace}.{table_name}`") assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + # `aws_role_session_name` can act as a shared secret (the trust policy can pin it, which is what the + # STS mock does), so every display surface hides it while `aws_role_arn` stays visible. This database + # carries no other secret, so `[HIDDEN]` can only come from the session name. + show_create = node.query(f"SHOW CREATE DATABASE {db_name_success}") + assert "miniorole" not in show_create + assert "arn::role" in show_create + assert "[HIDDEN]" in show_create + + engine_full = node.query( + f"SELECT engine_full FROM system.databases WHERE name = '{db_name_success}'" + ) + assert "miniorole" not in engine_full + assert "[HIDDEN]" in engine_full + + node.query("SYSTEM FLUSH LOGS system.query_log") + logged_create = node.query( + f"SELECT arrayStringConcat(groupArray(query), '\\n') FROM system.query_log " + f"WHERE query_kind = 'Create' AND type = 'QueryFinish' AND query LIKE '%{db_name_success}%'" + ) + assert "[HIDDEN]" in logged_create + assert "miniorole" not in logged_create + # Cleanup node.query(f"DROP DATABASE IF EXISTS {db_name_fail} SYNC") node.query(f"DROP DATABASE IF EXISTS {db_name_success} SYNC") +def test_sts_backup_restore_keeps_role_session_name(started_cluster): + """A backup archives the database definition with the real `aws_role_session_name`, although every + display surface shows `[HIDDEN]`. The STS mock grants the role only for the session name `miniorole`, + so the restored catalog can read its table only if the archived value was the real one.""" + node = started_cluster.instances["node1"] + + test_ref = f"test_sts_backup_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + + schema = Schema( + NestedField(field_id=1, name="id", field_type=StringType(), required=False), + NestedField(field_id=2, name="value", field_type=DoubleType(), required=False), + ) + table = create_table(catalog, root_namespace, table_name, schema, PartitionSpec(), DEFAULT_SORT_ORDER, dir=table_name) + table.append( + pa.Table.from_pylist( + [ + {"id": "row1", "value": 10.0}, + {"id": "row2", "value": 20.0}, + {"id": "row3", "value": 30.0}, + ] + ) + ) + + db_name = f"db_backup_{test_ref.replace('-', '_')}" + restored_db_name = f"{db_name}_restored" + create_clickhouse_glue_database( + started_cluster, + node, + db_name, + additional_settings={ + "aws_role_arn": "arn::role", + "aws_role_session_name": "miniorole", + }, + query_settings={"s3_allow_server_credentials_in_user_queries": 1}, + with_credentials=False, + ) + result = node.query(f"SELECT sum(value) FROM {db_name}.`{root_namespace}.{table_name}`") + assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + + # A DataLakeCatalog database owns no tables, so the backup holds just the database definition. The + # default server config allows File backups under its `backups` directory. + backup = f"File('{db_name}')" + node.query(f"BACKUP DATABASE {db_name} TO {backup}") + node.query(f"DROP DATABASE IF EXISTS {restored_db_name} SYNC") + node.query( + f"RESTORE DATABASE {db_name} AS {restored_db_name} FROM {backup}", + settings={ + "allow_database_glue_catalog": 1, + "s3_allow_server_credentials_in_user_queries": 1, + }, + ) + + # Readable only with the real session name: the STS mock rejects any other value. + result = node.query(f"SELECT sum(value) FROM {restored_db_name}.`{root_namespace}.{table_name}`") + assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + + # The archived value came back, and the display of the restored definition is masked like any other. + show_create = node.query(f"SHOW CREATE DATABASE {restored_db_name}") + assert "miniorole" not in show_create + assert "arn::role" in show_create + assert "[HIDDEN]" in show_create + + node.query(f"DROP DATABASE IF EXISTS {restored_db_name} SYNC") + node.query(f"DROP DATABASE IF EXISTS {db_name} SYNC") + + def test_sts_smoke_no_opt_in(started_cluster): """A Glue DataLakeCatalog with aws_role_arn set, no explicit aws_access_key_id/aws_secret_access_key, and no s3_allow_server_credentials_in_user_queries opt-in. GlueCatalog routes through the same diff --git a/tests/integration/test_storage_s3_queue/test_sts_smoke.py b/tests/integration/test_storage_s3_queue/test_sts_smoke.py index 761606c336af9..5b02ffebace71 100644 --- a/tests/integration/test_storage_s3_queue/test_sts_smoke.py +++ b/tests/integration/test_storage_s3_queue/test_sts_smoke.py @@ -111,14 +111,14 @@ def get_count(node, table_name): assert get_count(node, dst_table_name) == 10 assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) node.restart_clickhouse() assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) @@ -181,7 +181,7 @@ def test_s3_queue_extra_credentials_backup(started_cluster): # are equally invalid for MinIO, so a restored table that silently dropped # the clause would fail with the same error. Pin the round trip explicitly. assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference index 0b581d50ec4a4..5aa390bf9bb08 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference @@ -138,7 +138,7 @@ UNION id: 0, union_mode: UNION_ALL CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8 JOIN TREE IDENTIFIER id: 15, identifier: system.one --- session_token, the Google ADC secrets (google_adc_client_secret, google_adc_refresh_token) and\n-- the extra_credentials assume-role material (external_id) passed to the explicit-url or\n-- named-collection S3 form must be masked like secret_access_key. Every secret value below is tagged\n-- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE\n-- and logged query text.\n\n-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifiers\n-- (role_arn, role_session_name) stay visible while external_id is hidden.\nDROP TABLE IF EXISTS t_04510; +-- session_token, the Google ADC secrets (google_adc_client_secret, google_adc_refresh_token) and\n-- the extra_credentials assume-role material (external_id) passed to the explicit-url or\n-- named-collection S3 form must be masked like secret_access_key. Every secret value below is tagged\n-- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE\n-- and logged query text.\n\n-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifier\n-- (role_arn) stays visible while external_id is hidden.\nDROP TABLE IF EXISTS t_04510; CREATE TABLE t_04510 (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510\', \'ak\', \'[HIDDEN]\', session_token = \'[HIDDEN]\', google_adc_client_secret = \'[HIDDEN]\', google_adc_refresh_token = \'[HIDDEN]\', extra_credentials(role_arn = \'visible_role_arn\', external_id = \'[HIDDEN]\'), format = \'TSV\') SHOW CREATE TABLE t_04510 SETTINGS format_display_secrets_in_show_and_select = 0; DROP TABLE t_04510; diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql index f3258d0805dfb..4f7eee6fb5109 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql @@ -7,8 +7,8 @@ -- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE -- and logged query text. --- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifiers --- (role_arn, role_session_name) stay visible while external_id is hidden. +-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifier +-- (role_arn) stays visible while external_id is hidden. DROP TABLE IF EXISTS t_04510; CREATE TABLE t_04510 (x UInt8) ENGINE = S3('http://localhost:11111/test/04510', 'ak', 'SEKRIT_SAK', diff --git a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference index dff69852207d2..ba2bb9215f919 100644 --- a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference +++ b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference @@ -1,13 +1,13 @@ ---- default: aws_external_id hidden, role identifiers visible +--- default: aws_external_id and aws_role_session_name hidden, aws_role_arn visible aws_external_id = '[HIDDEN]' aws_access_key_id = '[HIDDEN]' aws_secret_access_key = '[HIDDEN]' aws_role_arn = 'arn:aws:iam::1:role/r' -aws_role_session_name = 'sess' +aws_role_session_name = '[HIDDEN]' OK: no secret in formatted query ---- show_secrets: aws_external_id visible +--- show_secrets: aws_external_id and aws_role_session_name visible aws_external_id = 'SECRET_THAT_MUST_NOT_LEAK' aws_access_key_id = 'SECRET_THAT_MUST_NOT_LEAK' aws_secret_access_key = 'SECRET_THAT_MUST_NOT_LEAK' aws_role_arn = 'arn:aws:iam::1:role/r' -aws_role_session_name = 'sess' +aws_role_session_name = 'SECRET_THAT_MUST_NOT_LEAK' diff --git a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh index f33348fb92d45..d846eca5fa41e 100755 --- a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh +++ b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash -# Regression test: aws_external_id is the shared secret of the AWS AssumeRole triple, so it must be -# redacted as [HIDDEN] when a DataLakeCatalog CREATE query is formatted (system.databases.engine_full, -# SHOW CREATE DATABASE), while aws_role_arn and aws_role_session_name are non-secret identifiers that -# stay visible. Uses clickhouse-format so it needs no live catalog and is safe to run in parallel. +# Regression test: aws_external_id is the shared secret of the AWS AssumeRole triple, and +# aws_role_session_name can be one too (a trust policy can require a specific value through the +# sts:RoleSessionName condition), so both must be redacted as [HIDDEN] when a DataLakeCatalog CREATE +# query is formatted (system.databases.engine_full, SHOW CREATE DATABASE), while aws_role_arn is a +# non-secret identifier that stays visible. Uses clickhouse-format so it needs no live catalog and is +# safe to run in parallel. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -17,7 +19,7 @@ aws_access_key_id = '${SECRET}', aws_secret_access_key = '${SECRET}', aws_external_id = '${SECRET}', aws_role_arn = 'arn:aws:iam::1:role/r', -aws_role_session_name = 'sess'" +aws_role_session_name = '${SECRET}'" show_settings() { local formatted="$1" @@ -32,9 +34,9 @@ show_settings() { done } -# Arm A: at the default the secret is redacted; arms C (non-secret identifiers stay visible) and +# Arm A: at the default the secrets are redacted; arms C (the non-secret role ARN stays visible) and # D (sibling AWS keys still redacted) are asserted by the same output. -echo "--- default: aws_external_id hidden, role identifiers visible" +echo "--- default: aws_external_id and aws_role_session_name hidden, aws_role_arn visible" formatted=$(echo "$query" | $CLICKHOUSE_FORMAT --oneline) show_settings "$formatted" if echo "$formatted" | grep -q "$SECRET"; then @@ -44,6 +46,6 @@ else fi # Arm B: an authorized caller can still retrieve the value, so the fix redacts rather than destroys. -echo "--- show_secrets: aws_external_id visible" +echo "--- show_secrets: aws_external_id and aws_role_session_name visible" formatted=$(echo "$query" | $CLICKHOUSE_FORMAT --oneline --show_secrets) show_settings "$formatted" diff --git a/tests/queries/0_stateless/05255_s3_extra_credentials_role_session_name_mask.reference b/tests/queries/0_stateless/05255_s3_extra_credentials_role_session_name_mask.reference new file mode 100644 index 0000000000000..2d56e8b432795 --- /dev/null +++ b/tests/queries/0_stateless/05255_s3_extra_credentials_role_session_name_mask.reference @@ -0,0 +1,38 @@ +CREATE TABLE default.t_05255\n(\n `x` UInt8\n)\nENGINE = S3(\'http://localhost:11111/test/05255\', \'ak\', \'[HIDDEN]\', \'TSV\', format = \'TSV\', extra_credentials(\'role_arn\' = \'visible_role_arn\', \'role_session_name\' = \'[HIDDEN]\', \'external_id\' = \'[HIDDEN]\')) +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: s3 + ARGUMENTS + LIST id: 4, nodes: 3 + CONSTANT id: 5, constant_value: \'http://localhost:11111/test/05255qt\', constant_value_type: String + CONSTANT id: 6, constant_value: \'Parquet\', constant_value_type: String + FUNCTION id: 7, function_name: extra_credentials, function_type: ordinary + ARGUMENTS + LIST id: 8, nodes: 2 + FUNCTION id: 9, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 10, nodes: 2 + IDENTIFIER id: 11, identifier: role_arn + CONSTANT id: 12, constant_value: \'visible_role_arn\', constant_value_type: String + FUNCTION id: 13, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 14, nodes: 2 + IDENTIFIER id: 15, identifier: role_session_name + CONSTANT id: 16, constant_value: [HIDDEN], constant_value_type: String +-- role_session_name inside extra_credentials(...) can act as a shared secret: a role\'s trust policy\n-- can require a specific value through the sts:RoleSessionName condition, and the ClickHouse Cloud\n-- guide documents exactly this use. It must therefore be masked like external_id, while role_arn is\n-- a non-secret identifier that stays visible. Every secret value below is tagged so the final\n-- assertion can prove none of them leaks. role_session_name used to be logged in plaintext.\n\n-- Engine form: SHOW CREATE hides role_session_name and external_id and keeps role_arn.\nDROP TABLE IF EXISTS t_05255; +CREATE TABLE t_05255 (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/05255\', \'ak\', \'[HIDDEN]\', extra_credentials(role_arn = \'visible_role_arn\', role_session_name = \'[HIDDEN]\', external_id = \'[HIDDEN]\'), format = \'TSV\') +SHOW CREATE TABLE t_05255 SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_05255; +SELECT * FROM s3(\'url_rsn\', \'Parquet\', extra_credentials(role_arn = \'arn:aws:iam::123456789012:role/visible_role\', role_session_name = \'[HIDDEN]\')) +SELECT * FROM s3(\'url_rsn_triple\', \'ak\', \'[HIDDEN]\', extra_credentials(role_session_name = \'[HIDDEN]\', role_arn = \'visible_role_arn\', external_id = \'[HIDDEN]\'), format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(\'url_rsn_ident\', \'ak\', \'[HIDDEN]\', extra_credentials(role_arn = visible_role_arn, role_session_name = \'[HIDDEN]\'), format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(nc_05255_missing, role_session_name = \'[HIDDEN]\', format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(nc_05255_missing, extra_credentials(role_arn = \'visible_role_arn\', role_session_name = \'[HIDDEN]\'), format = \'TSV\', structure = \'x UInt8\') +BACKUP TABLE nonexistent_05255 TO S3(\'url_bkp_rsn\', \'ak\', \'[HIDDEN]\', extra_credentials(equals(role_arn, \'visible_role_arn\'), role_session_name = \'[HIDDEN]\')) +BACKUP TABLE nonexistent_05255 TO S3(nc_bkp_05255_missing, role_session_name = \'[HIDDEN]\') +CREATE DATABASE db_05255_rsn ENGINE = Backup(\'\', S3(\'url_dbrsn\', \'ak\', \'[HIDDEN]\', extra_credentials(role_arn = \'visible_role_arn\', role_session_name = \'[HIDDEN]\'))) +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/05255qt\', \'Parquet\', extra_credentials(role_arn = \'visible_role_arn\', role_session_name = \'[HIDDEN]\')) +1 0 diff --git a/tests/queries/0_stateless/05255_s3_extra_credentials_role_session_name_mask.sql b/tests/queries/0_stateless/05255_s3_extra_credentials_role_session_name_mask.sql new file mode 100644 index 0000000000000..2dde039ff1072 --- /dev/null +++ b/tests/queries/0_stateless/05255_s3_extra_credentials_role_session_name_mask.sql @@ -0,0 +1,84 @@ +-- Tags: no-fasttest +-- no-fasttest: the S3 table engine is not available in the fast test build. + +-- role_session_name inside extra_credentials(...) can act as a shared secret: a role's trust policy +-- can require a specific value through the sts:RoleSessionName condition, and the ClickHouse Cloud +-- guide documents exactly this use. It must therefore be masked like external_id, while role_arn is +-- a non-secret identifier that stays visible. Every secret value below is tagged so the final +-- assertion can prove none of them leaks. role_session_name used to be logged in plaintext. + +-- Engine form: SHOW CREATE hides role_session_name and external_id and keeps role_arn. +DROP TABLE IF EXISTS t_05255; +CREATE TABLE t_05255 (x UInt8) +ENGINE = S3('http://localhost:11111/test/05255', 'ak', 'SEKRIT_SAK', + extra_credentials(role_arn = 'visible_role_arn', role_session_name = 'SEKRIT_RSN', external_id = 'SEKRIT_EID'), + format = 'TSV'); +SHOW CREATE TABLE t_05255 SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_05255; + +-- The forms below all fail at analysis (empty host / missing collection) before any network access, +-- and are logged with the secret replaced. Each carries a unique marker checked by the final assertion. + +-- Explicit-url function form in the shape users write it: no access key, the role alone authenticates. +SELECT * FROM s3('url_rsn', 'Parquet', + extra_credentials(role_arn = 'arn:aws:iam::123456789012:role/visible_role', role_session_name = 'SEKRIT_RSN')); -- { serverError BAD_ARGUMENTS } + +-- Together with the other two keys of the triple, at any position inside the map. +SELECT * FROM s3('url_rsn_triple', 'ak', 'SEKRIT_SAK', + extra_credentials(role_session_name = 'SEKRIT_RSNFIRST', role_arn = 'visible_role_arn', external_id = 'SEKRIT_EID'), + format = 'TSV', structure = 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- The parser evaluates an identifier value as a literal, so an identifier carries the secret too. +SELECT * FROM s3('url_rsn_ident', 'ak', 'SEKRIT_SAK', + extra_credentials(role_arn = visible_role_arn, role_session_name = SEKRIT_IDRSN), + format = 'TSV', structure = 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- Named-collection form: role_session_name as a named override of the collection. +SELECT * FROM s3(nc_05255_missing, role_session_name = 'SEKRIT_NCRSN', + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- Named-collection form with the nested map. +SELECT * FROM s3(nc_05255_missing, extra_credentials(role_arn = 'visible_role_arn', role_session_name = 'SEKRIT_NCMAPRSN'), + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- BACKUP ... TO S3: the explicit-url locator with the nested map, and the named-collection locator +-- with a named override. +BACKUP TABLE nonexistent_05255 TO S3('url_bkp_rsn', 'ak', 'SEKRIT_SAK', + extra_credentials(role_arn = 'visible_role_arn', role_session_name = 'SEKRIT_BKPRSN')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_05255 TO S3(nc_bkp_05255_missing, + role_session_name = 'SEKRIT_BKPNCRSN'); -- { serverError BAD_ARGUMENTS } + +-- The Backup database engine reconstructs the nested S3 destination. +CREATE DATABASE db_05255_rsn ENGINE = Backup('', S3('url_dbrsn', 'ak', 'SEKRIT_SAK', + extra_credentials(role_arn = 'visible_role_arn', role_session_name = 'SEKRIT_DBRSN'))); -- { serverError BAD_ARGUMENTS } + +-- The query-tree surface (EXPLAIN QUERY TREE) must hide the same value while keeping role_arn. +-- run_passes = 0 keeps the table function unresolved, so the collection and the storage are not touched. +SET enable_analyzer = 1; +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3('http://localhost:11111/test/05255qt', 'Parquet', extra_credentials(role_arn = 'visible_role_arn', role_session_name = 'SEKRIT_QTRSN')); + +SYSTEM FLUSH LOGS query_log; + +-- The exact logged text of every query above, in execution order: role_session_name must appear as +-- '[HIDDEN]' while role_arn and every other non-secret part stay visible verbatim. Each query has +-- exactly one terminal event: QueryFinish for the successful ones, an exception event for the +-- rejected ones. +SELECT query +FROM system.query_log +WHERE current_database = currentDatabase() + AND type != 'QueryStart' + AND query_kind != 'Set' -- sent by the test harness, not by this test + AND query NOT ILIKE 'SYSTEM FLUSH%' -- its own terminal event races with the flush it performs + AND query_id = initial_query_id -- only the statements issued here: a Replicated database logs + -- each DDL again from the replay worker, which inherits the + -- initiator's initial_query_id but gets a fresh query_id + AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE +ORDER BY event_time_microseconds; + +-- Assert the masking property over every row this test produced, replay rows included. +-- count() > 0 keeps an empty row set from passing vacuously. +SELECT count() > 0, countIf(query LIKE '%SEKRIT%') +FROM system.query_log +WHERE current_database = currentDatabase() + AND type != 'QueryStart' + AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE; diff --git a/tests/queries/0_stateless/05291_s3_backup_persists_role_session_name.reference b/tests/queries/0_stateless/05291_s3_backup_persists_role_session_name.reference new file mode 100644 index 0000000000000..6b8a708e65622 --- /dev/null +++ b/tests/queries/0_stateless/05291_s3_backup_persists_role_session_name.reference @@ -0,0 +1,20 @@ +-- archived Backup database definition keeps role_session_name (must be 1) +1 +-- archived Backup database definition contains no [HIDDEN] (must be 0) +0 +-- archived S3 table definition keeps role_session_name (must be 1) +1 +-- archived S3 table definition keeps external_id (must be 1) +1 +-- archived S3 table definition contains no [HIDDEN] (must be 0) +0 +-- restored Backup database reads its source table (must be 10) +10 +-- secret occurrences in SHOW CREATE DATABASE of the restored Backup database (must be 0) +0 +-- [HIDDEN] present in SHOW CREATE DATABASE of the restored Backup database (must be 1) +1 +-- secret occurrences in SHOW CREATE TABLE of the restored S3 table (must be 0) +0 +-- [HIDDEN] present in SHOW CREATE TABLE of the restored S3 table (must be 1) +1 diff --git a/tests/queries/0_stateless/05291_s3_backup_persists_role_session_name.sh b/tests/queries/0_stateless/05291_s3_backup_persists_role_session_name.sh new file mode 100755 index 0000000000000..6068832388076 --- /dev/null +++ b/tests/queries/0_stateless/05291_s3_backup_persists_role_session_name.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-encrypted-storage +# Tag no-fasttest: requires the S3 endpoint +# Tag no-encrypted-storage: a backup from an encrypted disk restores only to an encrypted disk, so the restored Backup database gets no parts. + +# `role_session_name` is shown as [HIDDEN] wherever a definition is displayed. A backup must still +# archive the definition with the real value, because RESTORE recreates the object from the archived +# text: an S3 table or a Backup database restored with the literal '[HIDDEN]' would assume its role with +# the wrong session name. The archived definitions are read back from the backup itself, so the check +# does not go through the display layer that masks them. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +client_opts=( + --allow_repeated_settings + --send_logs_level 'error' +) + +src=${CLICKHOUSE_DATABASE}_src +view=${CLICKHOUSE_DATABASE}_view +view_restored=${CLICKHOUSE_DATABASE}_view_restored +tbl=${CLICKHOUSE_DATABASE}.t_rsn +tbl_restored=${CLICKHOUSE_DATABASE}.t_rsn_restored + +inner_url="http://localhost:11111/test/backups/${CLICKHOUSE_DATABASE}/rsn_inner" +outer_url="http://localhost:11111/test/backups/${CLICKHOUSE_DATABASE}/rsn_outer" +# Access key id 'test', secret 'testtest': the credentials the stateless suite uses for S3. +inner="S3('${inner_url}', 'test', 'testtest')" +# The Backup database mounts the inner backup with the same key pair plus a role_session_name and no +# role_arn: nothing assumes a role, so no STS endpoint is needed, while the value still travels through +# every place a locator is archived. +inner_with_session="S3('${inner_url}', 'test', 'testtest', extra_credentials(role_session_name = 'SEKRIT_DBRSN'))" +outer="S3('${outer_url}', 'test', 'testtest')" + +# The S3 table carries the full assume-role triple. Nothing reads it here, so no STS endpoint is needed +# either: the table function connects lazily, on the first read. +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP DATABASE IF EXISTS ${src}; +DROP DATABASE IF EXISTS ${view}; +DROP DATABASE IF EXISTS ${view_restored}; +DROP TABLE IF EXISTS ${tbl}; +DROP TABLE IF EXISTS ${tbl_restored}; +CREATE DATABASE ${src}; +CREATE TABLE ${src}.t (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO ${src}.t SELECT number FROM numbers(10); +BACKUP DATABASE ${src} TO ${inner} FORMAT Null; +CREATE DATABASE ${view} ENGINE = Backup('${src}', ${inner_with_session}); +CREATE TABLE ${tbl} (id UInt64) ENGINE = S3('http://localhost:11111/test/${CLICKHOUSE_DATABASE}/t_rsn', 'ak', 'sk', + extra_credentials(role_arn = 'arn::role', role_session_name = 'SEKRIT_TRSN', external_id = 'SEKRIT_TEID'), format = 'CSV'); +BACKUP DATABASE ${view}, TABLE ${tbl} TO ${outer} FORMAT Null; +" + +# Reads one archived definition out of the outer backup, bypassing every display surface. +function archived() +{ + local path_in_backup=$1 && shift + ${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q \ + "SELECT line FROM s3('${outer_url}/${path_in_backup}', 'test', 'testtest', 'LineAsString') FORMAT TSVRaw" +} + +echo '-- archived Backup database definition keeps role_session_name (must be 1)' +archived "metadata/${view}.sql" | grep -c SEKRIT_DBRSN || true +echo '-- archived Backup database definition contains no [HIDDEN] (must be 0)' +archived "metadata/${view}.sql" | grep -c HIDDEN || true +echo '-- archived S3 table definition keeps role_session_name (must be 1)' +archived "metadata/${CLICKHOUSE_DATABASE}/t_rsn.sql" | grep -c SEKRIT_TRSN || true +echo '-- archived S3 table definition keeps external_id (must be 1)' +archived "metadata/${CLICKHOUSE_DATABASE}/t_rsn.sql" | grep -c SEKRIT_TEID || true +echo '-- archived S3 table definition contains no [HIDDEN] (must be 0)' +archived "metadata/${CLICKHOUSE_DATABASE}/t_rsn.sql" | grep -c HIDDEN || true + +# The round trip: both objects come back from the archived text. The restored Backup database mounts +# the inner backup through its archived locator, so it is readable only if that locator was archived +# as written. +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q \ + "RESTORE DATABASE ${view} AS ${view_restored}, TABLE ${tbl} AS ${tbl_restored} FROM ${outer} FORMAT Null" + +echo '-- restored Backup database reads its source table (must be 10)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SELECT count() FROM ${view_restored}.t" + +# On display the restored definitions are masked like any other: the archived value never shows. +echo '-- secret occurrences in SHOW CREATE DATABASE of the restored Backup database (must be 0)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SHOW CREATE DATABASE ${view_restored}" | grep -c SEKRIT || true +echo '-- [HIDDEN] present in SHOW CREATE DATABASE of the restored Backup database (must be 1)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SHOW CREATE DATABASE ${view_restored}" | grep -c -m1 '\[HIDDEN\]' +echo '-- secret occurrences in SHOW CREATE TABLE of the restored S3 table (must be 0)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SHOW CREATE TABLE ${tbl_restored}" | grep -c SEKRIT || true +echo '-- [HIDDEN] present in SHOW CREATE TABLE of the restored S3 table (must be 1)' +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -q "SHOW CREATE TABLE ${tbl_restored}" | grep -c -m1 '\[HIDDEN\]' + +${CLICKHOUSE_CLIENT} "${client_opts[@]}" -m -q " +DROP TABLE ${tbl_restored}; +DROP TABLE ${tbl}; +DROP DATABASE ${view_restored}; +DROP DATABASE ${view}; +DROP DATABASE ${src}; +" From ba36b14823e91e1a9a2f44ab1ab91baca05863a4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 30 Sep 2026 07:53:08 +0000 Subject: [PATCH 129/296] Backport #122767 to 26.8: Keeper: reply session expired to a client that asks to continue its session --- src/Server/KeeperTCPHandler.cpp | 26 ++++++++---- src/Server/KeeperTCPHandler.h | 10 ++++- tests/integration/test_keeper_session/test.py | 42 ++++++++++++++++++- .../src/jepsen/clickhouse/keeper/counter.clj | 9 ++-- .../src/jepsen/clickhouse/keeper/queue.clj | 6 ++- .../src/jepsen/clickhouse/keeper/set.clj | 10 +++-- .../src/jepsen/clickhouse/keeper/utils.clj | 10 +++++ 7 files changed, 94 insertions(+), 19 deletions(-) diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index 3ae1047289407..1235b44bb4a2a 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -276,10 +276,10 @@ KeeperTCPHandler::KeeperTCPHandler( } } -void KeeperTCPHandler::sendHandshake(bool has_leader, bool & use_compression) +void KeeperTCPHandler::sendHandshake(HandshakeResult result, bool & use_compression) { Coordination::write(Coordination::SERVER_HANDSHAKE_LENGTH, *out); - if (has_leader) + if (result != HandshakeResult::Rejected) { if (expect_opentelemetry_tracing_context) Coordination::write(Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_TRACING, *out); @@ -298,8 +298,11 @@ void KeeperTCPHandler::sendHandshake(bool has_leader, bool & use_compression) Coordination::write(Coordination::KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT, *out); } - Coordination::write(static_cast(session_timeout.totalMilliseconds()), *out); - Coordination::write(session_id, *out); + /// A zero timeout with a zero session id tells a ZooKeeper client that its session has expired. + const bool expired = result == HandshakeResult::SessionExpired; + Coordination::write(expired ? int32_t{0} : static_cast(session_timeout.totalMilliseconds()), *out); + /// A rejected client has no session, and would send any non-zero id back as the session to continue. + Coordination::write(result == HandshakeResult::Accepted ? session_id : int64_t{0}, *out); std::array passwd{}; Coordination::write(passwd, *out); out->next(); @@ -315,7 +318,6 @@ Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length, bool int32_t protocol_version = 0; int64_t last_zxid_seen = 0; int32_t timeout_ms = 0; - int64_t previous_session_id = 0; /// We don't support session restore. So previous session_id is always zero. std::array passwd {}; if (!isHandShake(handshake_length)) @@ -465,6 +467,14 @@ void KeeperTCPHandler::runImpl() if (keeper_dispatcher->isTCPConnectionDrainStarted() || keeper_dispatcher->isShuttingDown()) return; + /// Keeper cannot restore sessions, and a new session in place of the old one would go unnoticed by the client. + if (previous_session_id != 0) + { + LOG_INFO(log, "Client asked to continue session {}, which cannot be restored, replying that it has expired", previous_session_id); + sendHandshake(HandshakeResult::SessionExpired, use_compression); + return; + } + if (keeper_dispatcher->isServerActive()) { try @@ -476,7 +486,7 @@ void KeeperTCPHandler::runImpl() catch (const Exception & e) { LOG_WARNING(log, "Cannot receive session id {}", e.displayText()); - sendHandshake(/* has_leader */ false, use_compression); + sendHandshake(HandshakeResult::Rejected, use_compression); return; } @@ -489,12 +499,12 @@ void KeeperTCPHandler::runImpl() return; } - sendHandshake(/* has_leader */ true, use_compression); + sendHandshake(HandshakeResult::Accepted, use_compression); } else { LOG_WARNING(log, "Ignoring user request, because the server is not active yet"); - sendHandshake(/* has_leader */ false, use_compression); + sendHandshake(HandshakeResult::Rejected, use_compression); return; } diff --git a/src/Server/KeeperTCPHandler.h b/src/Server/KeeperTCPHandler.h index 820306a41fed9..6e1b9e5e96b7c 100644 --- a/src/Server/KeeperTCPHandler.h +++ b/src/Server/KeeperTCPHandler.h @@ -77,6 +77,8 @@ class KeeperTCPHandler : public Poco::Net::TCPServerConnection Poco::Timespan max_session_timeout; Poco::Timespan session_timeout; int64_t session_id{-1}; + /// Session the client asked to continue in its handshake, 0 for a new session. + int64_t previous_session_id{0}; Stopwatch session_stopwatch; SocketInterruptablePollWrapperPtr poll_wrapper; Poco::Timespan send_timeout; @@ -105,7 +107,13 @@ class KeeperTCPHandler : public Poco::Net::TCPServerConnection void cancelWriteBuffer() noexcept; ReadBuffer & getReadBuffer(); - void sendHandshake(bool has_leader, bool & use_compression); + enum class HandshakeResult + { + Accepted, + Rejected, + SessionExpired, + }; + void sendHandshake(HandshakeResult result, bool & use_compression); Poco::Timespan receiveHandshake(int32_t handshake_length, bool & use_compression); static bool isHandShake(int32_t handshake_length); diff --git a/tests/integration/test_keeper_session/test.py b/tests/integration/test_keeper_session/test.py index 07062f42f5e61..213e1f3b22bcb 100644 --- a/tests/integration/test_keeper_session/test.py +++ b/tests/integration/test_keeper_session/test.py @@ -36,6 +36,9 @@ reply_header_struct = struct.Struct("!iqi") stat_struct = struct.Struct("!qqqqiiiqiiq") +# Protocol version Keeper sends when it rejects a connection. +KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT = 42 + @pytest.fixture(scope="module") def started_cluster(): @@ -86,7 +89,9 @@ def read_buffer(bytes, offset): return bytes[index : index + length], offset -def handshake(node_name=node1.name, session_timeout=1000, session_id=0): +def handshake( + node_name=node1.name, session_timeout=1000, session_id=0, full_reply=False +): client = None try: client = get_keeper_socket(node_name) @@ -130,6 +135,12 @@ def handshake(node_name=node1.name, session_timeout=1000, session_id=0): read_only = False print("negotiated_timeout - session_id", negotiated_timeout, session_id) + if full_reply: + (reply_length,) = int_struct.unpack_from(data, 0) + rest = data[int_struct.size + reply_length :] + while chunk := client.recv(1_000): + rest += chunk + return proto_version, negotiated_timeout, session_id, rest return negotiated_timeout, session_id finally: if client is not None: @@ -149,6 +160,35 @@ def test_session_timeout(started_cluster): assert negotiated_timeout == 10000 +def test_handshake_to_continue_session_is_expired(started_cluster): + wait_nodes() + negotiated_timeout, session_id = handshake( + node1.name, session_timeout=8000, session_id=0 + ) + assert negotiated_timeout == 8000 and session_id > 0 + # Keeper cannot restore a session, so a request to continue one must not be answered with a new session. + # The expired reply is the last thing the server sends before it closes the connection. + assert handshake( + node1.name, session_timeout=8000, session_id=session_id, full_reply=True + ) == (0, 0, 0, b"") + + try: + node2.stop_clickhouse() + node3.stop_clickhouse() + keeper_utils.wait_until_quorum_lost(cluster, node1) + # A rejected client gets no session id that it would send back as the session to continue. + assert handshake( + node1.name, session_timeout=8000, session_id=0, full_reply=True + ) == (KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT, 8000, 0, b"") + assert handshake( + node1.name, session_timeout=8000, session_id=session_id, full_reply=True + ) == (0, 0, 0, b"") + finally: + node2.start_clickhouse() + node3.start_clickhouse() + wait_nodes() + + def test_session_close_shutdown(started_cluster): wait_nodes() diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj index 160599290aae4..cc8e3f678a10f 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj @@ -34,9 +34,12 @@ :type :ok :value (long (count (zk-list conn root-path)))) (catch Exception _ (assoc op :type :info, :error :connect-error))) - :final-read (chu/exec-with-retries 30 (fn [] (assoc op - :type :ok - :value (long (count (zk-list conn root-path)))))) + :final-read (chu/exec-with-retries 30 (fn [] + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (assoc op + :type :ok + :value (long (count (zk-list conn root-path)))))))) :add (try (do (zk-multi-create-many-seq-nodes conn (concat-path root-path "seq-") (:value op) :with-acl (:with-auth test)) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj index d26849ee127e1..be4b3f1481a63 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj @@ -45,8 +45,10 @@ :drain ; drain via delete is to long, just list all nodes (chu/exec-with-retries 30 (fn [] - (zk-sync conn) - (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn root-path)))))))) + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (zk-sync conn) + (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn root-path)))))))))) (teardown! [_ test]) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj index 87b4f91b5863c..afe25f28a8097 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj @@ -25,10 +25,12 @@ (invoke! [this test op] (case (:f op) :read (chu/exec-with-retries 30 (fn [] - (zk-sync conn) - (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k)))))) + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (zk-sync conn) + (assoc op + :type :ok + :value (read-string (:data (zk-get-str conn k)))))))) :add (try (do (zk-add-to-set conn k (:value op)) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj index 9947973e51aea..cfdf7d1a829aa 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj @@ -32,6 +32,16 @@ conn) conn))) +(defn with-fresh-conn + "Calls (f conn) on a new connection to node and closes it. Keeper cannot continue a session, + so a worker's connection that dropped after its last operation is expired." + [node with-auth f] + (let [conn (zk-connect node 9181 30000 with-auth)] + (try + (f conn) + (finally + (zk/close conn))))) + (defn zk-create-range [conn n & {:keys [with-acl] :or {with-acl false}}] (dorun (map (fn [v] (zk/create-all conn v From d94783f341100c9c84a27cc5e11849a7b0deff20 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 30 Sep 2026 08:19:18 +0000 Subject: [PATCH 130/296] Backport #122829 to 26.8: Fix crash on invalid Iceberg manifest-list entries --- .../DataLakes/Iceberg/IcebergWrites.cpp | 66 +++++++++---- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 7 ++ ...g_malformed_manifest_list_writer.reference | 4 + ..._iceberg_malformed_manifest_list_writer.sh | 96 +++++++++++++++++++ 4 files changed, 157 insertions(+), 16 deletions(-) create mode 100644 tests/queries/0_stateless/05292_iceberg_malformed_manifest_list_writer.reference create mode 100755 tests/queries/0_stateless/05292_iceberg_malformed_manifest_list_writer.sh diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 95e04d42d5d80..cb62e186cfbd1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -932,32 +932,66 @@ void generateManifestList( forEachAvroEntry(resolved_manifest_list_path, object_storage, context, "IcebergWrites", [&](const avro::GenericDatum & datum) { + if (datum.type() != avro::AVRO_RECORD) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list {} contains an entry with Avro type {}, but a record is required", + resolved_manifest_list_path, + static_cast(datum.type())); + const avro::GenericRecord & old_entry = datum.value(); + + auto validate_field_type = [&](const String & field_name, avro::Type expected_type) -> const avro::GenericDatum & + { + if (!old_entry.hasField(field_name)) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list {} entry is missing required field '{}'", + resolved_manifest_list_path, + field_name); + + const avro::GenericDatum & field = old_entry.field(field_name); + if (field.type() != expected_type) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list {} field '{}' has Avro type {}, but type {} is required", + resolved_manifest_list_path, + field_name, + static_cast(field.type()), + static_cast(expected_type)); + + return field; + }; + + const avro::GenericDatum & old_manifest_path = validate_field_type(Iceberg::f_manifest_path, avro::AVRO_STRING); + /// When a path filter is supplied, copy only the matching entries. if (!carry_forward_manifest_paths.empty() - && !carry_forward_manifest_paths.contains(old_entry.field(Iceberg::f_manifest_path).value())) + && !carry_forward_manifest_paths.contains(old_manifest_path.value())) return; avro::GenericDatum new_datum(schema.root()); avro::GenericRecord & new_entry = new_datum.value(); - new_entry.field(f_manifest_path) = old_entry.field(Iceberg::f_manifest_path); - new_entry.field(f_manifest_length) = old_entry.field(Iceberg::f_manifest_length); - new_entry.field(f_partition_spec_id) = old_entry.field(Iceberg::f_partition_spec_id); + + auto copy_required_field = [&](const String & field_name, avro::Type expected_type) + { + new_entry.field(field_name) = validate_field_type(field_name, expected_type); + }; + + new_entry.field(f_manifest_path) = old_manifest_path; + copy_required_field(Iceberg::f_manifest_length, avro::AVRO_LONG); + copy_required_field(Iceberg::f_partition_spec_id, avro::AVRO_INT); /// iceberg-spark changed `f_added_snapshot_id` from 'null, long' to 'long' (apache/iceberg#11626); rewrite with the new schema in case we read the old type. if (old_entry.hasField(Iceberg::f_added_snapshot_id)) { const avro::GenericDatum & old_added_snapshot_id_entry = old_entry.field(Iceberg::f_added_snapshot_id); - if (old_added_snapshot_id_entry.isUnion()) - { - if (old_added_snapshot_id_entry.unionBranch() == 0) /// it means add_snapshot_id is null - { - /// This only happens when we read data written by a old version of iceberg, which violates the spec of iceberg. - throw Exception( - ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "Manifest list {} has null value for field '{}', but it is required", - resolved_manifest_list_path, - Iceberg::f_added_snapshot_id); - } - } + if (old_added_snapshot_id_entry.type() != avro::AVRO_LONG) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list {} field '{}' has Avro type {}, but a non-null long is required", + resolved_manifest_list_path, + Iceberg::f_added_snapshot_id, + static_cast(old_added_snapshot_id_entry.type())); + new_entry.field(f_added_snapshot_id) = old_added_snapshot_id_entry.value(); } else diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 497f559d7c959..57f604b78c419 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -1610,6 +1610,13 @@ void forEachAvroEntry( auto reader_base = std::make_unique(std::move(input_stream), MAX_AVRO_SCHEMA_DEPTH); avro::DataFileReader reader(std::move(reader_base)); + if (reader.readerSchema().root()->type() != avro::AVRO_RECORD) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Avro file {} has root schema type {}, but Iceberg manifest-list entries must be records", + filename, + static_cast(reader.readerSchema().root()->type())); + avro::GenericDatum datum(reader.readerSchema()); while (reader.read(datum)) callback(datum); diff --git a/tests/queries/0_stateless/05292_iceberg_malformed_manifest_list_writer.reference b/tests/queries/0_stateless/05292_iceberg_malformed_manifest_list_writer.reference new file mode 100644 index 0000000000000..2e9db112b7753 --- /dev/null +++ b/tests/queries/0_stateless/05292_iceberg_malformed_manifest_list_writer.reference @@ -0,0 +1,4 @@ +--- malformed manifest list is rejected cleanly --- +ICEBERG_SPECIFICATION_VIOLATION +--- stateless server is still alive --- +1 diff --git a/tests/queries/0_stateless/05292_iceberg_malformed_manifest_list_writer.sh b/tests/queries/0_stateless/05292_iceberg_malformed_manifest_list_writer.sh new file mode 100755 index 0000000000000..069f6e0f755f7 --- /dev/null +++ b/tests/queries/0_stateless/05292_iceberg_malformed_manifest_list_writer.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: requires `IcebergLocal` (USE_AVRO build option) and Iceberg writes. + +# Regression test for a malformed manifest list reaching the Iceberg writer carry-forward path. +# `clickhouse local` contains the pre-fix fatal signal to a short-lived subprocess. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +WORK_DIR="${CLICKHOUSE_TMP}/iceberg_malformed_manifest_list_writer_${CLICKHOUSE_TEST_UNIQUE_NAME}" +LOCAL_PATH="${WORK_DIR}/db" +TABLE_ROOT="${WORK_DIR}/iceberg/t0" + +rm -rf "${WORK_DIR}" +mkdir -p "${LOCAL_PATH}" "${TABLE_ROOT}" +trap 'rm -rf "${WORK_DIR}"' EXIT + +${CLICKHOUSE_LOCAL} \ + --path "${LOCAL_PATH}" \ + --allow_insert_into_iceberg=1 \ + --multiquery \ + --query " + CREATE TABLE t0 (x Int32) ENGINE = IcebergLocal('${TABLE_ROOT}/'); + INSERT INTO t0 VALUES (1), (2); + " -- --user_files_path="${WORK_DIR}" + +MANIFEST_LIST=$(find "${TABLE_ROOT}/metadata" -maxdepth 1 -name 'snap-*.avro' -type f | sort | head -1) +if [ -z "${MANIFEST_LIST}" ]; then + echo "manifest list not found" + exit 1 +fi + +MALFORMED_MANIFEST_LIST_HEX=$(python3 - <<'PY' +def write_long(value): + value = (value << 1) ^ (value >> 63) + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + out.append(byte | 0x80 if value else byte) + if not value: + return bytes(out) + + +def write_bytes(value): + return write_long(len(value)) + value + + +schema = b'{"type":"array","items":"int"}' +sync = bytes.fromhex('7d4f6998c268758353f227bc9dba7b7e') +metadata = [(b'avro.codec', b'null'), (b'avro.schema', schema)] + +header = bytearray(b'Obj\x01') +header += write_long(len(metadata)) +for key, value in metadata: + header += write_bytes(key) + write_bytes(value) +header += write_long(0) + sync + +# One root object: an Avro array containing one int value, `[1]`. +datum = write_long(1) + write_long(1) + write_long(0) +block = write_long(1) + write_long(len(datum)) + datum + sync +print((bytes(header) + block).hex()) +PY +) + +echo '--- malformed manifest list is rejected cleanly ---' +output=$( + ${CLICKHOUSE_LOCAL} \ + --path "${LOCAL_PATH}" \ + --allow_insert_into_iceberg=1 \ + --engine_file_truncate_on_insert=1 \ + --multiquery \ + --query " + SELECT count() FROM t0 FORMAT Null; + INSERT INTO FUNCTION file('${MANIFEST_LIST}', RawBLOB) SELECT unhex('${MALFORMED_MANIFEST_LIST_HEX}'); + INSERT INTO t0 VALUES (3); + " -- --user_files_path="${WORK_DIR}" 2>&1 +) +status=$? + +if echo "${output}" | grep -qF 'ICEBERG_SPECIFICATION_VIOLATION'; then + echo 'ICEBERG_SPECIFICATION_VIOLATION' +elif [ "${status}" -eq 139 ]; then + echo 'SIGSEGV' +elif echo "${output}" | grep -qF 'Received signal'; then + echo 'Received signal' +elif echo "${output}" | grep -qF 'Segmentation fault'; then + echo 'Segmentation fault' +else + echo "${output}" | grep -E -m1 'Code:|Exception|signal|Segmentation|Aborted' || echo "exit status ${status}" +fi + +echo '--- stateless server is still alive ---' +${CLICKHOUSE_CLIENT} --query "SELECT 1" From 695d3810bca80f994f351385f049a7fd9a12eeb7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 30 Sep 2026 08:59:38 +0000 Subject: [PATCH 131/296] Backport #122693 to 26.8: Parse some parquet metadata lazily --- .../Formats/Impl/Parquet/Reader.cpp | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index 1ee01ab2bbd5e..d5ba590006c65 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -432,16 +432,25 @@ void Reader::prefilterAndInitRowGroups(const std::optional clickhouse_to_parquet_name; - const auto * query_side_column_mapper = format_filter_info->current_schema_column_mapper - ? format_filter_info->current_schema_column_mapper.get() - : format_filter_info->column_mapper.get(); - if (query_side_column_mapper && format_filter_info->column_mapper) - clickhouse_to_parquet_name = - query_side_column_mapper->makeMapping(format_filter_info->column_mapper->getFieldIdToClickHouseName()).first; + std::optional> clickhouse_to_parquet_name; + auto get_clickhouse_to_parquet_name = [&]() -> const std::unordered_map & + { + if (!clickhouse_to_parquet_name) + { + clickhouse_to_parquet_name.emplace(); + const auto * query_side_column_mapper = format_filter_info->current_schema_column_mapper + ? format_filter_info->current_schema_column_mapper.get() + : format_filter_info->column_mapper.get(); + if (query_side_column_mapper && format_filter_info->column_mapper) + *clickhouse_to_parquet_name + = query_side_column_mapper->makeMapping(format_filter_info->column_mapper->getFieldIdToClickHouseName()).first; + } + return *clickhouse_to_parquet_name; + }; auto resolve_geo_meta = [&](const String & ch_name) -> std::unordered_map::const_iterator { - if (auto it = clickhouse_to_parquet_name.find(ch_name); it != clickhouse_to_parquet_name.end()) + const auto & mapping = get_clickhouse_to_parquet_name(); + if (auto it = mapping.find(ch_name); it != mapping.end()) return geo_meta->find(it->second); return geo_meta->find(ch_name); }; @@ -453,7 +462,8 @@ void Reader::prefilterAndInitRowGroups(const std::optional String { - if (auto it = clickhouse_to_parquet_name.find(ch_name); it != clickhouse_to_parquet_name.end()) + const auto & mapping = get_clickhouse_to_parquet_name(); + if (auto it = mapping.find(ch_name); it != mapping.end()) return it->second; return ch_name; }; From da76d8ae6b11da5444889e260318bf3a97a025ce Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 30 Sep 2026 10:49:33 +0000 Subject: [PATCH 132/296] Backport #122904 to 26.8: Silence `--icf=safe` warnings for the stripped Rust libraries --- cmake/strip_rust_symbols.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/strip_rust_symbols.sh b/cmake/strip_rust_symbols.sh index cb91889088bfa..4987b189590fa 100755 --- a/cmake/strip_rust_symbols.sh +++ b/cmake/strip_rust_symbols.sh @@ -50,7 +50,8 @@ for sym in "$@"; do done # Localize all symbols except the public ones, then strip unneeded locals -"$OBJCOPY" $KEEP_FLAGS --strip-unneeded "$WORK_DIR/combined.o" "$WORK_DIR/stripped.o" +# Drop .llvm_addrsig: after ld -r and objcopy its sh_link is 0, so lld --icf=safe ignores it with a warning +"$OBJCOPY" $KEEP_FLAGS --strip-unneeded --remove-section=.llvm_addrsig "$WORK_DIR/combined.o" "$WORK_DIR/stripped.o" # Repackage as .a (replace original) rm -f "$LIB_PATH" From e8f84dd97537f6f95f86087c1aa9f2b1902d8ea6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 30 Sep 2026 17:44:23 +0000 Subject: [PATCH 133/296] Backport #123060 to 26.8: Hide queue engine secrets written with the legacy `s3queue_` prefix --- src/Parsers/ASTSetQuery.cpp | 8 +++- ...eue_legacy_prefix_secret_masking.reference | 7 ++++ ...4_s3queue_legacy_prefix_secret_masking.sql | 40 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05314_s3queue_legacy_prefix_secret_masking.reference create mode 100644 tests/queries/0_stateless/05314_s3queue_legacy_prefix_secret_masking.sql diff --git a/src/Parsers/ASTSetQuery.cpp b/src/Parsers/ASTSetQuery.cpp index 4c440a87852a4..43eb30fcc9378 100644 --- a/src/Parsers/ASTSetQuery.cpp +++ b/src/Parsers/ASTSetQuery.cpp @@ -58,12 +58,16 @@ static std::array engineSettingsToHide() /// disagree on what is secret. static std::optional renderSecretChangeValue(const SettingChange & change) { - if (auto masked = CoreSettings::renderSecretSettingValue(change.name, change.value)) + /// The queue engines also take every setting, a format setting included, with the legacy `s3queue_` prefix. + static constexpr std::string_view s3queue_prefix = "s3queue_"; + const String setting_name = change.name.starts_with(s3queue_prefix) ? change.name.substr(s3queue_prefix.size()) : change.name; + + if (auto masked = CoreSettings::renderSecretSettingValue(setting_name, change.value)) return masked; for (const auto * settings_to_hide : engineSettingsToHide()) { - auto it = settings_to_hide->find(change.name); + auto it = settings_to_hide->find(setting_name); if (it != settings_to_hide->end()) return it->second(change.value); } diff --git a/tests/queries/0_stateless/05314_s3queue_legacy_prefix_secret_masking.reference b/tests/queries/0_stateless/05314_s3queue_legacy_prefix_secret_masking.reference new file mode 100644 index 0000000000000..e2849f54c782d --- /dev/null +++ b/tests/queries/0_stateless/05314_s3queue_legacy_prefix_secret_masking.reference @@ -0,0 +1,7 @@ +CREATE TABLE default.t_legacy\n(\n `a` String\n)\nENGINE = S3Queue(\'http://whatever-we-dont-care:9001/root/t_legacy/*\', \'NOSIGN\', \'CSV\')\nSETTINGS mode = \'unordered\', s3queue_loading_retries = 7, s3queue_after_processing_move_secret_access_key = \'[HIDDEN]\', s3queue_after_processing_move_connection_string = \'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', s3queue_format_avro_schema_registry_url = \'http://[HIDDEN]@registry:8080/\' +CREATE TABLE default.t_canonical\n(\n `a` String\n)\nENGINE = S3Queue(\'http://whatever-we-dont-care:9001/root/t_canonical/*\', \'NOSIGN\', \'CSV\')\nSETTINGS mode = \'unordered\', after_processing_move_secret_access_key = \'[HIDDEN]\', after_processing_move_connection_string = \'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', format_avro_schema_registry_url = \'http://[HIDDEN]@registry:8080/\' +t_canonical 1 1 +t_legacy 1 1 +Create 1 +Create 1 +Alter 1 diff --git a/tests/queries/0_stateless/05314_s3queue_legacy_prefix_secret_masking.sql b/tests/queries/0_stateless/05314_s3queue_legacy_prefix_secret_masking.sql new file mode 100644 index 0000000000000..fe2670ed87906 --- /dev/null +++ b/tests/queries/0_stateless/05314_s3queue_legacy_prefix_secret_masking.sql @@ -0,0 +1,40 @@ +-- Tags: no-fasttest, zookeeper +-- no-fasttest: the `S3Queue` engine is not built in the fast test. + +-- The queue engines also take every setting with the legacy `s3queue_` prefix, and a secret written that way is +-- hidden in `SHOW CREATE TABLE`, `system.tables` and `system.query_log` as it is without the prefix. + +DROP TABLE IF EXISTS t_legacy; +DROP TABLE IF EXISTS t_canonical; + +CREATE TABLE t_legacy (a String) +ENGINE = S3Queue('http://whatever-we-dont-care:9001/root/t_legacy/*', NOSIGN, 'CSV') +SETTINGS mode = 'unordered', s3queue_loading_retries = 7, + s3queue_after_processing_move_secret_access_key = 'SEKRIT_05314_1', + s3queue_after_processing_move_connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_05314_2;', + s3queue_format_avro_schema_registry_url = 'http://user:SEKRIT_05314_3@registry:8080/'; + +CREATE TABLE t_canonical (a String) +ENGINE = S3Queue('http://whatever-we-dont-care:9001/root/t_canonical/*', NOSIGN, 'CSV') +SETTINGS mode = 'unordered', + after_processing_move_secret_access_key = 'SEKRIT_05314_4', + after_processing_move_connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_05314_5;', + format_avro_schema_registry_url = 'http://user:SEKRIT_05314_6@registry:8080/'; + +SHOW CREATE TABLE t_legacy; +SHOW CREATE TABLE t_canonical; + +SELECT name, position(create_table_query, 'SEKRIT') = 0, position(engine_full, 'SEKRIT') = 0 +FROM system.tables WHERE database = currentDatabase() ORDER BY name; + +ALTER TABLE t_legacy MODIFY SETTING s3queue_after_processing_move_secret_access_key = 'SEKRIT_05314_7'; + +SYSTEM FLUSH LOGS query_log; +SELECT query_kind, position(query, 'SEKRIT') = 0 +FROM system.query_log +WHERE event_date >= yesterday() AND current_database = currentDatabase() AND is_initial_query + AND type = 'QueryFinish' AND query_kind IN ('Create', 'Alter') +ORDER BY event_time_microseconds; + +DROP TABLE t_legacy; +DROP TABLE t_canonical; From d76d7856587d9f6f21d406e50affcf64085216a2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 30 Sep 2026 22:27:12 +0000 Subject: [PATCH 134/296] Backport #123114 to 26.8: S3 client check initial url in backup client --- src/Backups/BackupIO_S3.cpp | 3 + .../configs/remote_url_allow_hosts.xml | 5 ++ .../test_remote_host_filter.py | 56 +++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml create mode 100644 tests/integration/test_backup_restore_s3/test_remote_host_filter.py diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 57e9c45cbfd34..2b7c176d0ecdc 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -3,6 +3,7 @@ #if USE_AWS_S3 #include #include +#include #include #include #include @@ -131,6 +132,8 @@ class S3BackupClientCreator const S3Settings & settings, const ContextPtr & context) { + context->getGlobalContext()->getRemoteHostFilter().checkURL(s3_uri.uri); + Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); HTTPHeaderEntries headers; String session_token = settings.auth_settings[S3AuthSetting::session_token]; diff --git a/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml new file mode 100644 index 0000000000000..294b4bc7dcb49 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml @@ -0,0 +1,5 @@ + + + minio1:9001 + + diff --git a/tests/integration/test_backup_restore_s3/test_remote_host_filter.py b/tests/integration/test_backup_restore_s3/test_remote_host_filter.py new file mode 100644 index 0000000000000..977d02c84e734 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/test_remote_host_filter.py @@ -0,0 +1,56 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/remote_url_allow_hosts.xml"], + with_minio=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + node.query( + "CREATE TABLE t (id UInt64, s String) ENGINE = MergeTree ORDER BY id" + ) + node.query("INSERT INTO t VALUES (1, 'a'), (2, 'b')") + yield cluster + finally: + cluster.shutdown() + + +def test_backup_to_allowed_host(): + name = uuid.uuid4().hex + destination = f"S3('http://minio1:9001/root/data/backups/{name}', 'minio', '{minio_secret_key}')" + node.query(f"BACKUP TABLE t TO {destination}") + node.query(f"RESTORE TABLE t AS t_{name} FROM {destination}") + assert node.query(f"SELECT count() FROM t_{name}") == "2\n" + node.query(f"DROP TABLE t_{name} SYNC") + + +@pytest.mark.parametrize( + "url", + [ + "http://minio1:9002/root/data/backups/x", + "http://resolver:8080/root/data/backups/x", + "http://127.0.0.1:9/root/data/backups/x", + ], +) +def test_backup_to_disallowed_host(url): + destination = f"S3('{url}', 'minio', '{minio_secret_key}')" + settings = "SETTINGS backup_restore_s3_retry_attempts = 0" + + error = node.query_and_get_error(f"BACKUP TABLE t TO {destination} {settings}") + assert "UNACCEPTABLE_URL" in error, error + + error = node.query_and_get_error( + f"RESTORE TABLE t AS t_restored FROM {destination} {settings}" + ) + assert "UNACCEPTABLE_URL" in error, error From 85989f16e86349a9b1254c073efac3b9b46f3136 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 08:07:34 +0000 Subject: [PATCH 135/296] Backport #121621 to 26.8: Fix `NO_COMMON_TYPE` in case of `use_variant_as_common_type=0` for jemalloc fragmentation dashboard --- docs/concepts/features/performance/allocation-profiling.mdx | 2 +- programs/server/jemalloc.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/concepts/features/performance/allocation-profiling.mdx b/docs/concepts/features/performance/allocation-profiling.mdx index 7c51dfc01f933..383f8bca8cb8f 100644 --- a/docs/concepts/features/performance/allocation-profiling.mdx +++ b/docs/concepts/features/performance/allocation-profiling.mdx @@ -354,7 +354,7 @@ The same query can split the waste of each size class between its backtraces, in WITH 60e9 AS min_age_ns SELECT format('{} {}', if(s.est_old_objects > 0, s.stack, format('[unattributed];arena_{}_class_{}', toString(b.arena), toString(b.size))), - toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), b.waste)))) + toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), toFloat64(b.waste))))) FROM system.jemalloc_arena_bins AS b LEFT JOIN ( diff --git a/programs/server/jemalloc.html b/programs/server/jemalloc.html index fd785fa66f284..8505ae77e2453 100644 --- a/programs/server/jemalloc.html +++ b/programs/server/jemalloc.html @@ -2922,7 +2922,7 @@

Mutex Statistics

/// Both branches must come from the same scan of the sampled allocations: every /// scan flushes a fresh profile dump, so two independent subqueries could observe /// different dumps and double-count or drop a cell. - const query = `SELECT format('{} {}', if(s.est_old_objects > 0, s.stack, format('[unattributed];arena_{}_class_{}', toString(b.arena), toString(b.size))), toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), b.waste)))) FROM system.jemalloc_arena_bins AS b LEFT JOIN (${fragmentationPinnersSubquery(minAgeNs)}) AS s ON s.size_class = b.index AND s.arena = b.arena WHERE b.large = 0 AND b.waste >= ${minWaste}${purposeCond} SETTINGS allow_introspection_functions = 1`; + const query = `SELECT format('{} {}', if(s.est_old_objects > 0, s.stack, format('[unattributed];arena_{}_class_{}', toString(b.arena), toString(b.size))), toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), toFloat64(b.waste))))) FROM system.jemalloc_arena_bins AS b LEFT JOIN (${fragmentationPinnersSubquery(minAgeNs)}) AS s ON s.size_class = b.index AND s.arena = b.arena WHERE b.large = 0 AND b.waste >= ${minWaste}${purposeCond} SETTINGS allow_introspection_functions = 1`; const response = await fetch(buildUrl(query), fetchOptions); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); From c87233d6f0932cb3cdbb3d2851e75dd84aa94c75 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 1 Oct 2026 08:38:38 +0000 Subject: [PATCH 136/296] Fix the 26.8 build: replace the master-only row policy helper `Storages/getEffectiveRowPolicyFilter.h` does not exist on 26.8. The call site always passes the data part's MergeTree, which has no underlying storages, so the walk it does on master reduces to the table's own policy; the local helper here checks exactly that. The `Alias` case still holds: on 26.8 the alias pushes its policy into `query_info.row_level_filter`. Co-Authored-By: Claude Opus 5.5 --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 03feebb56429d..862a95339e5a4 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -21,7 +21,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -805,6 +806,15 @@ std::expected MergeTreeDataSelectExecutor::canUseInde } +static bool hasEffectiveRowPolicy(const IStorage & storage, const ContextPtr & context) +{ + auto storage_id = storage.getStorageID(); + if (!storage_id.hasDatabase()) + return false; + auto filter = context->getRowPolicyFilter(storage_id.getDatabaseName(), storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + return filter && !filter->isAlwaysTrue(); +} + RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByStatistics( const RangesInDataParts & parts, const StorageMetadataPtr & metadata_snapshot, @@ -832,7 +842,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByStatistics( || (mutations_snapshot && (mutations_snapshot->hasDataMutations() || mutations_snapshot->hasPatchParts())) || (!parts.empty() && parts.front().data_part->storage.hasEnabledMaskingPolicies(context)) || query_info.row_level_filter - || (!parts.empty() && getEffectiveRowPolicyFilter(parts.front().data_part->storage, context))) + || (!parts.empty() && hasEffectiveRowPolicy(parts.front().data_part->storage, context))) { return parts; } From 258ea89e2d2c1dfc4d43fa61c86a1987785982fb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 09:06:22 +0000 Subject: [PATCH 137/296] Backport #122681 to 26.8: Build Iceberg min/max hyperrectangles only for columns the filter uses --- .../Iceberg/ManifestFileIterator.cpp | 27 +++++++++---------- .../Iceberg/ManifestFilesPruning.cpp | 13 +++++---- .../DataLakes/Iceberg/ManifestFilesPruning.h | 3 +++ 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index fe5553bcc118a..01c067bc90090 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -511,20 +511,18 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) PruningReturnStatus pruning_status = PruningReturnStatus::NOT_PRUNED; if (filter_dag) { + const ManifestFilesPruner * current_pruner = getOrCreatePruner(entry->resolved_schema_id); + /// Compute per-column hyperrectangles for DATA files std::unordered_map hyperrectangles; if (parsed_entry->content_type == FileContentType::DATA) { - for (const auto & [column_id, bounds] : parsed_entry->value_bounds) + for (const auto & [column_id, column_type] : current_pruner->getMinMaxColumnTypes()) { - auto field_characteristics = schema_processor_ptr->tryGetFieldCharacteristics(resolved_schema_id, column_id); - /// If we don't have column characteristics, bounds don't have any sense. - /// This happens if the subfield is inside map or array, because we don't support - /// name generation for such subfields (we support names of nested subfields in structs only). - if (!field_characteristics) + auto bounds_it = parsed_entry->value_bounds.find(column_id); + if (bounds_it == parsed_entry->value_bounds.end()) continue; - - const auto & name_and_type = *field_characteristics; + const auto & bounds = bounds_it->second; String left_str; String right_str; @@ -532,13 +530,13 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) if (!bounds.first.tryGet(left_str) || !bounds.second.tryGet(right_str)) continue; - if (const auto type_id = name_and_type.type->getTypeId(); + if (const auto type_id = column_type->getTypeId(); type_id == DB::TypeIndex::Tuple || type_id == DB::TypeIndex::Map || type_id == DB::TypeIndex::Array || type_id == DB::TypeIndex::Variant) continue; - auto left = deserializeFieldFromBinaryRepr(left_str, name_and_type.type, true); - auto right = deserializeFieldFromBinaryRepr(right_str, name_and_type.type, false); + auto left = deserializeFieldFromBinaryRepr(left_str, column_type, true); + auto right = deserializeFieldFromBinaryRepr(right_str, column_type, false); if (!left || !right) { /// Pruning is skipped either way, but at scale 38 a bound that only loses its widened @@ -559,12 +557,12 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) /// declared expose that inversion, which is why they are read again here. std::optional declared_left = left; std::optional declared_right = right; - if (DB::WhichDataType(DB::removeNullable(name_and_type.type)).isDecimal()) + if (DB::WhichDataType(DB::removeNullable(column_type)).isDecimal()) { declared_left = deserializeFieldFromBinaryRepr( - left_str, name_and_type.type, true, /*compensate_rounding=*/false); + left_str, column_type, true, /*compensate_rounding=*/false); declared_right = deserializeFieldFromBinaryRepr( - right_str, name_and_type.type, false, /*compensate_rounding=*/false); + right_str, column_type, false, /*compensate_rounding=*/false); } /// A pair inverted as declared means the manifest's statistics are untrustworthy, so no @@ -586,7 +584,6 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) } } - const ManifestFilesPruner * current_pruner = getOrCreatePruner(entry->resolved_schema_id); pruning_status = current_pruner->canBePruned(entry, hyperrectangles); } insertRowToLogTable( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp index c7c76cbd14836..ab8e95a358b6f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp @@ -137,6 +137,7 @@ ManifestFilesPruner::ManifestFilesPruner( if (!name_and_type.has_value()) continue; + min_max_column_types.emplace(used_column_id, name_and_type->type); name_and_type->name = DB::backQuote(DB::toString(used_column_id)); ExpressionActionsPtr expression @@ -394,19 +395,17 @@ PruningReturnStatus ManifestFilesPruner::canBePruned( for (const auto & [column_id, key_condition] : min_max_key_conditions) { - std::optional name_and_type = schema_processor.tryGetFieldCharacteristics(initial_schema_id, column_id); - /// There is no such column in this manifest file - if (!name_and_type.has_value()) - { + auto type_it = min_max_column_types.find(column_id); + if (type_it == min_max_column_types.end()) continue; - } + const auto & column_type = type_it->second; auto info_it = entry->parsed_entry->columns_infos.find(column_id); bool has_no_nulls = info_it != entry->parsed_entry->columns_infos.end() && info_it->second.nulls_count.has_value() && *info_it->second.nulls_count == 0; - const DataTypes data_types{name_and_type->type}; + const DataTypes data_types{column_type}; if (entry->common_partition_specification) { @@ -419,7 +418,7 @@ PruningReturnStatus ManifestFilesPruner::canBePruned( auto range = rangeOfPartitionValue( partition_field.transform_name, partition_value[partition_field.tuple_index], - *removeNullable(name_and_type->type)); + *removeNullable(column_type)); if (range && !key_condition.mayBeTrueInRange(1, &range->left, &range->right, data_types)) return PruningReturnStatus::PARTITION_PRUNED; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h index 78c136167a880..e9aa14f97e57e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h @@ -43,6 +43,7 @@ class ManifestFilesPruner std::optional partition_key_condition; std::unordered_map min_max_key_conditions; + std::unordered_map min_max_column_types; /// NOTE: tricky part to support RENAME column. /// Takes ActionDAG representation of user's WHERE expression and /// rename columns to the their origina numeric ID's in iceberg @@ -58,6 +59,8 @@ class ManifestFilesPruner DB::ContextPtr context); PruningReturnStatus canBePruned(const ProcessedManifestFileEntryPtr & entry, const std::unordered_map & entry_hyperrectangles) const; + + const std::unordered_map & getMinMaxColumnTypes() const { return min_max_column_types; } }; } From 74f2e6b59474fb572dacf76d9da3a5a9bfb0db18 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 09:55:06 +0000 Subject: [PATCH 138/296] Backport #120516 to 26.8: Fix access checks in PREWHERE and indexHint() that used aliases --- src/Planner/CollectTableExpressionData.cpp | 27 +++- src/Planner/PlannerJoinTree.cpp | 17 +++ src/Planner/TableExpressionData.h | 23 ++++ src/Planner/Utils.cpp | 42 +++--- src/Planner/Utils.h | 4 + .../collectSelectedColumnsFromTable.cpp | 30 +--- src/Storages/MergeTree/MergeTreeData.cpp | 3 +- ...ewhere_alias_column_access_check.reference | 60 ++++++++ ...5218_prewhere_alias_column_access_check.sh | 129 ++++++++++++++++++ .../05293_index_hint_column_access.reference | 24 ++++ .../05293_index_hint_column_access.sh | 48 +++++++ ...nly_alter_guard_alias_dependency.reference | 1 + ...data_only_alter_guard_alias_dependency.sql | 28 ++++ 13 files changed, 387 insertions(+), 49 deletions(-) create mode 100644 tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference create mode 100755 tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh create mode 100644 tests/queries/0_stateless/05293_index_hint_column_access.reference create mode 100755 tests/queries/0_stateless/05293_index_hint_column_access.sh create mode 100644 tests/queries/0_stateless/05294_metadata_only_alter_guard_alias_dependency.reference create mode 100644 tests/queries/0_stateless/05294_metadata_only_alter_guard_alias_dependency.sql diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp index 996b518cb17f5..c3d7831aeeaa1 100644 --- a/src/Planner/CollectTableExpressionData.cpp +++ b/src/Planner/CollectTableExpressionData.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -18,6 +20,7 @@ #include #include #include +#include namespace DB @@ -62,7 +65,16 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContextgetOrCreateTableExpressionData(column_node->getColumnSource()); + index_hint_table_expression_data.markColumnForAccessCheck(column_node->getColumnName()); + } return; + } auto column_source_node = column_node->getColumnSource(); auto column_source_node_type = column_source_node->getNodeType(); @@ -106,6 +118,9 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContextgetExpression(); + /// The visitor above has registered the expression's source columns as read but not selected, and the + /// children walk that follows must not select them either: a grant on the ALIAS name is sufficient. + inlined_alias_expressions.insert(node.get()); return; } @@ -174,6 +189,8 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContextas()) return child_node != table_node->getMaterializedCTESubquery(); - return !(checkSubquery(child_node) || isAliasColumn(parent_node)); + return !(checkSubquery(child_node) || isAliasColumn(parent_node) || inlined_alias_expressions.contains(parent_node.get())); } static bool isIndexHintFunction(const QueryTreeNodePtr & node) @@ -263,6 +280,9 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContext inlined_alias_expressions; }; class CollectPrewhereTableExpressionVisitor : public ConstInDepthQueryTreeVisitor @@ -440,6 +460,11 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr const auto & selected_column_names = table_expression_data.getSelectedColumnsNames(); required_column_names_without_prewhere.insert(selected_column_names.begin(), selected_column_names.end()); + /// The visit below inlines ALIAS columns, which would hide their names from the access check. + /// Record what PREWHERE references first, so the same names are checked as for WHERE. + for (const auto & column_name : collectReferencedColumnNames(query_node_typed.getPrewhere(), prewhere_table_expression)) + table_expression_data.markColumnForAccessCheck(column_name); + collect_source_columns_visitor.setKeepAliasColumns(false); collect_source_columns_visitor.visit(query_node_typed.getPrewhere()); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 67ca7fb5945a6..52e09cafb0b39 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -722,6 +722,21 @@ bool applyTrivialCountWithSparsityFilterIfPossible( return true; } +/** Check the SELECT privilege for the columns that the planner resolved "away": `indexHint` arguments and ALIAS + * columns inlined into PREWHERE. Checked separately from the selected columns on purpose: a trivial query such as + * `SELECT count() FROM t` passes with a grant on any one column, while these names are always required. + */ +void checkAccessRightsForColumnsResolvedAway( + const TableNode & table_node, const TableExpressionData & table_expression_data, const ContextPtr & query_context) +{ + const auto & column_names = table_expression_data.getAccessCheckedColumnsNames(); + if (column_names.empty()) + return; + + checkAccessRights( + table_node.getStorage(), table_node.getStorageID(), table_node.getStorageSnapshot(), column_names, query_context); +} + void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expression, const SelectQueryOptions & select_query_options, PlannerContextPtr & planner_context) { const auto & query_context = planner_context->getQueryContext(); @@ -744,6 +759,8 @@ void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expr const auto & column_names_with_aliases = table_expression_data.getSelectedColumnsNames(); columns_names_allowed_to_select = checkAccessRights( table_node->getStorage(), table_node->getStorageID(), table_node->getStorageSnapshot(), column_names_with_aliases, query_context); + + checkAccessRightsForColumnsResolvedAway(*table_node, table_expression_data, query_context); } else if (table_function_node) { diff --git a/src/Planner/TableExpressionData.h b/src/Planner/TableExpressionData.h index 29b765a27dc77..b3943d38feb40 100644 --- a/src/Planner/TableExpressionData.h +++ b/src/Planner/TableExpressionData.h @@ -93,6 +93,24 @@ class TableExpressionData selected_column_names.push_back(column_name); } + /** Mark a column that the user references explicitly, but that never becomes a selected column. + * + * This is needed for columns that the planner resolves away before the access check runs : + * an ALIAS column inlined into PREWHERE and a column used only as an indexHint argument. + */ + void markColumnForAccessCheck(const std::string & column_name) + { + auto [_, inserted] = access_checked_column_names_set.emplace(column_name); + if (inserted) + access_checked_column_names.push_back(column_name); + } + + /// Get columns that are not selected, but still require a SELECT privilege check + const Names & getAccessCheckedColumnsNames() const + { + return access_checked_column_names; + } + /// Get columns that are requested from table expression, including ALIAS columns const Names & getSelectedColumnsNames() const { @@ -292,6 +310,11 @@ class TableExpressionData /// To deduplicate columns in `selected_column_names` NameSet selected_column_names_set; + /// Columns that the user references explicitly, but that are resolved away before access check. + Names access_checked_column_names; + /// To deduplicate columns in above + NameSet access_checked_column_names_set; + /// Expression to calculate ALIAS columns /// Keep alias name (String) + expression (ActionsDAG) pairs; vector preserves insertion order. AliasColumnExpressions alias_column_expressions; diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index a54f859f69fb3..32c307f617736 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -620,6 +620,29 @@ NameSet checkAccessRights( return {}; } +NameSet collectReferencedColumnNames(const QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression) +{ + NameSet column_names; + traverseQueryTree( + node, + [](const QueryTreeNodePtr & parent, const QueryTreeNodePtr &) + { + /// Don't go inside an ALIAS column expression: a grant on the alias name is sufficient. + const auto * column_node = parent->as(); + if (!column_node || !column_node->hasExpression()) + return true; + const auto & column_source = column_node->getColumnSourceOrNull(); + return !(column_source && column_source->getNodeType() == QueryTreeNodeType::TABLE); + }, + [&](const QueryTreeNodePtr & current) + { + const auto * column_node = current->as(); + if (column_node && column_node->getColumnSourceOrNull().get() == table_expression.get()) + column_names.insert(column_node->getColumnName()); + }); + return column_names; +} + static void checkAccessRightsForFilter(const QueryTreeNodePtr & filter_query_tree, const QueryTreeNodePtr & table_expression, const ContextPtr & query_context) @@ -650,24 +673,7 @@ static void checkAccessRightsForFilter(const QueryTreeNodePtr & filter_query_tre return; } - NameSet column_names; - traverseQueryTree( - filter_query_tree, - [](const QueryTreeNodePtr & parent, const QueryTreeNodePtr &) - { - /// Don't go inside an ALIAS column expression: a grant on the alias name is sufficient. - const auto * column_node = parent->as(); - if (!column_node || !column_node->hasExpression()) - return true; - const auto & column_source = column_node->getColumnSourceOrNull(); - return !(column_source && column_source->getNodeType() == QueryTreeNodeType::TABLE); - }, - [&](const QueryTreeNodePtr & node) - { - const auto * column_node = node->as(); - if (column_node && column_node->getColumnSourceOrNull().get() == table_expression.get()) - column_names.insert(column_node->getColumnName()); - }); + NameSet column_names = collectReferencedColumnNames(filter_query_tree, table_expression); if (column_names.empty()) return; diff --git a/src/Planner/Utils.h b/src/Planner/Utils.h index 3b504afb52861..032a614a73d7d 100644 --- a/src/Planner/Utils.h +++ b/src/Planner/Utils.h @@ -92,6 +92,10 @@ QueryTreeNodePtr replaceTableExpressionsWithDummyTables( SelectQueryInfo buildSelectQueryInfo(const QueryTreeNodePtr & query_tree, const PlannerContextPtr & planner_context); +/// Names of `table_expression` columns referenced from `node`, with ALIAS columns under their own name: +/// their expressions are not entered, because a grant on the alias name is sufficient to use it. +NameSet collectReferencedColumnNames(const QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression); + /// Check if current user has privileges to SELECT columns from table /// Throws an exception if access to any column from `column_names` is not granted /// If `column_names` is empty, check access to any columns and return names of accessible columns diff --git a/src/Planner/collectSelectedColumnsFromTable.cpp b/src/Planner/collectSelectedColumnsFromTable.cpp index 25d1ae549e964..6fecac2fd34b8 100644 --- a/src/Planner/collectSelectedColumnsFromTable.cpp +++ b/src/Planner/collectSelectedColumnsFromTable.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include @@ -21,12 +20,6 @@ class CollectSelectedColumnsFromTableVisitor : public InDepthQueryTreeVisitorWit void enterImpl(QueryTreeNodePtr & node) { - if (isIndexHintFunction(node)) - { - is_inside_index_hint_function = true; - return; - } - auto * column_node = node->as(); if (!column_node) return; @@ -38,24 +31,10 @@ class CollectSelectedColumnsFromTableVisitor : public InDepthQueryTreeVisitorWit if (!source_table || source_table->getStorageID() != storage_id) return; - /// A special case for the "indexHint" function. We don't need its arguments for execution if column's source table is MergeTree. - /// Instead, we prepare an ActionsDAG for its arguments and store it inside a function (see ActionsDAG::buildFilterActionsDAG). - /// So this optimization allows not to read arguments of "indexHint" (if not needed in other contexts) but only to use index analysis for them. - if (is_inside_index_hint_function && source_table->getStorage()->isMergeTree()) - return; - + /// Note that arguments of the "indexHint" function need to be checked for SELECT privilege selected_columns.insert(column_node->getColumnName()); } - void leaveImpl(QueryTreeNodePtr & node) - { - if (isIndexHintFunction(node)) - { - is_inside_index_hint_function = false; - return; - } - } - bool isAliasColumn(const QueryTreeNodePtr & node) const { const auto * column_node = node->as(); @@ -73,19 +52,12 @@ class CollectSelectedColumnsFromTableVisitor : public InDepthQueryTreeVisitorWit return !isAliasColumn(parent_node); } - bool isIndexHintFunction(const QueryTreeNodePtr & node) const - { - return node->as() && node->as()->getFunctionName() == "indexHint"; - } - std::vector getSelectedColumns() const { return std::vector(selected_columns.begin(), selected_columns.end()); } private: - /// True if we are traversing arguments of function "indexHint". - bool is_inside_index_hint_function = false; const StorageID & storage_id; std::unordered_set selected_columns; }; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 592d37ed175b2..eb0a0814121ab 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5071,8 +5071,9 @@ Names expressionSourceColumns(const ASTPtr & ast, const ColumnsDescription & col auto planner_context = std::make_shared(analysis_context, global_planner_context, SelectQueryOptions{}); collectSetsAndSourceColumns(expression, planner_context, /*keep_alias_columns=*/ false); + /// ALIAS columns are inlined above, so the physical columns their expressions read are the dependencies. if (const auto * table_expression_data = planner_context->getTableExpressionDataOrNull(table_node)) - return table_expression_data->getSelectedColumnsNames(); + return table_expression_data->getColumnNames(); return {}; } diff --git a/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference new file mode 100644 index 0000000000000..018a9a02fa76d --- /dev/null +++ b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference @@ -0,0 +1,60 @@ +-- user granted only pub +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias +[0,1,2,3,4,5,6,7] +SELECT pub FROM t_prewhere_alias PREWHERE pub = 2 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret_alias >= 500) +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500) +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(pub < 5) +1 +-- user granted pub and the alias columns, but not their source +SELECT secret_alias FROM t_prewhere_alias PREWHERE pub = 2 +200 +SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +2 +SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression + 0 = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +ACCESS_DENIED +-- user granted pub and the source column, but not the alias columns +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +2 +SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +[4,5,6,7] +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500) +0 +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 501) +1 +-- row policy over a column the user is not granted +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias +[0,1,3,4,5,6,7] +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias PREWHERE pub > 1 +[3,4,5,6,7] diff --git a/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh new file mode 100755 index 0000000000000..89efe05350649 --- /dev/null +++ b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer, no-parallel-replicas +# no-parallel-replicas: reading an ALIAS column with a grant on only that ALIAS column is denied under +# parallel replicas. That is an unrelated pre-existing bug (reproducible on master without this test's +# subject), so exclude the setting rather than encode the wrong behaviour in the reference. + +# Column-level SELECT grants must be enforced for columns that the planner resolves away before the +# access check runs, so that PREWHERE cannot be used as an oracle over a column the user cannot read: +# +# 1. An ALIAS column referenced in PREWHERE is replaced by its expression, so it used to never reach +# the list of selected columns and was not access checked at all, while the same alias in SELECT or +# WHERE was correctly denied. +# 2. A column used only as an `indexHint` argument is never read, only used for index analysis, but +# which granules survive the analysis is observable in the result. +# +# In both cases the required privilege is the one on the name written in the query, exactly as in +# WHERE: referencing an ALIAS column requires a grant on the ALIAS column itself, and a grant on the +# physical columns its expression reads is neither sufficient nor required. Those source columns are +# still read from disk to compute the ALIAS - the administrator authored the ALIAS expression and so +# chose what it exposes - but their values never reach the user. + +# Most of the queries below are expected to be denied, and each denial would otherwise add an +# log line and a stack trace to the output, because the harness streams server-side logs to the client. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_PUB="user_pub_${CLICKHOUSE_DATABASE}" +USER_ALIAS="user_alias_${CLICKHOUSE_DATABASE}" +USER_SOURCE="user_source_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP USER IF EXISTS ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; + + CREATE TABLE t_prewhere_alias + ( + secret Int32, + pub Int32, + secret_alias Int32 ALIAS secret, + secret_alias_expression Int32 ALIAS secret_alias + 1 + ) + ENGINE = MergeTree ORDER BY secret SETTINGS index_granularity = 1, min_bytes_for_wide_part = 0; + + INSERT INTO t_prewhere_alias (secret, pub) SELECT number * 100, number FROM numbers(8); + + CREATE USER ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; + GRANT SELECT(pub) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_PUB}; + GRANT SELECT(pub, secret_alias, secret_alias_expression) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_ALIAS}; + GRANT SELECT(pub, secret) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_SOURCE}; +" + +# Run all the queries of one user in a single session, echoing each query before its own result so +# that every line of the reference is attributable. --ignore-error keeps the session going past the +# ACCESS_DENIED answers that most of these queries are expected to produce; without it the first +# denial would end the session and silently drop every query after it. +# +# An exception is printed by the client as exactly three lines - the "Received exception" banner, the +# "Code: ..." message and the echoed query - of which only the error name is stable: the banner holds +# the server version and the message holds the user name, which embeds ${CLICKHOUSE_DATABASE}. Reduce +# the block to that name, keeping it distinct per error so that a change of error shows up as a diff +# rather than being masked. +run_all() +{ + local user=$1 + shift + + local sql="" + local query + for query in "$@" + do + # The echoed query is derived from the query itself, so the two cannot drift apart. + sql+="SELECT '${query//\'/\'\'}'; +${query}; +" + done + + ${CLICKHOUSE_CLIENT} --user "${user}" --multiquery --ignore-error --query "${sql}" 2>&1 \ + | sed -E '/^Received exception/d; /^\(query:/d; s/^Code: [0-9]+\..*\(([A-Z_]+)\)$/\1/' +} + +echo "-- user granted only pub" +run_all "${USER_PUB}" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias" \ + "SELECT pub FROM t_prewhere_alias PREWHERE pub = 2" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT count() FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret_alias >= 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(pub < 5)" + +echo "-- user granted pub and the alias columns, but not their source" +run_all "${USER_ALIAS}" \ + "SELECT secret_alias FROM t_prewhere_alias PREWHERE pub = 2" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression + 0 = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" + +echo "-- user granted pub and the source column, but not the alias columns" +run_all "${USER_SOURCE}" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 501)" + +# A row policy is defined by an administrator, so it may reference columns the user cannot read. +echo "-- row policy over a column the user is not granted" +${CLICKHOUSE_CLIENT} --query "CREATE ROW POLICY p_prewhere_alias ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias USING secret_alias_expression != 201 TO ${USER_PUB}" +run_all "${USER_PUB}" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias PREWHERE pub > 1" + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP ROW POLICY p_prewhere_alias ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias; + DROP TABLE t_prewhere_alias; + DROP USER ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; +" diff --git a/tests/queries/0_stateless/05293_index_hint_column_access.reference b/tests/queries/0_stateless/05293_index_hint_column_access.reference new file mode 100644 index 0000000000000..0301425635605 --- /dev/null +++ b/tests/queries/0_stateless/05293_index_hint_column_access.reference @@ -0,0 +1,24 @@ +=== granted: id +--- SELECT count() FROM t_index_hint_access WHERE id = 3 AND indexHint(secret > 5000) +ACCESS_DENIED +--- SELECT count() FROM t_index_hint_access WHERE id = 3 AND indexHint(secret < 5000) +ACCESS_DENIED +--- SELECT count() FROM t_index_hint_access WHERE indexHint(secret < 5000) +ACCESS_DENIED +--- SELECT count() FROM t_index_hint_access PREWHERE indexHint(secret < 5000) +ACCESS_DENIED +--- SELECT count() FROM (SELECT id FROM t_index_hint_access WHERE indexHint(secret < 5000)) +ACCESS_DENIED +--- SELECT count() FROM t_index_hint_access WHERE id = 3 AND indexHint(secret_alias > 5000) +ACCESS_DENIED +--- SELECT count() FROM t_index_hint_access AS l JOIN t_index_hint_access AS r ON l.id = r.id WHERE indexHint(r.secret < 5000) +ACCESS_DENIED +--- SELECT count() FROM t_index_hint_access WHERE indexHint(id < 5) +10 +=== granted: id, secret, secret_alias +--- SELECT count() FROM t_index_hint_access WHERE id = 3 AND indexHint(secret > 5000) +0 +--- SELECT count() FROM t_index_hint_access WHERE id = 3 AND indexHint(secret < 5000) +1 +--- SELECT count() FROM t_index_hint_access WHERE id = 3 AND indexHint(secret_alias > 5000) +1 diff --git a/tests/queries/0_stateless/05293_index_hint_column_access.sh b/tests/queries/0_stateless/05293_index_hint_column_access.sh new file mode 100755 index 0000000000000..ab29b226a3c6f --- /dev/null +++ b/tests/queries/0_stateless/05293_index_hint_column_access.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Tags: no-parallel-replicas + +# A column referenced only inside `indexHint` is not read, but index analysis prunes granules by it, +# so `count()` acts as an oracle for its values. It must require the same SELECT grant as a regular predicate. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_05293_${CLICKHOUSE_DATABASE}" +table="${CLICKHOUSE_DATABASE}.t_index_hint_access" + +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS $table; + CREATE TABLE $table (id UInt64, secret UInt64, secret_alias ALIAS secret) + ENGINE = MergeTree ORDER BY secret SETTINGS index_granularity = 1; + INSERT INTO $table SELECT number, number * 1000 FROM numbers(10); + DROP USER IF EXISTS $user; + CREATE USER $user; + GRANT SELECT(id) ON $table TO $user; +" + +function query_as_user() +{ + echo "--- ${1//${CLICKHOUSE_DATABASE}./}" + ${CLICKHOUSE_CLIENT} --user "$user" --query "$1" 2>&1 | grep -oE '^[0-9]+$|ACCESS_DENIED' | uniq +} + +echo "=== granted: id" +query_as_user "SELECT count() FROM $table WHERE id = 3 AND indexHint(secret > 5000)" +query_as_user "SELECT count() FROM $table WHERE id = 3 AND indexHint(secret < 5000)" +query_as_user "SELECT count() FROM $table WHERE indexHint(secret < 5000)" +query_as_user "SELECT count() FROM $table PREWHERE indexHint(secret < 5000)" +query_as_user "SELECT count() FROM (SELECT id FROM $table WHERE indexHint(secret < 5000))" +query_as_user "SELECT count() FROM $table WHERE id = 3 AND indexHint(secret_alias > 5000)" +query_as_user "SELECT count() FROM $table AS l JOIN $table AS r ON l.id = r.id WHERE indexHint(r.secret < 5000)" +query_as_user "SELECT count() FROM $table WHERE indexHint(id < 5)" + +${CLICKHOUSE_CLIENT} --query "GRANT SELECT(secret, secret_alias) ON $table TO $user" + +echo "=== granted: id, secret, secret_alias" +query_as_user "SELECT count() FROM $table WHERE id = 3 AND indexHint(secret > 5000)" +query_as_user "SELECT count() FROM $table WHERE id = 3 AND indexHint(secret < 5000)" +query_as_user "SELECT count() FROM $table WHERE id = 3 AND indexHint(secret_alias > 5000)" + +${CLICKHOUSE_CLIENT} --query "DROP USER $user; DROP TABLE $table" diff --git a/tests/queries/0_stateless/05294_metadata_only_alter_guard_alias_dependency.reference b/tests/queries/0_stateless/05294_metadata_only_alter_guard_alias_dependency.reference new file mode 100644 index 0000000000000..e2c68f15f4ecb --- /dev/null +++ b/tests/queries/0_stateless/05294_metadata_only_alter_guard_alias_dependency.reference @@ -0,0 +1 @@ +(1,'') (1) diff --git a/tests/queries/0_stateless/05294_metadata_only_alter_guard_alias_dependency.sql b/tests/queries/0_stateless/05294_metadata_only_alter_guard_alias_dependency.sql new file mode 100644 index 0000000000000..f92699c6879d6 --- /dev/null +++ b/tests/queries/0_stateless/05294_metadata_only_alter_guard_alias_dependency.sql @@ -0,0 +1,28 @@ +-- A stored MATERIALIZED column that reads the altered named tuple only through an ALIAS column +-- must block a metadata-only ALTER exactly like a direct reference does: nothing would recompute it. + +DROP TABLE IF EXISTS t_guard_plain_alias; +DROP TABLE IF EXISTS t_guard_function_alias; + +CREATE TABLE t_guard_plain_alias (t Tuple(a Int64), t_alias ALIAS t, m String MATERIALIZED toString(t_alias)) +ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_guard_plain_alias (t) VALUES ((1)); + +CREATE TABLE t_guard_function_alias (t Tuple(a Int64), s ALIAS toString(t), m String MATERIALIZED s) +ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_guard_function_alias (t) VALUES ((1)); + +ALTER TABLE t_guard_plain_alias MODIFY COLUMN t Tuple(a Int64, b String) +SETTINGS allow_metadata_only_named_tuple_alter = 1; -- { serverError ALTER_OF_COLUMN_IS_FORBIDDEN } + +ALTER TABLE t_guard_function_alias MODIFY COLUMN t Tuple(a Int64, b String) +SETTINGS allow_metadata_only_named_tuple_alter = 1; -- { serverError ALTER_OF_COLUMN_IS_FORBIDDEN } + +-- Without the lazy conversion the change runs as a full mutation that recomputes the column. +ALTER TABLE t_guard_function_alias MODIFY COLUMN t Tuple(a Int64, b String) +SETTINGS allow_metadata_only_named_tuple_alter = 0, mutations_sync = 2; + +SELECT t, m FROM t_guard_function_alias; + +DROP TABLE t_guard_plain_alias; +DROP TABLE t_guard_function_alias; From cd12432877aba6ae598a9dc5218a9876c2e2a7f0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 10:30:31 +0000 Subject: [PATCH 139/296] Backport #121398 to 26.8: Skip the direct join when the right table has a row policy --- src/Functions/FunctionJoinGet.cpp | 12 +- src/Interpreters/JoinedTables.cpp | 20 ++ .../Access/ParserCreateRowPolicyQuery.cpp | 302 ++++++++++++++++++ src/Planner/PlannerJoinTree.cpp | 20 +- src/Storages/StorageJoin.cpp | 2 + .../05233_direct_join_row_policy.reference | 43 +++ .../05233_direct_join_row_policy.sql | 63 ++++ .../05234_row_policy_storage_join.reference | 12 + .../05234_row_policy_storage_join.sql | 31 ++ 9 files changed, 500 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05233_direct_join_row_policy.reference create mode 100644 tests/queries/0_stateless/05233_direct_join_row_policy.sql create mode 100644 tests/queries/0_stateless/05234_row_policy_storage_join.reference create mode 100644 tests/queries/0_stateless/05234_row_policy_storage_join.sql diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index e3d7b87f97d8e..3c71c00821c1e 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include namespace DB { @@ -21,6 +23,7 @@ namespace Setting namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -146,7 +149,14 @@ ExecutableFunctionPtr FunctionJoinGet::prepare(const ColumnsWithTypeAndName &) c Names column_names = storage_join->getKeyNames(); column_names.push_back(attr_name); - context->checkAccess(AccessType::SELECT, storage_join->getStorageID(), column_names); + const auto storage_id = storage_join->getStorageID(); + context->checkAccess(AccessType::SELECT, storage_id, column_names); + + /// The hash table is read as is, so a row policy on the table cannot be applied here any more than in a JOIN. + auto row_policy_filter = context->getRowPolicyFilter(storage_id.getDatabaseName(), storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot use {} because a row policy is applied on table {} with the Join engine", function_name, storage_id.getNameForLogs()); return std::make_unique(function_name, context, table_lock, storage_join, result_columns); } diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index ebdc9bd48d15c..c64b2735268ab 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -1,5 +1,7 @@ #include +#include +#include #include #include @@ -40,6 +42,7 @@ namespace Setting namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int ALIAS_REQUIRED; extern const int AMBIGUOUS_COLUMN_NAME; extern const int LOGICAL_ERROR; @@ -344,6 +347,23 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se { auto joined_table_id = context->resolveStorageID(table_to_join.database_and_table_name); StoragePtr storage = DatabaseCatalog::instance().tryGetTable(joined_table_id, context); + + /// A special storage replaces the right-side plan, and with it the `FilterStep` carrying the + /// table's row policy, so such a table has to be joined as an ordinary stream. A `Join` table + /// is a prebuilt hash table read as is, so it cannot be filtered at all. + if (storage) + { + auto row_policy_filter = context->getRowPolicyFilter( + joined_table_id.getDatabaseName(), joined_table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + { + if (typeid_cast(storage.get())) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot join table {} with the Join engine because a row policy is applied on it", joined_table_id.getNameForLogs()); + storage = nullptr; + } + } + if (storage) { if (auto storage_join = std::dynamic_pointer_cast(storage); storage_join) diff --git a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp index 63dcf97699614..e6e8bea7217d9 100644 --- a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp @@ -315,3 +315,305 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & return true; } } + +namespace DB +{ + +void registerStatementRowPolicy(StatementFactory & factory) +{ + factory.registerStatement("CREATE ROW POLICY", + { + .description = R"DOCS_MD( +Creates a [row policy](/concepts/features/security/access-rights#row-policy-management), i.e. a filter used to determine which rows a user can read from a table. + + +Row policies make sense only for users with readonly access. If a user can modify a table or copy partitions between tables, it defeats the restrictions of row policies. + + +Syntax: + +```sql +-- Multiple names on one table target +CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [, ...] + [ON CLUSTER cluster_name] + ON { [db.]table | db.* } + [IN access_storage_type] + [[FOR SELECT] USING {condition | NONE}] + [AS {PERMISSIVE | RESTRICTIVE}] + [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] + +-- One name on multiple table targets +CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name + [ON CLUSTER cluster_name] + ON { [db.]table | db.* } [, ...] + [IN access_storage_type] + [[FOR SELECT] USING {condition | NONE}] + [AS {PERMISSIVE | RESTRICTIVE}] + [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] + +-- Mixed packing: each name paired with its own table target +CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] + policy_name ON { [db.]table | db.* } [, policy_name ON { [db.]table | db.* } ...] + [ON CLUSTER cluster_name] + [IN access_storage_type] + [[FOR SELECT] USING {condition | NONE}] + [AS {PERMISSIVE | RESTRICTIVE}] + [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] +``` + +`ParserRowPolicyNames` accepts **three** packing forms (not a full Cartesian product): + +1. **Multiple names, one target** — `pol1, pol2 ON table1` creates each listed name on that single table (or `db.*`). +2. **One name, multiple targets** — `pol1 ON table1, table2` creates the same short name on each listed target. +3. **Mixed pairs** — `p1 ON t1, p2 ON t2` creates each name only on its paired target. + +A multi-name list **cannot** be combined with a multi-table `ON` list in one group: `p1, p2 ON t1, t2` is rejected. After a multi-name group, you also cannot append another comma-separated `name ON target` group in the same statement. + +Optional `ON CLUSTER` applies to the whole statement (one cluster name). ClickHouse does **not** accept a different `ON CLUSTER` per policy name packed into a single create — run separate `CREATE ROW POLICY` statements when policies must be created on different clusters. + +`CREATE ROW POLICY` requires the [CREATE ROW POLICY](/reference/statements/grant#access-management) privilege on the table the policy is created on. `OR REPLACE` throws away an existing policy of the same name, including which roles it applies to, so it additionally requires the [DROP ROW POLICY](/reference/statements/grant#access-management) privilege on that table. The `DROP ROW POLICY` privilege is required whether or not the policy already exists, so the statement cannot be used to find out which policies exist. + +## Multiple names and tables {#multiple-names-and-tables} + +Valid: + +```sql +-- Several policy names, one table +CREATE ROW POLICY pol1, pol2, pol3 ON table1 + FOR SELECT USING id = 1 + TO accountant; + +-- One policy name, several tables +CREATE ROW POLICY IF NOT EXISTS pol1 ON table1, table2, table3 + FOR SELECT USING id = 1 + TO accountant; + +-- Mixed packing: different name per table +CREATE ROW POLICY p4 ON db.table, p5 ON db2.table2 + USING a = b; + +-- Same policy on several tables, on a cluster +CREATE ROW POLICY IF NOT EXISTS pol1 ON CLUSTER replicated_cluster ON table1, table2 + FOR SELECT USING id = 1 + TO accountant; +``` + +Invalid: + +```sql +-- Multi-name × multi-table in one ON-group (not a Cartesian product) +CREATE ROW POLICY p1, p2 ON t1, t2 + FOR SELECT USING id = 1 + TO accountant; + +-- Different clusters per name in one statement +CREATE ROW POLICY pol1 ON CLUSTER cluster1 ON table1, pol2 ON CLUSTER cluster2 ON table2 +``` + +## USING clause {#using-clause} + +Defines a filter condition for a table. A user can only see rows for which the condition is true (evaluates to a non-zero value). This is similar to adding an extra `WHERE` condition to every query the user runs against the table. + +For example, the following policy limits `analyst_role` to rows from the EU: + +```sql +CREATE ROW POLICY region_filter ON db.orders +USING region = 'EU' +TO analyst_role; +``` + +With this policy, `SELECT * FROM db.orders` returns the same rows as `SELECT * FROM db.orders WHERE region = 'EU'` would. + +## TO Clause {#to-clause} + +In the `TO` section you can provide a list of users and roles this policy should work for. For example, `CREATE ROW POLICY ... TO accountant, john@localhost`. + +Keyword `ALL` means all the ClickHouse users, including current user. Keyword `ALL EXCEPT` allows excluding some users from the all users list, for example, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` + +Roles named in the `TO` section, including those after `ALL EXCEPT`, are matched against the current user's enabled roles ([`system.enabled_roles`](/reference/system-tables/enabled_roles)), not against every role granted to the user, so [`SET ROLE`](/reference/statements/set-role) can change which policies apply. + +## AS Clause {#as-clause} + +It's allowed to have more than one policy enabled on the same table for the same user at one time. So we need a way to combine the conditions from multiple policies. + +By default, policies are combined using the boolean `OR` operator. For example, the following policies: + +```sql +CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter +CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio +``` + +enable the user `peter` to see rows with either `b=1` or `c=2`. + +The `AS` clause specifies how policies should be combined with other policies. Policies can be either permissive or restrictive. By default, policies are permissive, which means they are combined using the boolean `OR` operator. + +A policy can be defined as restrictive as an alternative. Restrictive policies are combined using the boolean `AND` operator. + +Here is the general formula: + +```text +row_is_visible = (one or more of the conditions from the permissive policies that apply to the current user and their enabled roles are non-zero) AND + (all of the conditions from the restrictive policies that apply to the current user and their enabled roles are non-zero) +``` + +If no permissive condition applies, the first condition has no effect and only the restrictive policies decide, because `access_control_improvements.users_without_row_policies_can_read_rows` is enabled by default. A user to whom no condition applies therefore sees every row, and `access_control_improvements.throw_on_unmatched_row_policies`, disabled by default, raises an exception instead when the table does have conditions and none of them apply. + +For example, the following policies: + +```sql +CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter +CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio +``` + +enable the user `peter` to see rows only if both `b=1` AND `c=2`. + +Database policies are combined with table policies. + +For example, the following policies: + +```sql +CREATE ROW POLICY pol1 ON mydb.* USING b=1 TO mira, peter +CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio +``` + +enable the user `peter` to see table1 rows only if both `b=1` AND `c=2`, although +any other table in mydb would have only `b=1` policy applied for the user. + +## Tables that read from other tables {#tables-that-read-from-other-tables} + +A row policy filters rows where the data is actually read. An `Alias` table returns the rows of its target table as its own, so the row policies of the target apply to reads through the alias as well, combined with the policies of the alias itself using a logical `AND`. A `Merge` table applies the policies of the tables it reads from. One exception: when a matched table reads remotely, such as a `Distributed` table, the remote server processes the query before the policy is applied, because the policy runs above that table's read rather than at the read. Such a query can fail, when it aggregates without selecting the policy's columns, or return fewer rows than the policy allows, when the remote server applies an `ORDER BY ... LIMIT` to rows the policy would have hidden. Define the policy on the underlying local tables of each remote server instead. + +This does not extend to every table that reads from another table. A `Buffer` table and a materialized view read through their destination or target table do **not** inherit that table's row policies: the policy is written against the target's schema and, for a view with `SQL SECURITY DEFINER`, is evaluated for a different user than the one running the read. Define the policy on the table users actually query in those cases. + +## Distributed and remote-backed tables {#distributed-and-remote-backed-tables} + +A row policy filters rows where the table data is actually read. A table that delegates reading to remote servers, such as a [Distributed](/reference/engines/table-engines/special/distributed) table or a wrapper over one (for example, a materialized view with a `Distributed` target), only ships the query text to the remote servers and cannot apply the policy filter to the remote read. To keep the filter from being silently dropped, queries to such a table by users the policy applies to are rejected with an `ILLEGAL_PREWHERE` error. + +Instead, define the policy on the underlying local tables on each remote server; it is applied there when the shipped query reads them: + +```sql +-- Filters reads of local_table on this server, including reads shipped by a Distributed table over it. +CREATE ROW POLICY filter ON mydb.local_table USING a < 1000 TO john; +``` + + +This works while the query is shipped as text, which is the default. With [`serialize_query_plan = 1`](/reference/settings/session-settings/serialize#serialize_query_plan) the initiator ships an already-built read plan instead, and a remote server executing such a plan does not apply its own row policies, so a read of a `Distributed` table over `local_table` returns unfiltered rows. Keep `serialize_query_plan = 0` for users whose row policies must be enforced. See [issue #112891](https://github.com/ClickHouse/ClickHouse/issues/112891). + + +## Join tables {#join-tables} + +A [Join](/reference/engines/table-engines/special/join) table is a prepared hash table that a `JOIN` or `joinGet` reads as is, so its rows cannot be filtered there. A policy on such a table, including a database-wide `ON db.*` policy, filters a plain `SELECT` from the table, but while it applies, `JOIN` and `joinGet` queries against the table fail with `ACCESS_DENIED`. + +## ON CLUSTER Clause {#on-cluster-clause} + +Allows creating row policies on a cluster, see [Distributed DDL](/reference/statements/distributed-ddl). This is also the convenient way to create the policy on the local tables of every server of the cluster. + +## Examples {#examples} + +`CREATE ROW POLICY filter1 ON mydb.mytable USING a<1000 TO accountant, john@localhost` + +`CREATE ROW POLICY filter2 ON mydb.mytable USING a<1000 AND b=5 TO ALL EXCEPT mira` + +`CREATE ROW POLICY filter3 ON mydb.mytable USING 1 TO admin` + +`CREATE ROW POLICY filter4 ON mydb.* USING 1 TO admin` +)DOCS_MD", + .syntax = R"( +CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [, ...] + [ON CLUSTER cluster_name] + ON { [db.]table | db.* } [, ...] + [IN access_storage_type] + [[FOR SELECT] USING {condition | NONE}] + [AS {PERMISSIVE | RESTRICTIVE}] + [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] +)", + .parent = "CREATE", + .related = {"ALTER ROW POLICY", "CREATE MASKING POLICY", "CREATE ROLE", "DROP", "SHOW"}, + }); + + factory.registerStatement("ALTER ROW POLICY", + { + .description = R"DOCS_MD( +Changes row policy. + +Syntax: + +```sql +-- Rename: exactly one fully qualified policy (one name on one target). +-- RENAME TO may be combined with the same optional alteration clauses as below. +ALTER [ROW] POLICY [IF EXISTS] name + ON { [database.]table | database.* } + RENAME TO new_name + [ON CLUSTER cluster_name] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] + +-- Multiple names on one table target (no RENAME) +ALTER [ROW] POLICY [IF EXISTS] name [, ...] + [ON CLUSTER cluster_name] + ON { [database.]table | database.* } + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] + +-- One name on multiple table targets (no RENAME) +ALTER [ROW] POLICY [IF EXISTS] name + [ON CLUSTER cluster_name] + ON { [database.]table | database.* } [, ...] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] + +-- Mixed packing: each name paired with its own table target (no RENAME) +ALTER [ROW] POLICY [IF EXISTS] + name ON { [database.]table | database.* } [, name ON { [database.]table | database.* } ...] + [ON CLUSTER cluster_name] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +`RENAME TO` is only accepted when the statement names **exactly one** policy on **one** table target. Packed multi-name or multi-table lists cannot include `RENAME TO` — rename those policies in separate statements. On that single-policy form, `RENAME TO` can still be combined with other alterations such as `AS`, `USING`, and `TO` in the same statement. + +Without `RENAME TO`, packing matches `ParserRowPolicyNames`: multiple names on **one** target, one name on **multiple** targets, or mixed `name ON target` pairs. Multi-name × multi-table (`p1, p2 ON t1, t2`) is **not** accepted. Optional `ON CLUSTER` applies once to the whole statement; run separate `ALTER ROW POLICY` statements when different clusters are required. + +Examples: + +```sql +-- Single-policy rename +ALTER ROW POLICY p1 ON db.table RENAME TO p1_new; + +-- Rename plus other alterations on the same single policy +ALTER POLICY old_name ON db.table RENAME TO new_name USING id > 10; + +-- Multiple names, one table +ALTER POLICY p1, p2 ON db.table TO ALL; + +-- One name, multiple tables +ALTER POLICY p1 ON db.table, db.table2 USING NONE; + +-- Mixed targets without rename +ALTER POLICY p1 ON db.table, p2 ON db2.table2 TO ALL; +``` +)DOCS_MD", + .syntax = R"( +ALTER [ROW] POLICY [IF EXISTS] name [, ...] + ON { [database.]table | database.* } [, ...] + [RENAME TO new_name] + [ON CLUSTER cluster_name] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +)", + .parent = "ALTER", + .related = {"CREATE ROW POLICY", "ALTER", "SHOW"}, + }); +} + +} diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 67ca7fb5945a6..2e0ee3dbd986c 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -184,6 +185,7 @@ namespace ErrorCodes extern const int PARAMETER_OUT_OF_BOUND; extern const int TOO_MANY_COLUMNS; extern const int UNSUPPORTED_METHOD; + extern const int ACCESS_DENIED; } namespace @@ -3121,12 +3123,22 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode( join_node, planner_context); - PreparedJoinStorage prepared_join; - bool allow_storage_join = right_join_tree_query_plan.used_row_policies.empty() + /// A prepared storage replaces the right-side plan, and with it the `FilterStep` applying the + /// table's row policy, so such a table is joined as a stream. A `Join` table is a prebuilt + /// hash table read as is, so it cannot be filtered at all. + bool right_table_has_row_policy = !right_join_tree_query_plan.used_row_policies.empty(); + + PreparedJoinStorage prepared_join = tryGetStorageInTableJoin(join_node.getRightTableExpressionNode(), planner_context); + if (prepared_join.storage_join && right_table_has_row_policy) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot join table {} with the Join engine because a row policy is applied on it", + prepared_join.storage_join->getStorageID().getNameForLogs()); + + bool allow_storage_join = !right_table_has_row_policy && right_join_tree_query_plan.stage == QueryProcessingStage::FetchColumns && right_join_tree_query_plan.useful_sets.empty(); - if (allow_storage_join) - prepared_join = tryGetStorageInTableJoin(join_node.getRightTableExpressionNode(), planner_context); + if (!allow_storage_join) + prepared_join = {}; if (prepared_join) { bool use_nulls = settings[Setting::join_use_nulls] && isLeftOrFull(join_node.getKind()); diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index cbbb62afde5a5..cdc7a5c14c2f6 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -641,6 +641,8 @@ Default value: `1`. The `Join`-engine tables can't be used in `GLOBAL JOIN` operations. +A [row policy](/reference/statements/create/row-policy) on a `Join`-engine table filters a plain `SELECT` from it, but a `JOIN` or `joinGet` reads the prepared hash table as is and cannot filter its rows, so while a policy applies to the table such queries fail with `ACCESS_DENIED`. + The `Join`-engine allows to specify [join_use_nulls](/reference/settings/session-settings/join#join_use_nulls) setting in the `CREATE TABLE` statement. [SELECT](/reference/statements/select/index) query should have the same `join_use_nulls` value. ## Usage examples {#example} diff --git a/tests/queries/0_stateless/05233_direct_join_row_policy.reference b/tests/queries/0_stateless/05233_direct_join_row_policy.reference new file mode 100644 index 0000000000000..523709ccc7546 --- /dev/null +++ b/tests/queries/0_stateless/05233_direct_join_row_policy.reference @@ -0,0 +1,43 @@ +-- plain select +1 public public-1 +3 public public-3 +-- inner +1 public public-1 +3 public public-3 +-- inner, policy column not selected +1 public-1 +3 public-3 +-- left +1 1 public-1 +2 0 +3 3 public-3 +5 0 +-- left, join_use_nulls +1 1 public-1 +2 \N \N +3 3 public-3 +5 \N \N +-- left any +1 public-1 +2 +3 public-3 +5 +-- left semi +1 +3 +-- left anti +2 +5 +-- key space enumeration +1 public-1 +3 public-3 +-- using +1 public-1 +3 public-3 +-- no direct join under a row policy +0 +-- direct join without a policy +1 public public-1 +2 hidden hidden-2 +3 public public-3 +Algorithm: DirectKeyValueJoin diff --git a/tests/queries/0_stateless/05233_direct_join_row_policy.sql b/tests/queries/0_stateless/05233_direct_join_row_policy.sql new file mode 100644 index 0000000000000..e2a073ca13a26 --- /dev/null +++ b/tests/queries/0_stateless/05233_direct_join_row_policy.sql @@ -0,0 +1,63 @@ +-- Tags: use-rocksdb + +DROP TABLE IF EXISTS kv_rls; +DROP TABLE IF EXISTS probe_rls; + +CREATE TABLE kv_rls (key UInt64, tenant String, secret String) ENGINE = EmbeddedRocksDB PRIMARY KEY key; +INSERT INTO kv_rls VALUES (1, 'public', 'public-1'), (2, 'hidden', 'hidden-2'), (3, 'public', 'public-3'), (4, 'hidden', 'hidden-4'); + +CREATE TABLE probe_rls (key UInt64) ENGINE = TinyLog; +INSERT INTO probe_rls VALUES (1), (2), (3), (5); + +CREATE ROW POLICY kv_rls_public ON kv_rls FOR SELECT USING tenant = 'public' TO CURRENT_USER; + +SET join_algorithm = 'direct,hash'; +SET join_use_nulls = 0; + +SELECT '-- plain select'; +SELECT key, tenant, secret FROM kv_rls ORDER BY key; + +SELECT '-- inner'; +SELECT p.key, kv.tenant, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; + +SELECT '-- inner, policy column not selected'; +SELECT p.key, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; + +SELECT '-- left'; +SELECT p.key, kv.key, kv.secret FROM probe_rls AS p LEFT JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; + +SELECT '-- left, join_use_nulls'; +SELECT p.key, kv.key, kv.secret FROM probe_rls AS p LEFT JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key SETTINGS join_use_nulls = 1; + +SELECT '-- left any'; +SELECT p.key, kv.secret FROM probe_rls AS p LEFT ANY JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; + +SELECT '-- left semi'; +SELECT p.key FROM probe_rls AS p LEFT SEMI JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; + +SELECT '-- left anti'; +SELECT p.key FROM probe_rls AS p LEFT ANTI JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; + +SELECT '-- key space enumeration'; +SELECT kv.key, kv.secret FROM numbers(10) AS n INNER JOIN kv_rls AS kv ON kv.key = n.number ORDER BY kv.key; + +SELECT '-- using'; +SELECT key, secret FROM probe_rls INNER JOIN kv_rls USING (key) ORDER BY key; + +SELECT '-- no direct join under a row policy'; +SELECT count() FROM ( + EXPLAIN actions = 1 + SELECT p.key, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key +) WHERE explain LIKE '%DirectKeyValueJoin%'; + +DROP ROW POLICY kv_rls_public ON kv_rls; + +SELECT '-- direct join without a policy'; +SELECT p.key, kv.tenant, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; +SELECT extract(explain, 'Algorithm: \\w+') FROM ( + EXPLAIN actions = 1 + SELECT p.key, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key +) WHERE explain LIKE '%Algorithm:%'; + +DROP TABLE kv_rls; +DROP TABLE probe_rls; diff --git a/tests/queries/0_stateless/05234_row_policy_storage_join.reference b/tests/queries/0_stateless/05234_row_policy_storage_join.reference new file mode 100644 index 0000000000000..2ac892e8f5052 --- /dev/null +++ b/tests/queries/0_stateless/05234_row_policy_storage_join.reference @@ -0,0 +1,12 @@ +-- without a policy +1 a +2 b +3 +b \N +-- with a policy +1 a +-- after dropping the policy +1 a +2 b +3 +b \N diff --git a/tests/queries/0_stateless/05234_row_policy_storage_join.sql b/tests/queries/0_stateless/05234_row_policy_storage_join.sql new file mode 100644 index 0000000000000..91d6f948686d2 --- /dev/null +++ b/tests/queries/0_stateless/05234_row_policy_storage_join.sql @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS join_rls; +DROP TABLE IF EXISTS probe_join_rls; +DROP ROW POLICY IF EXISTS join_rls_policy ON join_rls; + +CREATE TABLE join_rls (key UInt64, value String) ENGINE = Join(ANY, LEFT, key); +INSERT INTO join_rls VALUES (1, 'a'), (2, 'b'); + +CREATE TABLE probe_join_rls (key UInt64) ENGINE = TinyLog; +INSERT INTO probe_join_rls VALUES (1), (2), (3); + +SELECT '-- without a policy'; +SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; +SELECT joinGet(join_rls, 'value', toUInt64(2)), joinGetOrNull(join_rls, 'value', toUInt64(3)); + +CREATE ROW POLICY join_rls_policy ON join_rls FOR SELECT USING value = 'a' TO CURRENT_USER; + +SELECT '-- with a policy'; +SELECT key, value FROM join_rls ORDER BY key; +SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; -- { serverError ACCESS_DENIED } +SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key SETTINGS join_algorithm = 'hash'; -- { serverError ACCESS_DENIED } +SELECT joinGet(join_rls, 'value', toUInt64(2)); -- { serverError ACCESS_DENIED } +SELECT joinGetOrNull(join_rls, 'value', toUInt64(2)); -- { serverError ACCESS_DENIED } + +DROP ROW POLICY join_rls_policy ON join_rls; + +SELECT '-- after dropping the policy'; +SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; +SELECT joinGet(join_rls, 'value', toUInt64(2)), joinGetOrNull(join_rls, 'value', toUInt64(3)); + +DROP TABLE join_rls; +DROP TABLE probe_join_rls; From 4a0ed782f3d39bec3303a94858308d87ba5fc989 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 21 Sep 2026 06:35:24 +0000 Subject: [PATCH 140/296] Hide credentials in NATS/RabbitMQ addresses, XDBC connection strings and URL settings Six call sites still masked a credential embedded in a URL-shaped value with maskURIPassword, an unanchored scan that reproduces the old regular expression ([^:]+://[^:]*):([^@]*)@(.*) exactly: it masks from the first ':' after any '://' to the first '@'. The masked extent has to cover whatever the value's consumer treats as a credential, and that scan matches no consumer in the tree, so it is at once too narrow (a password containing '@' keeps its tail, and a userinfo with no password is not masked at all) and too wide (a credential-free URL whose query string contains '@' is rewritten into a host that is not the real one). All three show up in SHOW CREATE, system.tables and system.query_log. Swapping in maskURIUserinfo at every site, which is what the issue asks for, would replace a partial leak with a full one at five of the six. That function implements the RFC 3986 bound, which is Poco::URI's bound, and three of these consumers are not Poco::URI: - libnats trims the value, takes the scheme as optional and ends the userinfo at the LAST '@' of the whole remainder, unbounded by the '/?#' that closes an authority (contrib/nats-io/src/url.c, natsUrl_Create; both halves go into the CONNECT frame in conn.c). So nats://u:pa/ss@h:4222 authenticates with password pa/ss, which maskURIUserinfo leaves untouched, and user:pass@h:4222 authenticates while neither masker touches it. - AMQP-CPP ends the login at the FIRST '@' after the scheme, again unbounded (contrib/AMQP-CPP/include/amqpcpp/address.h). - an XDBC connection string is never parsed by ClickHouse: it is read verbatim in ITableFunctionXDBC and forwarded to the bridge as one opaque parameter, so its grammar is the driver's. This repository's own examples put the password in a query parameter, and the majority spelling is DSN=...;Pwd=... . No URI scan can bound a credential in that. A password containing '/' is an ordinary shape, so this is not a corner case. Apply one rule instead: a value is echoed with only its credential hidden where ClickHouse owns its grammar, and a value whose grammar belongs to someone else is not echoed. The second half is already the policy in DatabaseURL.cpp and in the old XDBC fallback. Concretely, the two broker addresses and the XDBC connection string are hidden whole (for the brokers, exactly when the value carries an '@', which is a superset of anything either parser can read as a credential and is empty precisely when there is nothing to hide, so the documented forms such as localhost:4222 and amqp://h:5672/v stay fully visible). The three query-level URL settings are read through Poco::URI, so there the partial mask is right and is kept: it moves from maskURIPassword to maskURIUserinfo, which is what fixes the reporter's three cases. They gain a fail-closed leg, because nothing validates them at SET time, resolveURLBase accepts a base whose '://' is anywhere rather than at the start, and a statement is masked for logging before its settings are validated: a value with no scheme in front of it therefore reaches a log, and no parser can say which part of it is the credential. url_base additionally reaches the Azure URL parser, where abfs and abfss put a non-secret container in front of the '@', so it gets its own rule that skips the userinfo leg for those two schemes; az and azure have no '@' in their grammar and need no exception. Two holes in the XDBC named-collection branch close as a side effect of dropping its bespoke helper for findSecretNamedArgument: only the first occurrence of a duplicated datasource key used to be masked, and a key written as a constant expression (concat('data', 'source') = ...) was applied by the named-collection parser while being invisible to this finder, so the whole connection string was echoed. The fail-closed scan that already covered that hazard for the TLS keys is extracted and reused, leaving one carrier for it. The anchored-scheme test is likewise reduced to one carrier, findURIAuthority, which also replaces a private copy in DatabaseURL.cpp that used the locale-dependent std::isalpha and std::isalnum. Deliberate consequences: a broker address carrying a stray '@' and any URI-shaped connection string are now hidden whole, and format_display_secrets_in_show_and_select still shows the real value to a privileged reader. maskURIUserinfo itself is unchanged, which the re2 equivalence test proves by passing unedited. Closes: https://github.com/ClickHouse/ClickHouse/issues/121257 --- src/Common/maskURIPassword.h | 40 ++++++--- src/Core/SettingsSecrets.h | 40 ++++++++- src/Databases/DatabaseURL.cpp | 25 +----- src/Parsers/FunctionSecretArgumentsFinder.cpp | 72 ++++++---------- src/Parsers/FunctionSecretArgumentsFinder.h | 12 +-- src/Parsers/tests/gtest_Parser.cpp | 7 +- src/Storages/NATS/NATS_fwd.h | 8 +- src/Storages/RabbitMQ/RabbitMQ_fwd.h | 8 +- .../test_mask_sensitive_info/test.py | 20 ++--- .../integration/test_storage_rabbitmq/test.py | 2 +- ...etting_ast_json_and_secret_parts.reference | 2 +- .../05056_settings_secret_masking.reference | 10 +-- ...oker_and_xdbc_credential_masking.reference | 28 ++++++ ...5233_broker_and_xdbc_credential_masking.sh | 85 +++++++++++++++++++ 14 files changed, 244 insertions(+), 115 deletions(-) create mode 100644 tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference create mode 100755 tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh diff --git a/src/Common/maskURIPassword.h b/src/Common/maskURIPassword.h index afc505b185e8c..d1f88c5e43b7d 100644 --- a/src/Common/maskURIPassword.h +++ b/src/Common/maskURIPassword.h @@ -73,15 +73,11 @@ inline bool maskURIPassword(std::string * uri) return false; } -/** Mask the userinfo part of a URL: `scheme://anything@rest` becomes `scheme://[HIDDEN]@rest`. - * Returns whether anything was masked. - * - * This used to be the regular expression `^([a-zA-Z][a-zA-Z0-9+.-]*://)[^/?#]+@` rewritten to - * `\1[HIDDEN]@`. Only a match at the start of the string counts, and the userinfo is taken - * greedily up to the last '@' before the path, so a password that itself contains an at-sign is - * masked whole. `src/Common/tests/gtest_mask_uri_password.cpp` checks this against re2. +/** The offset just past the `://` of a value that starts with an RFC 3986 scheme, and `npos` when it + * does not start with one. A value with no scheme at its start has no authority that a URI parser + * would recognise, so nothing in it can be located as a credential by position. */ -inline bool maskURIUserinfo(std::string & url) +inline size_t findURIAuthority(std::string_view uri) { static constexpr std::string_view SEPARATOR = "://"; @@ -90,19 +86,35 @@ inline bool maskURIUserinfo(std::string & url) auto is_letter = [](char c) { return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'); }; auto is_letter_or_digit = [&](char c) { return is_letter(c) || ('0' <= c && c <= '9'); }; - if (url.empty() || !is_letter(url[0])) - return false; + if (uri.empty() || !is_letter(uri[0])) + return std::string_view::npos; size_t scheme_end = 1; - while (scheme_end < url.length() - && (is_letter_or_digit(url[scheme_end]) || url[scheme_end] == '+' || url[scheme_end] == '.' || url[scheme_end] == '-')) + while (scheme_end < uri.length() + && (is_letter_or_digit(uri[scheme_end]) || uri[scheme_end] == '+' || uri[scheme_end] == '.' || uri[scheme_end] == '-')) ++scheme_end; - if (url.compare(scheme_end, SEPARATOR.length(), SEPARATOR) != 0) + if (uri.compare(scheme_end, SEPARATOR.length(), SEPARATOR) != 0) + return std::string_view::npos; + + return scheme_end + SEPARATOR.length(); +} + +/** Mask the userinfo part of a URL: `scheme://anything@rest` becomes `scheme://[HIDDEN]@rest`. + * Returns whether anything was masked. + * + * This used to be the regular expression `^([a-zA-Z][a-zA-Z0-9+.-]*://)[^/?#]+@` rewritten to + * `\1[HIDDEN]@`. Only a match at the start of the string counts, and the userinfo is taken + * greedily up to the last '@' before the path, so a password that itself contains an at-sign is + * masked whole. `src/Common/tests/gtest_mask_uri_password.cpp` checks this against re2. + */ +inline bool maskURIUserinfo(std::string & url) +{ + size_t authority_begin = findURIAuthority(url); + if (authority_begin == std::string::npos) return false; /// `[^/?#]+@` - the userinfo, greedy, so it ends at the last '@' before the path. - size_t authority_begin = scheme_end + SEPARATOR.length(); size_t authority_end = url.find_first_of("/?#", authority_begin); if (authority_end == std::string::npos) authority_end = url.length(); diff --git a/src/Core/SettingsSecrets.h b/src/Core/SettingsSecrets.h index dd6ad89674a76..77d883ee75ec9 100644 --- a/src/Core/SettingsSecrets.h +++ b/src/Core/SettingsSecrets.h @@ -27,14 +27,48 @@ using ValueMaskingFunc = std::function; /// precondition holds by construction and needs no check. inline bool maskURLCredentials(String & value) { - bool masked = maskURIPassword(&value); + /// Nothing validates these settings when they are set, `StorageURL::resolveURLBase` accepts a base + /// whose `://` is anywhere rather than at the start, and a statement is masked for logging before + /// its settings are validated. So a value with no scheme in front still reaches a log, and no URI + /// parser can locate the credential inside it: hide such a value whole. + if (findURIAuthority(value) == String::npos && value.contains('@')) + { + value = "[HIDDEN]"; + return true; + } + + bool masked = maskURIUserinfo(value); masked |= maskPresignedURLParameters(value); return masked; } +/// Under `abfs`/`abfss` the part in front of the `@` is the container and not a credential (the +/// Hadoop grammar `abfss://@.dfs.core.windows.net/`), and the credential +/// is the SAS in the query string. `az`/`azure` have no `@` in their grammar and need no exception. +/// Keep the two names in sync with `parseAzureURL` in `src/Storages/StorageURL.cpp`, which compares +/// them after lowercasing; `Core` cannot depend on `Storages` to share the list. +inline bool maskURLBaseCredentials(String & value) +{ + static constexpr std::string_view azure_container_schemes[] = {"abfs", "abfss"}; + + if (size_t authority = findURIAuthority(value); authority != String::npos) + { + String scheme = value.substr(0, authority - 3); + for (auto & c : scheme) + if ('A' <= c && c <= 'Z') + c += 'a' - 'A'; + + for (auto azure_scheme : azure_container_schemes) + if (scheme == azure_scheme) + return maskPresignedURLParameters(value); + } + + return maskURLCredentials(value); +} + /// The settings of the query-level `Settings` collection whose value can carry a credential, and how /// each one is masked. `system.query_log.query` shows -/// `format_avro_schema_registry_url = 'http://user:[HIDDEN]@registry:8080'`, so every other place that +/// `format_avro_schema_registry_url = 'http://[HIDDEN]@registry:8080'`, so every other place that /// prints the same value hides the same secret through this map. /// /// Mirrors the per-engine `SETTINGS_TO_HIDE` maps (`Kafka_fwd.h`, `NATS_fwd.h`, ...), which do this @@ -42,7 +76,7 @@ inline bool maskURLCredentials(String & value) static inline std::unordered_map SETTINGS_TO_HIDE = { {"format_avro_schema_registry_url", maskURLCredentials}, - {"url_base", maskURLCredentials}, + {"url_base", maskURLBaseCredentials}, {"s3_base", maskURLCredentials}, }; diff --git a/src/Databases/DatabaseURL.cpp b/src/Databases/DatabaseURL.cpp index b08e1332830f6..5ff8e6312e0fa 100644 --- a/src/Databases/DatabaseURL.cpp +++ b/src/Databases/DatabaseURL.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -42,26 +43,6 @@ namespace ErrorCodes namespace { -/// Check that the string starts with a valid RFC 3986 scheme followed by "://". -bool hasURLScheme(const String & url) -{ - auto scheme_end = url.find("://"); - if (scheme_end == String::npos || scheme_end == 0) - return false; - - if (!std::isalpha(static_cast(url[0]))) - return false; - - for (size_t i = 1; i < scheme_end; ++i) - { - char c = url[i]; - if (!std::isalnum(static_cast(c)) && c != '+' && c != '-' && c != '.') - return false; - } - - return true; -} - /// A table of a `URL` database is a thin wrapper over the `url` table function, which dispatches /// the URL to the matching backend (`file`, `s3`, `azureBlobStorage`, ...). struct URLTableDelegate @@ -265,7 +246,7 @@ DatabaseURL::DatabaseURL(const String & name_, const String & base_url_, Context : IDatabase(name_), WithContext(context_->getGlobalContext()), base_url(base_url_) { /// Not echoed back: password masking anchors on the `://` this value lacks, so it would log the password. - if (!base_url.empty() && !hasURLScheme(base_url)) + if (!base_url.empty() && findURIAuthority(base_url) == String::npos) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The base URL of a URL database must contain a scheme (e.g. https://)"); } @@ -274,7 +255,7 @@ String DatabaseURL::getTableURL(const String & name) const { String resolved = StorageURL::resolveURLBase(name, base_url, "base URL of the URL database"); - if (!hasURLScheme(resolved)) + if (findURIAuthority(resolved) == String::npos) return {}; return resolved; } diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 95bfbc06b57e5..6e184d5131223 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -396,11 +396,8 @@ void FunctionSecretArgumentsFinder::findMySQLFunctionSecretArguments() } } -void FunctionSecretArgumentsFinder::findTLSCredentialsSecretArguments(size_t start) +void FunctionSecretArgumentsFinder::markNamedArgumentsWithUnreadableKeys(size_t start) { - for (const auto & key : tls_credentials_secret_keys) - findSecretNamedArgument(key, start); - /// The named-collection parser does not require the key of a `key = value` argument to be a plain /// literal or identifier: `getKeyValueFromASTImpl` evaluates it as a constant expression, so /// `mysql(creds, concat('ssl_ca', '_pem') = 'SECRET', table = 't')` passes a TLS credential too. @@ -423,6 +420,14 @@ void FunctionSecretArgumentsFinder::findTLSCredentialsSecretArguments(size_t sta } } +void FunctionSecretArgumentsFinder::findTLSCredentialsSecretArguments(size_t start) +{ + for (const auto & key : tls_credentials_secret_keys) + findSecretNamedArgument(key, start); + + markNamedArgumentsWithUnreadableKeys(start); +} + void FunctionSecretArgumentsFinder::findMongoDBSecretArguments() { String uri; @@ -493,12 +498,17 @@ void FunctionSecretArgumentsFinder::findArrowFlightSecretArguments() void FunctionSecretArgumentsFinder::findXDBCSecretArguments() { + /// The connection string is never parsed by ClickHouse: `ITableFunctionXDBC` forwards it verbatim + /// to the bridge, so its grammar belongs to the JDBC/ODBC driver. It can carry the password in a + /// query parameter (`jdbc('mysql://host:3306/?user=root&password=root', ...)`, from this function's + /// own documentation) or as `Pwd=` in a `KEY=value;` list, neither of which a URI scan locates. + /// There is no extent that can be kept visible, so the value is hidden whole; + /// `format_display_secrets_in_show_and_select` still shows the real one to a privileged reader. if (isNamedCollectionName(0)) { /// jdbc(named_collection, ..., datasource = 'DSN', ...) /// odbc(named_collection, ..., connection_settings = 'DSN', ...) /// `datasource` and `connection_settings` are mutually exclusive aliases. - /// If the value is a URI, mask only the password; otherwise hide the whole value. /// If somehow both are present (invalid query), hide all named arguments. ssize_t ds_idx = findNamedArgument(nullptr, "datasource", 1); ssize_t cs_idx = findNamedArgument(nullptr, "connection_settings", 1); @@ -509,56 +519,22 @@ void FunctionSecretArgumentsFinder::findXDBCSecretArguments() result.start = 1; result.count = function->arguments->size() - 1; result.are_named = true; + return; } - else if (ds_idx >= 0) - maskXDBCSecretNamedArgument("datasource", 1); - else if (cs_idx >= 0) - maskXDBCSecretNamedArgument("connection_settings", 1); + + findSecretNamedArgument("datasource", 1); + findSecretNamedArgument("connection_settings", 1); + markNamedArgumentsWithUnreadableKeys(1); } else { /// jdbc('DSN', schema, table) / jdbc('DSN', table) /// odbc('DSN', schema, table) / odbc('DSN', table) /// JDBC('DSN', database, table) / ODBC('DSN', database, table) - /// The connection string may be a URI with credentials embedded, - /// e.g. scheme://username:password@host:port/dbname - /// If so, mask only the password part; otherwise hide the whole argument. - String uri; - if (tryGetStringFromArgument(0, &uri)) - { - if (maskURIPassword(&uri)) - { - chassert(result.count == 0); - result.start = 0; - result.count = 1; - result.replacement = std::move(uri); - return; - } - } markSecretArgument(0, false); } } -void FunctionSecretArgumentsFinder::maskXDBCSecretNamedArgument(std::string_view key, size_t start) -{ - String value; - ssize_t arg_idx = findNamedArgument(&value, key, start); - if (arg_idx < 0) - return; - - if (!value.empty() && maskURIPassword(&value)) - { - result.are_named = true; - result.start = arg_idx; - result.count = 1; - result.replacement = std::move(value); - } - else - { - markSecretArgument(arg_idx, /* argument_is_named= */ true); - } -} - void FunctionSecretArgumentsFinder::findS3FunctionSecretArguments(bool is_cluster_function) { /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument. @@ -1014,7 +990,7 @@ void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() /// The only positional argument the engine accepts is the name of a named collection, so the /// credentials can only appear as named overrides. The `SETTINGS` clause form is masked /// separately by `NATS::SETTINGS_TO_HIDE`, and this function masks the same keys the same way: - /// the secrets are hidden whole, while `nats_url` keeps everything but its userinfo password. + /// the secrets are hidden whole, and so is a `nats_url` that carries an '@'. /// `nats_server_list` is hidden whole because each list entry can carry userinfo credentials. /// Fail closed on a key we cannot read as a plain literal: it can name a secret setting. for (size_t i = 0; i < function->arguments->size(); ++i) @@ -1042,8 +1018,10 @@ void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() String url; if (equals_func->arguments->at(1)->tryGetString(&url, /* allow_identifier= */ false)) { - if (maskURIPassword(&url)) - result.replaced_arguments[i] = "nats_url = " + quoteString(url); + /// An '@' is the only reliable sign of a credential here, and there is no extent to + /// keep visible; see the `nats_url` rule in `NATS_fwd.h` for why. + if (url.contains('@')) + markSecretArgument(i, /* argument_is_named= */ true); } else { diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 821c215939aaa..3c86e22c3b3bc 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -120,6 +120,8 @@ class FunctionSecretArgumentsFinder /// Named arguments carrying NATS credentials. They are the setting names, because the `NATS` engine /// takes its arguments as overrides of a named collection (`NATS(collection, nats_token = '...')`). /// `nats_server_list` is a destination and can carry URI userinfo credentials, so hide it whole. + /// `nats_url` is not here because the documented form (`'localhost:4222'`) carries no credential and + /// stays visible; it is handled separately, by the presence of an '@'. /// Keep in sync with `NATS::SETTINGS_TO_HIDE`, which masks the same secrets in the `SETTINGS` clause. static constexpr std::string_view nats_secret_keys[] = {"nats_password", "nats_token", "nats_credential_file", "nats_credentials", "nats_server_list"}; @@ -171,11 +173,6 @@ class FunctionSecretArgumentsFinder void findRedisTableEngineSecretArguments(); void findArrowFlightSecretArguments(); void findXDBCSecretArguments(); - - /// Similar to `findSecretNamedArgument`, but if the value is a URI with credentials, - /// masks only the password part instead of hiding the entire value. - void maskXDBCSecretNamedArgument(std::string_view key, size_t start); - void findS3FunctionSecretArguments(bool is_cluster_function); void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function); bool maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named = false, size_t start = 0); @@ -248,6 +245,11 @@ class FunctionSecretArgumentsFinder /// duplicate-key validation runs, so `session_token = 'a', session_token = 'b'` must hide both. bool findSecretNamedArgument(std::string_view key, size_t start = 0); + /// Hides the value of every `key = value` argument from `start` on whose key this finder cannot read + /// as a plain literal or identifier. The named-collection parser evaluates such a key as a constant + /// expression, so it can name a secret argument that `findNamedArgument` never sees. + void markNamedArgumentsWithUnreadableKeys(size_t start); + /// Masks the secrets of an S3 named-collection form: the secret named overrides (every occurrence, /// in any order; the span covering them may hide a non-secret argument in between, which is safe) /// and the `headers(...)` / `extra_credentials(...)` map overrides. diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index 588067e1751f8..9e9a699dc4b37 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -296,8 +296,8 @@ TEST(ParserCreateQuery, MaskNATSTableEngineCredentials) TEST(ParserCreateQuery, MaskNATSTableEngineURLPassword) { - /// A `nats_url` override can carry the credentials in its userinfo. Only the password is hidden, - /// keeping the rest of the url visible, the same way the `SETTINGS` clause form is masked. + /// A `nats_url` override carrying an '@' is hidden whole, the same way the `SETTINGS` clause form + /// is masked: libnats reads a credential that no URI masker can bound. const String query = "CREATE TABLE test_nats (key UInt64) " "ENGINE = NATS(nats1, nats_url = 'nats://plain_user:plain_password@example.com:4222')"; @@ -308,7 +308,8 @@ TEST(ParserCreateQuery, MaskNATSTableEngineURLPassword) const String masked = ast->formatForLogging(); EXPECT_EQ(masked.find("plain_password"), String::npos); - EXPECT_NE(masked.find("nats://plain_user:[HIDDEN]@example.com:4222"), String::npos); + EXPECT_EQ(masked.find("plain_user"), String::npos); + EXPECT_NE(masked.find("nats_url = '[HIDDEN]'"), String::npos); } TEST(ParserCreateQuery, MaskNATSTableEngineServerListPassword) diff --git a/src/Storages/NATS/NATS_fwd.h b/src/Storages/NATS/NATS_fwd.h index f9b1043f7b7d0..a74a9d3687c45 100644 --- a/src/Storages/NATS/NATS_fwd.h +++ b/src/Storages/NATS/NATS_fwd.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include namespace NATS @@ -26,7 +25,12 @@ static inline std::unordered_map SETTINGS_TO_HIDE = std::string masked_value; if (!value.tryGet(masked_value)) return {}; - DB::maskURIPassword(&masked_value); + /// libnats trims the value, takes the scheme as optional, and ends the userinfo at the LAST + /// '@' of the whole value (`contrib/nats-io/src/url.c`, `natsUrl_Create`), so its credential + /// can run past anything an RFC 3986 authority covers. An '@' is therefore the only reliable + /// sign that this address carries one, and there is no extent to keep visible. + if (masked_value.contains('@')) + masked_value = "[HIDDEN]"; return fmt::format("'{}'", masked_value); }} }; diff --git a/src/Storages/RabbitMQ/RabbitMQ_fwd.h b/src/Storages/RabbitMQ/RabbitMQ_fwd.h index af4a3c6fd2bb6..d0f5ccf96090c 100644 --- a/src/Storages/RabbitMQ/RabbitMQ_fwd.h +++ b/src/Storages/RabbitMQ/RabbitMQ_fwd.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include namespace RabbitMQ @@ -21,7 +20,12 @@ static inline std::unordered_map SETTINGS_TO_HIDE = std::string masked_value; if (!value.tryGet(masked_value)) return {}; - DB::maskURIPassword(&masked_value); + /// AMQP-CPP ends the login at the FIRST '@' after the scheme, unbounded by the `/?#` that + /// closes an RFC 3986 authority (`contrib/AMQP-CPP/include/amqpcpp/address.h`, `Address`), so + /// its credential can run past anything a URI masker covers. An '@' is therefore the only + /// reliable sign that this address carries one, and there is no extent to keep visible. + if (masked_value.contains('@')) + masked_value = "[HIDDEN]"; return fmt::format("'{}'", masked_value); }} }; diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 4ea3878add595..24b95f0764938 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -396,8 +396,8 @@ def generate_create_table_numbered(tail): generate_create_table_numbered(f"(`x` int) ENGINE = AzureQueue('{azure_storage_account_url}', 'cont', '*', '{azure_account_name}', '[HIDDEN]', 'CSV', 'none') SETTINGS mode = 'unordered'"), generate_create_table_numbered(f"(`x` int) ENGINE = AzureBlobStorage('{masked_sas_conn_string}', 'exampledatasets', 'example.csv')"), generate_create_table_numbered("(`x` int) ENGINE = S3('https://my-s3-endpoint/bucket/data.csv', 'myaccess', '[HIDDEN]', 'CSV')"), - generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://schema_user:[HIDDEN]@'"), - generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://schema_user:[HIDDEN]@domain.com'"), + generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://[HIDDEN]@'"), + generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://[HIDDEN]@domain.com'"), generate_create_table_numbered("(`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'CSV', access_key_id = 'minio', secret_access_key = '[HIDDEN]', compression_method = 'gzip')"), generate_create_table_numbered("(`x` int) ENGINE = ArrowFlight('arrowflight1:5006', 'dataset', 'arrowflight_user', '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = ArrowFlight(named_collection_1, host = 'arrowflight1', port = 5006, dataset = 'dataset', username = 'arrowflight_user', password = '[HIDDEN]')"), @@ -405,12 +405,12 @@ def generate_create_table_numbered(tail): generate_create_table_numbered("(`x` int) ENGINE = Redis('localhost', 0, '[HIDDEN]') PRIMARY KEY x"), generate_create_table_numbered("(`x` int) ENGINE = JDBC('[HIDDEN]', 'mydb', 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = ODBC('[HIDDEN]', 'mydb', 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = JDBC('jdbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = ODBC('odbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = JDBC('[HIDDEN]', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = ODBC('[HIDDEN]', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = 'jdbc://user:[HIDDEN]@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = 'odbc://user:[HIDDEN]@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', external_table = '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', datasource = '[HIDDEN]', external_table = '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = NATS SETTINGS nats_url = 'localhost:4222', nats_subjects = 'subject', nats_format = 'JSONEachRow', nats_token = '[HIDDEN]'"), @@ -656,12 +656,12 @@ def make_test_case(i): "CREATE TABLE tablefunc49 (`x` int) AS redis('localhost', 'key', 'key Int64', 0, '[HIDDEN]')", "CREATE TABLE tablefunc50 (`x` int) AS jdbc('[HIDDEN]', 'mydb', 'mytable')", "CREATE TABLE tablefunc51 (`x` int) AS odbc('[HIDDEN]', 'mydb', 'mytable')", - "CREATE TABLE tablefunc52 (`x` int) AS jdbc('jdbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", - "CREATE TABLE tablefunc53 (`x` int) AS odbc('odbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", + "CREATE TABLE tablefunc52 (`x` int) AS jdbc('[HIDDEN]', 'mydb', 'mytable')", + "CREATE TABLE tablefunc53 (`x` int) AS odbc('[HIDDEN]', 'mydb', 'mytable')", "CREATE TABLE tablefunc54 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]')", "CREATE TABLE tablefunc55 (`x` int) AS odbc(named_collection_1, connection_settings = '[HIDDEN]')", - "CREATE TABLE tablefunc56 (`x` int) AS jdbc(named_collection_1, datasource = 'jdbc://user:[HIDDEN]@localhost:5432/mydb')", - "CREATE TABLE tablefunc57 (`x` int) AS odbc(named_collection_1, connection_settings = 'odbc://user:[HIDDEN]@localhost:5432/mydb')", + "CREATE TABLE tablefunc56 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]')", + "CREATE TABLE tablefunc57 (`x` int) AS odbc(named_collection_1, connection_settings = '[HIDDEN]')", "CREATE TABLE tablefunc58 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]')", "CREATE TABLE tablefunc59 (`x` int) AS jdbc(named_collection_1, connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', datasource = '[HIDDEN]')", "CREATE TABLE tablefunc60 (`x` int) AS deltaLakeS3('http://minio1:9001/root/data/test11.csv.gz', 'minio', '[HIDDEN]')", diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 252fc892b9e48..9f02e5fbe90e1 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -3602,7 +3602,7 @@ def test_hiding_credentials(rabbitmq_cluster, db, unique): instance.query("SYSTEM FLUSH LOGS") message = instance.query(f"SELECT message FROM system.text_log WHERE message ILIKE '%CREATE TABLE {db}.{table_name}%'") assert "rabbitmq_password = \\'[HIDDEN]\\'" in message - assert "rabbitmq_address = \\'amqp://root:[HIDDEN]@rabbitmq1:5672/\\'" in message + assert "rabbitmq_address = \\'[HIDDEN]\\'" in message def test_rabbitmq_default_mode_nack_on_parse_error(rabbitmq_cluster, db, unique): diff --git a/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference b/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference index 209f37b46ed43..f726717f90dba 100644 --- a/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference +++ b/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference @@ -8,7 +8,7 @@ TYPE_MISMATCH 1 SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:pass@localhost\' TYPE_MISMATCH -SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:[HIDDEN]@localhost\' 1 +SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://[HIDDEN]@localhost\' 1 TYPE_MISMATCH TYPE_MISMATCH SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:pass@localhost\' diff --git a/tests/queries/0_stateless/05056_settings_secret_masking.reference b/tests/queries/0_stateless/05056_settings_secret_masking.reference index ed9415436a87d..e1fa25e506f9c 100644 --- a/tests/queries/0_stateless/05056_settings_secret_masking.reference +++ b/tests/queries/0_stateless/05056_settings_secret_masking.reference @@ -1,13 +1,13 @@ 1 1 1 1 1 1 1 -http://u:[HIDDEN]@reg:8080/ -https://u:[HIDDEN]@example.com/d/ +http://[HIDDEN]@reg:8080/ +https://[HIDDEN]@example.com/d/ 1 1 -CREATE SETTINGS PROFILE `profile` SETTINGS format_avro_schema_registry_url = \'http://u:[HIDDEN]@reg:8080/\' -http://u:[HIDDEN]@reg:8080/ +CREATE SETTINGS PROFILE `profile` SETTINGS format_avro_schema_registry_url = \'http://[HIDDEN]@reg:8080/\' +http://[HIDDEN]@reg:8080/ 1 1 1 1 1 1 https://h/f?X-Amz-Signature=c05056presignedsignature -http://u:[HIDDEN]@reg:8080/ http://u:[HIDDEN]@reg:8080/ +http://[HIDDEN]@reg:8080/ http://[HIDDEN]@reg:8080/ s3://bucket/prefix/ diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference new file mode 100644 index 0000000000000..3f7a95b33b5ae --- /dev/null +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference @@ -0,0 +1,28 @@ +a1_nats_slash_password SET nats_url = '[HIDDEN]' +a2_nats_no_scheme SET nats_url = '[HIDDEN]' +a3_nats_leading_space SET nats_url = '[HIDDEN]' +a4_nats_control SET nats_url = 'localhost:4222' +b1_amqp_slash_password SET rabbitmq_address = '[HIDDEN]' +b2_amqp_control SET rabbitmq_address = 'amqp://h:5672/v' +c1_nats_engine_argument CREATE TABLE t05233 (`x` UInt8) ENGINE = NATS(nc05233, nats_url = '[HIDDEN]') +c2_nats_engine_control CREATE TABLE t05233 (x UInt8) ENGINE = NATS(nc05233, nats_url = 'localhost:4222') +d1_jdbc_query_parameter CREATE TABLE db05233absent.t05233 (`x` UInt8) ENGINE = JDBC('[HIDDEN]', 'db', 't') +d2_jdbc_at_in_password CREATE TABLE db05233absent.t05233 (`x` UInt8) ENGINE = JDBC('[HIDDEN]', 'db', 't') +d3_odbc_key_value_form CREATE TABLE db05233absent.t05233 (`x` UInt8) ENGINE = ODBC('[HIDDEN]', 'db', 't') +d4_odbc_named_argument SELECT * FROM odbc(nc05233, connection_settings = '[HIDDEN]') +d5_jdbc_duplicate_key SELECT * FROM jdbc(nc05233, datasource = '[HIDDEN]', datasource = '[HIDDEN]') +d6_jdbc_computed_key SELECT * FROM jdbc(nc05233, concat('data', 'source') = '[HIDDEN]') +d7_jdbc_both_aliases SELECT * FROM jdbc(nc05233, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]') +d8_mysql_computed_key SELECT * FROM mysql(nc05233, concat('ssl_ca', '_pem') = '[HIDDEN]', `table` = 't') +e10_url_base_abfss_space SELECT 1 SETTINGS url_base = '[HIDDEN]' +e11_avro_in_engine CREATE TABLE t05233 (`x` UInt8) ENGINE = NATS(nc05233) SETTINGS format_avro_schema_registry_url = 'http://[HIDDEN]@reg:8080/' +e1_avro_at_in_password SELECT 1 SETTINGS format_avro_schema_registry_url = 'http://[HIDDEN]@reg:8080/' +e2_avro_userinfo_only SELECT 1 SETTINGS format_avro_schema_registry_url = 'http://[HIDDEN]@reg:8080/' +e3_url_base_no_userinfo SELECT 1 SETTINGS url_base = 'http://h:8080/d/?email=a@b.com' +e4_url_base_no_scheme SELECT 1 SETTINGS url_base = '[HIDDEN]' +e5_url_base_control SELECT 1 SETTINGS url_base = 'https://h/d/' +e6_s3_base_presigned SELECT 1 SETTINGS s3_base = 'https://b/f.csv?X-Amz-Signature=[HIDDEN]' +e7_url_base_abfss SELECT 1 SETTINGS url_base = 'abfss://container@account.dfs.core.windows.net/d/' +e8_avro_abfss_control SELECT 1 SETTINGS format_avro_schema_registry_url = 'abfss://[HIDDEN]@h/' +e9_url_base_az_control SELECT 1 SETTINGS url_base = 'az://[HIDDEN]@account.blob.core.windows.net/c/' +0 diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh new file mode 100755 index 0000000000000..6dd462de82f2f --- /dev/null +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# A credential embedded in a broker address, in an XDBC connection string or in a URL-valued setting +# must not reach `system.query_log`. Every statement below is rejected, and none of them needs a +# broker, a bridge or an existing named collection: masking runs when the statement is formatted for +# logging, before it is validated. Each positive case is paired with the control that has to stay +# fully visible, and each credential is a distinct `leak05233*` canary, so a leak points straight at +# the site that leaked it. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +PREFIX="05233_${CLICKHOUSE_DATABASE}_$(random_str 8)" +ARMS=0 + +arm() { # arm + ARMS=$((ARMS + 1)) + $CLICKHOUSE_CLIENT --query_id="${PREFIX}_$1" --log_queries=1 -q "$2" > /dev/null 2>&1 +} + +# `nats_url`: libnats ends the userinfo at the last '@' of the whole value, with the scheme optional +# and the value trimmed, so a password containing '/' and a scheme-less address both authenticate. +arm a1_nats_slash_password "SET nats_url = 'nats://u:pa/leak05233nats@h:4222'" +arm a2_nats_no_scheme "SET nats_url = 'u:leak05233noscheme@h:4222'" +arm a3_nats_leading_space "SET nats_url = ' nats://u:pa/leak05233space@h:4222'" +arm a4_nats_control "SET nats_url = 'localhost:4222'" + +# `rabbitmq_address`: AMQP-CPP ends the login at the first '@' after the scheme, also unbounded by the +# '/' that closes an RFC 3986 authority. +arm b1_amqp_slash_password "SET rabbitmq_address = 'amqp://u:pa/leak05233amqp@h:5672/v'" +arm b2_amqp_control "SET rabbitmq_address = 'amqp://h:5672/v'" + +# The engine-argument form has its own masking site and must agree with the `SETTINGS` clause above. +arm c1_nats_engine_argument "CREATE TABLE t05233 (x UInt8) ENGINE = NATS(nc05233, nats_url = 'nats://u:pa/leak05233arg@h:4222')" +arm c2_nats_engine_control "CREATE TABLE t05233 (x UInt8) ENGINE = NATS(nc05233, nats_url = 'localhost:4222')" + +# An XDBC connection string is forwarded to the driver verbatim, so its grammar is the driver's: the +# password can sit in a query parameter or in a `KEY=value;` list, and no URI scan bounds it. The +# positional form is written against a database this test never creates, because a `jdbc(...)` table +# function that reaches argument validation then spends 30 seconds trying to start the bridge. +arm d1_jdbc_query_parameter "CREATE TABLE db05233absent.t05233 (x UInt8) ENGINE = JDBC('mysql://u:p@h/?password=leak05233param', 'db', 't')" +arm d2_jdbc_at_in_password "CREATE TABLE db05233absent.t05233 (x UInt8) ENGINE = JDBC('jdbc://user:pa@leak05233at@h:5432/db', 'db', 't')" +arm d3_odbc_key_value_form "CREATE TABLE db05233absent.t05233 (x UInt8) ENGINE = ODBC('DSN=x;Uid=u;Pwd=leak05233kv', 'db', 't')" +arm d4_odbc_named_argument "SELECT * FROM odbc(nc05233, connection_settings = 'odbc://u:pa@leak05233odbc@h/db')" +arm d5_jdbc_duplicate_key "SELECT * FROM jdbc(nc05233, datasource = 'a://u:leak05233dup1@h/1', datasource = 'b://u:leak05233dup2@h/2')" +arm d6_jdbc_computed_key "SELECT * FROM jdbc(nc05233, concat('data', 'source') = 'jdbc://u:leak05233computed@h/db')" +arm d7_jdbc_both_aliases "SELECT * FROM jdbc(nc05233, datasource = 'a://u:leak05233both1@h/1', connection_settings = 'b://u:leak05233both2@h/2')" +# Control for the fail-closed unreadable-key scan the XDBC branch now shares with the TLS keys. +arm d8_mysql_computed_key "SELECT * FROM mysql(nc05233, concat('ssl_ca', '_pem') = 'leak05233mysql', table = 't')" + +# The query-level URL settings are read through `Poco::URI`, so host and path stay visible and only +# the userinfo is hidden. A value with no scheme in front of it is hidden whole instead. +arm e1_avro_at_in_password "SELECT 1 SETTINGS format_avro_schema_registry_url = 'http://user:pa@leak05233avro@reg:8080/'" +arm e2_avro_userinfo_only "SELECT 1 SETTINGS format_avro_schema_registry_url = 'http://leak05233token@reg:8080/'" +arm e3_url_base_no_userinfo "SELECT 1 SETTINGS url_base = 'http://h:8080/d/?email=a@b.com'" +arm e4_url_base_no_scheme "SELECT 1 SETTINGS url_base = ' http://u:leak05233space@h/d/'" +arm e5_url_base_control "SELECT 1 SETTINGS url_base = 'https://h/d/'" +arm e6_s3_base_presigned "SELECT 1 SETTINGS s3_base = 'https://b/f.csv?X-Amz-Signature=leak05233sig'" +# `abfs`/`abfss` put a non-secret container in front of the '@', and only for `url_base`, the one of +# the three settings that reaches the Azure URL parser. +arm e7_url_base_abfss "SELECT 1 SETTINGS url_base = 'abfss://container@account.dfs.core.windows.net/d/'" +arm e8_avro_abfss_control "SELECT 1 SETTINGS format_avro_schema_registry_url = 'abfss://u:leak05233avroabfss@h/'" +arm e9_url_base_az_control "SELECT 1 SETTINGS url_base = 'az://u:leak05233az@account.blob.core.windows.net/c/'" +arm e10_url_base_abfss_space "SELECT 1 SETTINGS url_base = ' abfss://c@a.dfs.core.windows.net/d/'" +# The same setting reached through an `ENGINE = ... SETTINGS` clause rather than through a query. +arm e11_avro_in_engine "CREATE TABLE t05233 (x UInt8) ENGINE = NATS(nc05233) SETTINGS format_avro_schema_registry_url = 'http://user:pa@leak05233engine@reg:8080/'" + +for _ in {1..120}; do + $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" + LOGGED=$($CLICKHOUSE_CLIENT -q "SELECT uniqExact(query_id) FROM system.query_log + WHERE event_date >= yesterday() AND current_database = currentDatabase() AND query_id LIKE '${PREFIX}\_%'") + [ "$LOGGED" -ge "$ARMS" ] && break + sleep 0.5 +done + +$CLICKHOUSE_CLIENT -q "SELECT replaceOne(query_id, '${PREFIX}_', '') AS arm, max(query) + FROM system.query_log + WHERE event_date >= yesterday() AND current_database = currentDatabase() AND query_id LIKE '${PREFIX}\_%' + GROUP BY arm ORDER BY arm FORMAT TSVRaw" + +# No canary in any column that prints the statement or its settings, in any arm. +$CLICKHOUSE_CLIENT -q "SELECT countIf(position(concat(query, formatted_query, toString(Settings)), 'leak05233') > 0) + FROM system.query_log + WHERE event_date >= yesterday() AND current_database = currentDatabase() AND query_id LIKE '${PREFIX}\_%'" From 66942793b54423bd444c3bd4d4d2f108c517d387 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 21 Sep 2026 09:19:18 +0000 Subject: [PATCH 141/296] Mask broker engine-argument overrides and drop the abfs container exception Three defects found reviewing the previous commit. The abfs/abfss exception in CoreSettings::SETTINGS_TO_HIDE masked strictly LESS than the code it replaced. maskURLBaseCredentials returned maskPresignedURLParameters(value) alone for those two schemes, so url_base = 'abfss://c:pw@a.dfs.core.windows.net/d/' logged pw verbatim where maskURIPassword had logged abfss://c:[HIDDEN]@a... . That is a masking regression in one of the settings this change set names, and it also contradicted the rule the rest of the work applies: the exception's own comment said the credential of such a base is the SAS in the query string, and then delegated to a helper that carries only the AWS and GCS parameter names and so cannot mask a SAS. A value whose credential ClickHouse cannot bound is not echoed with a partial mask. The exception is therefore deleted rather than widened, url_base goes back to maskURLCredentials, all three URL settings share one rule for every scheme, and Core no longer mirrors a copy of the Azure grammar that had to be kept in sync with parseAzureURL. This supersedes the paragraph of the previous commit message that described the container as staying visible. findTableEngineSecretArguments dispatched on the engine name and had branches for NATS and XDBC but none for RabbitMQ or Kafka, although exactly three engines read named-collection overrides out of their engine arguments (tryGetNamedCollectionWithOverrides(args.engine_args, ...) in StorageKafkaUtils.cpp, StorageNATS.cpp and StorageRabbitMQ.cpp) and only NATS had a finder. So ENGINE = RabbitMQ(nc, rabbitmq_address = 'amqp://u:secret@h/') ENGINE = RabbitMQ(nc, rabbitmq_password = 'secret') ENGINE = Kafka(nc, kafka_sasl_password = 'secret') reached SHOW CREATE, system.tables.create_table_query and system.query_log.query in cleartext, while the same settings written in a SETTINGS clause were hidden by the engine's own SETTINGS_TO_HIDE. Rather than copy the NATS loop, its body becomes findBrokerTableEngineSecretArguments(secret_keys, address_key), so the requirement that the finder and SETTINGS_TO_HIDE agree has one carrier; NATS and RabbitMQ are two thin wrappers over it. Kafka deliberately does not use it, because Kafka still accepts the legacy positional form Kafka('brokers', 'topics', 'group', 'format', ...) whose arguments are all non-secret, so failing closed on a positional argument there would hide information a user is entitled to see. NATS and RabbitMQ accept no positional argument other than the collection name, so for those two a positional is hidden. In the XDBC named-collection branch every helper walks equals arguments only, so a positional argument after the collection name was never masked and jdbc(nc, 'jdbc://u:pw@h/db') echoed the connection string. Such a statement is invalid and XDBC rejects it, but rejection happens after the statement has been formatted for logging. The branch now hides any argument after the collection name that is not a readable key = value pair, which is the treatment the NATS sibling already gave that shape. Tests: seven arms added to 05233_broker_and_xdbc_credential_masking, each positive paired with the control that has to stay fully visible, in particular an '@'-free rabbitmq_address and Kafka's legacy positional form. Three gtests added next to the NATS ones, each asserting the engine-argument spelling, the SETTINGS spelling and that control. Every existing NATS assertion, the re2 equivalence test and the TLS expectations pass unedited. --- src/Core/SettingsSecrets.h | 26 +--- src/Parsers/FunctionSecretArgumentsFinder.cpp | 56 +++++++-- src/Parsers/FunctionSecretArgumentsFinder.h | 17 +++ src/Parsers/tests/gtest_Parser.cpp | 119 ++++++++++++++++++ ...oker_and_xdbc_credential_masking.reference | 9 +- ...5233_broker_and_xdbc_credential_masking.sh | 16 ++- 6 files changed, 207 insertions(+), 36 deletions(-) diff --git a/src/Core/SettingsSecrets.h b/src/Core/SettingsSecrets.h index 77d883ee75ec9..11c49b1f82907 100644 --- a/src/Core/SettingsSecrets.h +++ b/src/Core/SettingsSecrets.h @@ -42,30 +42,6 @@ inline bool maskURLCredentials(String & value) return masked; } -/// Under `abfs`/`abfss` the part in front of the `@` is the container and not a credential (the -/// Hadoop grammar `abfss://@.dfs.core.windows.net/`), and the credential -/// is the SAS in the query string. `az`/`azure` have no `@` in their grammar and need no exception. -/// Keep the two names in sync with `parseAzureURL` in `src/Storages/StorageURL.cpp`, which compares -/// them after lowercasing; `Core` cannot depend on `Storages` to share the list. -inline bool maskURLBaseCredentials(String & value) -{ - static constexpr std::string_view azure_container_schemes[] = {"abfs", "abfss"}; - - if (size_t authority = findURIAuthority(value); authority != String::npos) - { - String scheme = value.substr(0, authority - 3); - for (auto & c : scheme) - if ('A' <= c && c <= 'Z') - c += 'a' - 'A'; - - for (auto azure_scheme : azure_container_schemes) - if (scheme == azure_scheme) - return maskPresignedURLParameters(value); - } - - return maskURLCredentials(value); -} - /// The settings of the query-level `Settings` collection whose value can carry a credential, and how /// each one is masked. `system.query_log.query` shows /// `format_avro_schema_registry_url = 'http://[HIDDEN]@registry:8080'`, so every other place that @@ -76,7 +52,7 @@ inline bool maskURLBaseCredentials(String & value) static inline std::unordered_map SETTINGS_TO_HIDE = { {"format_avro_schema_registry_url", maskURLCredentials}, - {"url_base", maskURLBaseCredentials}, + {"url_base", maskURLCredentials}, {"s3_base", maskURLCredentials}, }; diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 6e184d5131223..b21297dbcef5b 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -525,6 +525,17 @@ void FunctionSecretArgumentsFinder::findXDBCSecretArguments() findSecretNamedArgument("datasource", 1); findSecretNamedArgument("connection_settings", 1); markNamedArgumentsWithUnreadableKeys(1); + + /// After the collection name every argument must be a named override. A positional one is + /// invalid, and the statement is formatted for logging before validation rejects it, so it + /// can carry the connection string itself: hide it whole (fail closed). + for (size_t i = 1; i < function->arguments->size(); ++i) + { + const auto equals_func = function->arguments->at(i)->getFunction(); + if (!equals_func || equals_func->name() != "equals" || !equals_func->hasArguments() + || equals_func->arguments->size() != 2) + markSecretArgument(i, /* argument_is_named= */ false); + } } else { @@ -972,6 +983,21 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() /// NATS(named_collection, nats_password = 'password', nats_credentials = '...', ...) findNATSTableEngineSecretArguments(); } + else if (engine_name == "RabbitMQ") + { + /// RabbitMQ(named_collection, rabbitmq_address = '...', rabbitmq_password = '...') + findRabbitMQTableEngineSecretArguments(); + } + else if (engine_name == "Kafka") + { + /// Kafka(named_collection, kafka_sasl_password = '...') - `registerStorageKafka` reads named + /// overrides of the collection, so an override carries the secret the `SETTINGS` clause form + /// hides through `Kafka::SETTINGS_TO_HIDE`. The legacy positional form + /// (`Kafka('brokers', 'topics', 'group', 'format', ...)`) carries no secret and stays visible, + /// so this form does not fail closed on a positional argument. + findSecretNamedArgument("kafka_sasl_password", 1); + markNamedArgumentsWithUnreadableKeys(1); + } else if ((engine_name == "JDBC") || (engine_name == "ODBC")) { /// JDBC('DSN', database, table) @@ -981,17 +1007,21 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() } } -void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() +void FunctionSecretArgumentsFinder::findBrokerTableEngineSecretArguments( + std::span secret_keys, std::string_view address_key) { /// NATS(named_collection [, nats_password = 'password'] [, nats_token = 'token'] /// [, nats_credential_file = '/path'] [, nats_credentials = 'user JWT and seed'] /// [, nats_url = 'nats://user:password@host:4222'] /// [, nats_server_list = 'nats://user:password@host:4222,...'], ...) - /// The only positional argument the engine accepts is the name of a named collection, so the + /// RabbitMQ(named_collection [, rabbitmq_password = 'password'] + /// [, rabbitmq_address = 'amqp://user:password@host:5672/vhost'], ...) + /// The only positional argument these engines accept is the name of a named collection, so the /// credentials can only appear as named overrides. The `SETTINGS` clause form is masked - /// separately by `NATS::SETTINGS_TO_HIDE`, and this function masks the same keys the same way: - /// the secrets are hidden whole, and so is a `nats_url` that carries an '@'. - /// `nats_server_list` is hidden whole because each list entry can carry userinfo credentials. + /// separately by the engine's own `SETTINGS_TO_HIDE`, and this function masks the same keys the + /// same way: the secrets are hidden whole, and so is an address that carries an '@'. + /// A key list can hold a destination (`nats_server_list`), which is hidden whole because each + /// list entry can carry userinfo credentials. /// Fail closed on a key we cannot read as a plain literal: it can name a secret setting. for (size_t i = 0; i < function->arguments->size(); ++i) { @@ -1013,13 +1043,13 @@ void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() { markSecretArgument(i, /* argument_is_named= */ true); } - else if (key == "nats_url") + else if (key == address_key) { String url; if (equals_func->arguments->at(1)->tryGetString(&url, /* allow_identifier= */ false)) { /// An '@' is the only reliable sign of a credential here, and there is no extent to - /// keep visible; see the `nats_url` rule in `NATS_fwd.h` for why. + /// keep visible; see the address rule in the engine's `_fwd.h` for why. if (url.contains('@')) markSecretArgument(i, /* argument_is_named= */ true); } @@ -1030,13 +1060,23 @@ void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() markSecretArgument(i, /* argument_is_named= */ true); } } - else if (std::find(std::begin(nats_secret_keys), std::end(nats_secret_keys), key) != std::end(nats_secret_keys)) + else if (std::find(secret_keys.begin(), secret_keys.end(), key) != secret_keys.end()) { markSecretArgument(i, /* argument_is_named= */ true); } } } +void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() +{ + findBrokerTableEngineSecretArguments(nats_secret_keys, "nats_url"); +} + +void FunctionSecretArgumentsFinder::findRabbitMQTableEngineSecretArguments() +{ + findBrokerTableEngineSecretArguments(rabbitmq_secret_keys, "rabbitmq_address"); +} + void FunctionSecretArgumentsFinder::findExternalDistributedTableEngineSecretArguments() { if (isNamedCollectionName(1)) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 3c86e22c3b3bc..9aee325aafd37 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -126,6 +127,12 @@ class FunctionSecretArgumentsFinder static constexpr std::string_view nats_secret_keys[] = {"nats_password", "nats_token", "nats_credential_file", "nats_credentials", "nats_server_list"}; + /// Named arguments carrying RabbitMQ credentials, as the setting names, for the same reason as + /// `nats_secret_keys`. `rabbitmq_address` is not here because the documented form carries no + /// credential and stays visible; it is handled by the presence of an '@'. + /// Keep in sync with `RabbitMQ::SETTINGS_TO_HIDE`. + static constexpr std::string_view rabbitmq_secret_keys[] = {"rabbitmq_password"}; + void markSecretArgument(size_t index, bool argument_is_named = false); /// `headers(..)` and `extra_credentials(..)` are nested maps whose values are secret auth material @@ -221,7 +228,17 @@ class FunctionSecretArgumentsFinder void findRedisFunctionSecretArguments(); void findYTsaurusStorageTableEngineSecretArguments(); void findBigQuerySecretArguments(); + /// Masks the named-collection override form of a broker engine + /// (`NATS(collection, nats_token = '...')`, `RabbitMQ(collection, rabbitmq_address = '...')`). + /// `secret_keys` are hidden whole; `address_key` is hidden only when its value carries an '@', + /// because the documented address form carries no credential. Fails closed on a positional + /// argument and on a key this finder cannot read as a plain literal. + /// Keep the key lists in sync with the engine's own `SETTINGS_TO_HIDE`, which masks the same + /// settings in the `SETTINGS` clause. + void findBrokerTableEngineSecretArguments( + std::span secret_keys, std::string_view address_key); void findNATSTableEngineSecretArguments(); + void findRabbitMQTableEngineSecretArguments(); void findDatabaseEngineSecretArguments(); void findMySQLDatabaseSecretArguments(); void findS3DatabaseSecretArguments(); diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index 9e9a699dc4b37..73253f5d6c46d 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -380,6 +380,125 @@ TEST(ParserCreateQuery, MaskNATSTableEnginePositionalArguments) EXPECT_NE(masked.find("[HIDDEN]"), String::npos); } +TEST(ParserCreateQuery, MaskXDBCTableEnginePositionalAfterCollection) +{ + /// After a collection name every XDBC argument must be a named override, but the statement is + /// formatted for logging before validation rejects a positional one, and that positional can be + /// the connection string itself. The engine spelling takes more positional arguments than the + /// table function does, so it is asserted separately. + const String query = + "CREATE TABLE test_jdbc (key UInt64) " + "ENGINE = JDBC(jdbc1, 'DSN=mydb;Uid=user;Pwd=plain_password', 'mydb', 'mytable')"; + + DB::ParserCreateQuery parser; + DB::ASTPtr ast = DB::parseQuery(parser, query, 0, 0, 0); + + const String masked = ast->formatForLogging(); + + EXPECT_EQ(masked.find("plain_password"), String::npos); + EXPECT_EQ(masked.find("Uid=user"), String::npos); + /// The collection name is the one legitimate positional argument and stays visible. + EXPECT_NE(masked.find("jdbc1"), String::npos); + EXPECT_NE(masked.find("[HIDDEN]"), String::npos); + + /// A named override of the same connection string keeps its key visible, as before. + const String named_query = + "CREATE TABLE test_jdbc (key UInt64) " + "ENGINE = JDBC(jdbc1, datasource = 'DSN=mydb;Uid=user;Pwd=plain_named_password', " + "external_database = 'mydb', external_table = 'mytable')"; + + DB::ASTPtr named_ast = DB::parseQuery(parser, named_query, 0, 0, 0); + const String named_masked = named_ast->formatForLogging(); + + EXPECT_EQ(named_masked.find("plain_named_password"), String::npos); + EXPECT_NE(named_masked.find("datasource = '[HIDDEN]'"), String::npos); + /// The non-secret named arguments stay visible: the positional scan must not widen to them. + EXPECT_NE(named_masked.find("external_table = 'mytable'"), String::npos); +} + +TEST(ParserCreateQuery, MaskRabbitMQTableEngineCredentials) +{ + /// `RabbitMQ` also takes its settings as overrides of a named collection, so the same credentials + /// reach `SHOW CREATE TABLE` through the engine arguments and through the `SETTINGS` clause. + const String query = + "CREATE TABLE test_rabbitmq (key UInt64) ENGINE = RabbitMQ(rabbitmq1, " + "rabbitmq_password = 'plain_password', " + "rabbitmq_address = 'amqp://plain_user:plain_address_password@example.com:5672/vhost')"; + + DB::ParserCreateQuery parser; + DB::ASTPtr ast = DB::parseQuery(parser, query, 0, 0, 0); + + const String masked = ast->formatForLogging(); + + EXPECT_EQ(masked.find("plain_password"), String::npos); + EXPECT_EQ(masked.find("plain_address_password"), String::npos); + EXPECT_EQ(masked.find("plain_user"), String::npos); + /// The keys of the named overrides are not secrets and stay visible, as does the collection name. + EXPECT_NE(masked.find("rabbitmq1"), String::npos); + EXPECT_NE(masked.find("rabbitmq_password = '[HIDDEN]'"), String::npos); + EXPECT_NE(masked.find("rabbitmq_address = '[HIDDEN]'"), String::npos); + + /// An address with no '@' carries no credential and stays fully visible. + const String control_query = + "CREATE TABLE test_rabbitmq (key UInt64) " + "ENGINE = RabbitMQ(rabbitmq1, rabbitmq_address = 'amqp://example.com:5672/vhost')"; + + DB::ASTPtr control_ast = DB::parseQuery(parser, control_query, 0, 0, 0); + EXPECT_NE(control_ast->formatForLogging().find("amqp://example.com:5672/vhost"), String::npos); + + /// The `SETTINGS` clause form is masked by `RabbitMQ::SETTINGS_TO_HIDE` and must agree. + const String settings_query = + "CREATE TABLE test_rabbitmq_settings (key UInt64) ENGINE = RabbitMQ " + "SETTINGS rabbitmq_password = 'plain_settings_password'"; + + DB::ASTPtr settings_ast = DB::parseQuery(parser, settings_query, 0, 0, 0); + const String settings_masked = settings_ast->formatForLogging(); + + EXPECT_EQ(settings_masked.find("plain_settings_password"), String::npos); + EXPECT_NE(settings_masked.find("rabbitmq_password = '[HIDDEN]'"), String::npos); +} + +TEST(ParserCreateQuery, MaskKafkaTableEngineCredentials) +{ + /// `Kafka` reads named overrides of a collection too, so `kafka_sasl_password` needs masking in the + /// engine arguments and not only in the `SETTINGS` clause. + const String query = + "CREATE TABLE test_kafka (key UInt64) ENGINE = Kafka(kafka1, kafka_sasl_password = 'plain_password')"; + + DB::ParserCreateQuery parser; + DB::ASTPtr ast = DB::parseQuery(parser, query, 0, 0, 0); + + const String masked = ast->formatForLogging(); + + EXPECT_EQ(masked.find("plain_password"), String::npos); + EXPECT_NE(masked.find("kafka1"), String::npos); + EXPECT_NE(masked.find("kafka_sasl_password = '[HIDDEN]'"), String::npos); + + /// Unlike `NATS` and `RabbitMQ`, `Kafka` accepts a legacy positional form whose arguments are all + /// non-secret, so a positional argument must stay visible rather than fail closed. + const String positional_query = + "CREATE TABLE test_kafka (key UInt64) " + "ENGINE = Kafka('broker:9092', 'topic', 'group', 'JSONEachRow')"; + + DB::ASTPtr positional_ast = DB::parseQuery(parser, positional_query, 0, 0, 0); + const String positional_masked = positional_ast->formatForLogging(); + + EXPECT_NE(positional_masked.find("broker:9092"), String::npos); + EXPECT_NE(positional_masked.find("group"), String::npos); + EXPECT_EQ(positional_masked.find("[HIDDEN]"), String::npos); + + /// The `SETTINGS` clause form is masked by `Kafka::SETTINGS_TO_HIDE` and must agree. + const String settings_query = + "CREATE TABLE test_kafka_settings (key UInt64) ENGINE = Kafka " + "SETTINGS kafka_sasl_password = 'plain_settings_password'"; + + DB::ASTPtr settings_ast = DB::parseQuery(parser, settings_query, 0, 0, 0); + const String settings_masked = settings_ast->formatForLogging(); + + EXPECT_EQ(settings_masked.find("plain_settings_password"), String::npos); + EXPECT_NE(settings_masked.find("kafka_sasl_password = '[HIDDEN]'"), String::npos); +} + TEST_P(ParserTest, parseQuery) { const auto & parser = std::get<0>(GetParam()); diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference index 3f7a95b33b5ae..e4fbeaba3b3e3 100644 --- a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference @@ -14,15 +14,22 @@ d5_jdbc_duplicate_key SELECT * FROM jdbc(nc05233, datasource = '[HIDDEN]', datas d6_jdbc_computed_key SELECT * FROM jdbc(nc05233, concat('data', 'source') = '[HIDDEN]') d7_jdbc_both_aliases SELECT * FROM jdbc(nc05233, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]') d8_mysql_computed_key SELECT * FROM mysql(nc05233, concat('ssl_ca', '_pem') = '[HIDDEN]', `table` = 't') +d9_jdbc_positional_after_collection SELECT * FROM jdbc(nc05233, '[HIDDEN]') e10_url_base_abfss_space SELECT 1 SETTINGS url_base = '[HIDDEN]' e11_avro_in_engine CREATE TABLE t05233 (`x` UInt8) ENGINE = NATS(nc05233) SETTINGS format_avro_schema_registry_url = 'http://[HIDDEN]@reg:8080/' +e12_url_base_abfss_password SELECT 1 SETTINGS url_base = 'abfss://[HIDDEN]@a.dfs.core.windows.net/d/' e1_avro_at_in_password SELECT 1 SETTINGS format_avro_schema_registry_url = 'http://[HIDDEN]@reg:8080/' e2_avro_userinfo_only SELECT 1 SETTINGS format_avro_schema_registry_url = 'http://[HIDDEN]@reg:8080/' e3_url_base_no_userinfo SELECT 1 SETTINGS url_base = 'http://h:8080/d/?email=a@b.com' e4_url_base_no_scheme SELECT 1 SETTINGS url_base = '[HIDDEN]' e5_url_base_control SELECT 1 SETTINGS url_base = 'https://h/d/' e6_s3_base_presigned SELECT 1 SETTINGS s3_base = 'https://b/f.csv?X-Amz-Signature=[HIDDEN]' -e7_url_base_abfss SELECT 1 SETTINGS url_base = 'abfss://container@account.dfs.core.windows.net/d/' +e7_url_base_abfss SELECT 1 SETTINGS url_base = 'abfss://[HIDDEN]@account.dfs.core.windows.net/d/' e8_avro_abfss_control SELECT 1 SETTINGS format_avro_schema_registry_url = 'abfss://[HIDDEN]@h/' e9_url_base_az_control SELECT 1 SETTINGS url_base = 'az://[HIDDEN]@account.blob.core.windows.net/c/' +f1_rabbitmq_engine_argument CREATE TABLE t05233 (`x` UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = '[HIDDEN]') +f2_rabbitmq_engine_password CREATE TABLE t05233 (`x` UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_password = '[HIDDEN]') +f3_rabbitmq_engine_control CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = 'amqp://h:5672/v') +f4_kafka_engine_password CREATE TABLE t05233 (`x` UInt8) ENGINE = Kafka(nc05233, kafka_sasl_password = '[HIDDEN]') +f5_kafka_positional_control CREATE TABLE t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow') 0 diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh index 6dd462de82f2f..1a2e17f0e1abe 100755 --- a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh @@ -35,6 +35,15 @@ arm b2_amqp_control "SET rabbitmq_address = 'amqp://h:5672/v'" arm c1_nats_engine_argument "CREATE TABLE t05233 (x UInt8) ENGINE = NATS(nc05233, nats_url = 'nats://u:pa/leak05233arg@h:4222')" arm c2_nats_engine_control "CREATE TABLE t05233 (x UInt8) ENGINE = NATS(nc05233, nats_url = 'localhost:4222')" +# `Kafka`, `NATS` and `RabbitMQ` all read their settings as named overrides of the collection, so an +# override of a secret setting needs the same masking as the `SETTINGS` clause. Kafka's legacy +# positional form carries no secret, so its positional arguments stay visible. +arm f1_rabbitmq_engine_argument "CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = 'amqp://u:leak05233rmqarg@h:5672/v')" +arm f2_rabbitmq_engine_password "CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_password = 'leak05233rmqpw')" +arm f3_rabbitmq_engine_control "CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = 'amqp://h:5672/v')" +arm f4_kafka_engine_password "CREATE TABLE t05233 (x UInt8) ENGINE = Kafka(nc05233, kafka_sasl_password = 'leak05233kafkapw')" +arm f5_kafka_positional_control "CREATE TABLE t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow')" + # An XDBC connection string is forwarded to the driver verbatim, so its grammar is the driver's: the # password can sit in a query parameter or in a `KEY=value;` list, and no URI scan bounds it. The # positional form is written against a database this test never creates, because a `jdbc(...)` table @@ -48,6 +57,8 @@ arm d6_jdbc_computed_key "SELECT * FROM jdbc(nc05233, concat('data', 'source arm d7_jdbc_both_aliases "SELECT * FROM jdbc(nc05233, datasource = 'a://u:leak05233both1@h/1', connection_settings = 'b://u:leak05233both2@h/2')" # Control for the fail-closed unreadable-key scan the XDBC branch now shares with the TLS keys. arm d8_mysql_computed_key "SELECT * FROM mysql(nc05233, concat('ssl_ca', '_pem') = 'leak05233mysql', table = 't')" +# After the collection name every argument must be named, so a positional one is hidden whole. +arm d9_jdbc_positional_after_collection "SELECT * FROM jdbc(nc05233, 'jdbc://u:leak05233pos@h/db')" # The query-level URL settings are read through `Poco::URI`, so host and path stay visible and only # the userinfo is hidden. A value with no scheme in front of it is hidden whole instead. @@ -57,12 +68,13 @@ arm e3_url_base_no_userinfo "SELECT 1 SETTINGS url_base = 'http://h:8080/d/?ema arm e4_url_base_no_scheme "SELECT 1 SETTINGS url_base = ' http://u:leak05233space@h/d/'" arm e5_url_base_control "SELECT 1 SETTINGS url_base = 'https://h/d/'" arm e6_s3_base_presigned "SELECT 1 SETTINGS s3_base = 'https://b/f.csv?X-Amz-Signature=leak05233sig'" -# `abfs`/`abfss` put a non-secret container in front of the '@', and only for `url_base`, the one of -# the three settings that reaches the Azure URL parser. +# All three URL settings follow the one rule, for every scheme: the userinfo is hidden and the rest +# stays visible, and a value whose scheme is not at the start is hidden whole. arm e7_url_base_abfss "SELECT 1 SETTINGS url_base = 'abfss://container@account.dfs.core.windows.net/d/'" arm e8_avro_abfss_control "SELECT 1 SETTINGS format_avro_schema_registry_url = 'abfss://u:leak05233avroabfss@h/'" arm e9_url_base_az_control "SELECT 1 SETTINGS url_base = 'az://u:leak05233az@account.blob.core.windows.net/c/'" arm e10_url_base_abfss_space "SELECT 1 SETTINGS url_base = ' abfss://c@a.dfs.core.windows.net/d/'" +arm e12_url_base_abfss_password "SELECT 1 SETTINGS url_base = 'abfss://c:leak05233abfs@a.dfs.core.windows.net/d/'" # The same setting reached through an `ENGINE = ... SETTINGS` clause rather than through a query. arm e11_avro_in_engine "CREATE TABLE t05233 (x UInt8) ENGINE = NATS(nc05233) SETTINGS format_avro_schema_registry_url = 'http://user:pa@leak05233engine@reg:8080/'" From 5c93ecf8f19075d14ae18cb005c33539929bdd0e Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:43:00 +0000 Subject: [PATCH 142/296] Start the Kafka and XDBC named-argument scans at index 0 Two defects found reviewing the previous commit, both the same shape one argument index further out: a scan that starts at index 1 is only correct when index 0 is proven to be a named-collection name. The Kafka branch the previous commit added scanned from index 1, but Kafka is the one engine here whose collection name is optional: registerStorageKafka falls through to the legacy positional path when there is no identifier at index 0, and rejects a malformed call in checkAndGetLiteralArgument only after the statement has been formatted for logging. So ENGINE = Kafka(kafka_sasl_password = 'secret', 'clickhouse') was masked nowhere and reached system.query_log.query and the server log in cleartext. The shape is one users write; 03305_fix_kafka_table_with_kw_arguments exists for exactly ENGINE = Kafka(a = '1', 'clickhouse'), and kafka_sasl_password is the engine's only secret setting. Both scans now start at 0. That cannot widen the mask, because findNamedArgument and markNamedArgumentsWithUnreadableKeys skip any argument that is not an equals function with two arguments, which is what keeps the legacy positional form fully visible. findXDBCSecretArguments had the mirror image in its other branch. The else branch is taken whenever index 0 is not an identifier, and it hid only index 0, so in a keyword-argument call with no collection the connection string sits at another index and was echoed: jdbc(external_table = 't', datasource = 'DSN=x;Uid=u;Pwd=secret') jdbc and odbc have no validity precondition and ITableFunctionXDBC validates after formatting, so the call is logged. Whether the credential leaked depended only on the order the keys were typed in, while the named-argument style without a collection is what a user coming from s3(url = ..., access_key_id = ...) writes. The two aliases and the unreadable-key scan now run from index 1 in that branch too, on top of the unchanged markSecretArgument(0, false). For a valid positional call all three additions are no-ops, since no equals argument exists, so the schema and table arguments stay visible. The class is now closed rather than the two examples: every other named-argument scan in the file that starts at a non-zero index sits inside an isNamedCollectionName test, so index 0 is proven to be an identifier there, and these two branches were the only ones without that property. Two arms are added to 05233_broker_and_xdbc_credential_masking, each keeping its non-secret neighbour visible so it doubles as the over-masking control, plus one gtest for the XDBC engine spelling and one assertion block for the Kafka one. --- src/Parsers/FunctionSecretArgumentsFinder.cpp | 14 ++++++-- src/Parsers/tests/gtest_Parser.cpp | 33 +++++++++++++++++++ ...oker_and_xdbc_credential_masking.reference | 2 ++ ...5233_broker_and_xdbc_credential_masking.sh | 6 ++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index b21297dbcef5b..4038ac7ed5d0f 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -543,6 +543,13 @@ void FunctionSecretArgumentsFinder::findXDBCSecretArguments() /// odbc('DSN', schema, table) / odbc('DSN', table) /// JDBC('DSN', database, table) / ODBC('DSN', database, table) markSecretArgument(0, false); + + /// A named argument means this is not the positional form at all, and the connection string + /// can then sit at any index under either alias. Validation rejects such a call only after the + /// statement has been formatted for logging, so hide those values too (fail closed). + findSecretNamedArgument("datasource", 1); + findSecretNamedArgument("connection_settings", 1); + markNamedArgumentsWithUnreadableKeys(1); } } @@ -994,9 +1001,10 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() /// overrides of the collection, so an override carries the secret the `SETTINGS` clause form /// hides through `Kafka::SETTINGS_TO_HIDE`. The legacy positional form /// (`Kafka('brokers', 'topics', 'group', 'format', ...)`) carries no secret and stays visible, - /// so this form does not fail closed on a positional argument. - findSecretNamedArgument("kafka_sasl_password", 1); - markNamedArgumentsWithUnreadableKeys(1); + /// so this form does not fail closed on a positional argument. That form also makes the + /// collection name optional, so the scan starts at index 0: a named argument can be the first. + findSecretNamedArgument("kafka_sasl_password", 0); + markNamedArgumentsWithUnreadableKeys(0); } else if ((engine_name == "JDBC") || (engine_name == "ODBC")) { diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index 73253f5d6c46d..634465edcde62 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -416,6 +416,25 @@ TEST(ParserCreateQuery, MaskXDBCTableEnginePositionalAfterCollection) EXPECT_NE(named_masked.find("external_table = 'mytable'"), String::npos); } +TEST(ParserCreateQuery, MaskXDBCNamedArgumentsWithoutCollection) +{ + /// A named argument at index 0 is not a collection name, so this call is not the positional form: + /// the connection string can be under either alias at any index, and the statement is formatted + /// for logging before validation rejects it. + const String query = + "CREATE TABLE test_jdbc (key UInt64) ENGINE = JDBC(external_database = 'mydb', " + "datasource = 'DSN=mydb;Uid=user;Pwd=plain_password')"; + + DB::ParserCreateQuery parser; + DB::ASTPtr ast = DB::parseQuery(parser, query, 0, 0, 0); + + const String masked = ast->formatForLogging(); + + EXPECT_EQ(masked.find("plain_password"), String::npos); + EXPECT_EQ(masked.find("Uid=user"), String::npos); + EXPECT_NE(masked.find("datasource = '[HIDDEN]'"), String::npos); +} + TEST(ParserCreateQuery, MaskRabbitMQTableEngineCredentials) { /// `RabbitMQ` also takes its settings as overrides of a named collection, so the same credentials @@ -487,6 +506,20 @@ TEST(ParserCreateQuery, MaskKafkaTableEngineCredentials) EXPECT_NE(positional_masked.find("group"), String::npos); EXPECT_EQ(positional_masked.find("[HIDDEN]"), String::npos); + /// The legacy positional form makes the collection name optional, so a named argument can be the + /// first one, and the statement is formatted for logging before it is rejected. + const String first_arg_query = + "CREATE TABLE test_kafka (key UInt64) " + "ENGINE = Kafka(kafka_sasl_password = 'plain_first_password', 'clickhouse')"; + + DB::ASTPtr first_arg_ast = DB::parseQuery(parser, first_arg_query, 0, 0, 0); + const String first_arg_masked = first_arg_ast->formatForLogging(); + + EXPECT_EQ(first_arg_masked.find("plain_first_password"), String::npos); + EXPECT_NE(first_arg_masked.find("kafka_sasl_password = '[HIDDEN]'"), String::npos); + /// The positional argument beside it is not a secret and stays visible. + EXPECT_NE(first_arg_masked.find("'clickhouse'"), String::npos); + /// The `SETTINGS` clause form is masked by `Kafka::SETTINGS_TO_HIDE` and must agree. const String settings_query = "CREATE TABLE test_kafka_settings (key UInt64) ENGINE = Kafka " diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference index e4fbeaba3b3e3..104821b8bd643 100644 --- a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference @@ -6,6 +6,7 @@ b1_amqp_slash_password SET rabbitmq_address = '[HIDDEN]' b2_amqp_control SET rabbitmq_address = 'amqp://h:5672/v' c1_nats_engine_argument CREATE TABLE t05233 (`x` UInt8) ENGINE = NATS(nc05233, nats_url = '[HIDDEN]') c2_nats_engine_control CREATE TABLE t05233 (x UInt8) ENGINE = NATS(nc05233, nats_url = 'localhost:4222') +d10_jdbc_named_without_collection SELECT * FROM jdbc('[HIDDEN]', datasource = '[HIDDEN]') d1_jdbc_query_parameter CREATE TABLE db05233absent.t05233 (`x` UInt8) ENGINE = JDBC('[HIDDEN]', 'db', 't') d2_jdbc_at_in_password CREATE TABLE db05233absent.t05233 (`x` UInt8) ENGINE = JDBC('[HIDDEN]', 'db', 't') d3_odbc_key_value_form CREATE TABLE db05233absent.t05233 (`x` UInt8) ENGINE = ODBC('[HIDDEN]', 'db', 't') @@ -32,4 +33,5 @@ f2_rabbitmq_engine_password CREATE TABLE t05233 (`x` UInt8) ENGINE = RabbitMQ(nc f3_rabbitmq_engine_control CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = 'amqp://h:5672/v') f4_kafka_engine_password CREATE TABLE t05233 (`x` UInt8) ENGINE = Kafka(nc05233, kafka_sasl_password = '[HIDDEN]') f5_kafka_positional_control CREATE TABLE t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow') +f6_kafka_first_argument_secret CREATE TABLE t05233 (`x` UInt8) ENGINE = Kafka(kafka_sasl_password = '[HIDDEN]', 'clickhouse') 0 diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh index 1a2e17f0e1abe..556dfbbd3a481 100755 --- a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh @@ -43,6 +43,9 @@ arm f2_rabbitmq_engine_password "CREATE TABLE t05233 (x UInt8) ENGINE = RabbitM arm f3_rabbitmq_engine_control "CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = 'amqp://h:5672/v')" arm f4_kafka_engine_password "CREATE TABLE t05233 (x UInt8) ENGINE = Kafka(nc05233, kafka_sasl_password = 'leak05233kafkapw')" arm f5_kafka_positional_control "CREATE TABLE t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow')" +# That positional form also makes the collection name optional, so a named override can be the first +# argument. The positional beside it stays visible, so this arm is its own over-masking control. +arm f6_kafka_first_argument_secret "CREATE TABLE t05233 (x UInt8) ENGINE = Kafka(kafka_sasl_password = 'leak05233kafkafirst', 'clickhouse')" # An XDBC connection string is forwarded to the driver verbatim, so its grammar is the driver's: the # password can sit in a query parameter or in a `KEY=value;` list, and no URI scan bounds it. The @@ -59,6 +62,9 @@ arm d7_jdbc_both_aliases "SELECT * FROM jdbc(nc05233, datasource = 'a://u:le arm d8_mysql_computed_key "SELECT * FROM mysql(nc05233, concat('ssl_ca', '_pem') = 'leak05233mysql', table = 't')" # After the collection name every argument must be named, so a positional one is hidden whole. arm d9_jdbc_positional_after_collection "SELECT * FROM jdbc(nc05233, 'jdbc://u:leak05233pos@h/db')" +# A named argument at index 0 is not a collection name, so the connection string can then sit at any +# index under either alias. +arm d10_jdbc_named_without_collection "SELECT * FROM jdbc(external_table = 't05233', datasource = 'DSN=x;Uid=u;Pwd=leak05233nocoll')" # The query-level URL settings are read through `Poco::URI`, so host and path stay visible and only # the userinfo is hidden. A value with no scheme in front of it is hidden whole instead. From ba517e21e331c40625ea8d8d790bdfeb424b5a07 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:40:41 +0000 Subject: [PATCH 143/296] Start the positional MySQL TLS scans at index 0 findMySQLFunctionSecretArguments and findMySQLDatabaseSecretArguments scanned for TLS credentials from index 5 and index 4 in their positional branches, one past the last positional slot. The keys those scans hide are PEM material (ssl_ca_pem, ssl_cert_pem, ssl_key_pem and the three PostgreSQL spellings), the contents of a certificate or a private key, so a credential written before the positional arguments was masked nowhere: mysql(ssl_key_pem = '', '127.0.0.1:3306', 'db', 't', 'u') CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '', '127.0.0.1:3306', 'db', 'u') Such a call is invalid, but ASTFunction::formatImpl runs the finder whenever show_secrets is false, which is before any argument is validated, so the statement reaches system.query_log.query and the server log in cleartext first. The first function serves the mysql and postgresql table functions and the MySQL, PostgreSQL and MaterializedPostgreSQL table engines; the second serves the MySQL, PostgreSQL, MaterializedPostgreSQL, Remote and RemoteSecure database engines. Both scans now start at 0. That cannot widen the mask, because findNamedArgument and markNamedArgumentsWithUnreadableKeys skip any argument that is not an equals function with exactly two arguments, so every positional literal stays visible. The pre-existing arms of 04648_mysql_tls_credentials keep asserting that byte for byte. Two corrections to the previous commit's message. It claimed the class was closed, on the grounds that every other named-argument scan in the file starting at a non-zero index sits inside an isNamedCollectionName test and that the two branches it fixed were the only ones without that property. That was wrong. These two positional MySQL branches have the same shape and sit in the negative branch of the same test, where index 0 is proven not to be a collection name, and they are closed here. The statement that holds now is that no named-argument scan in the file starts past index 0 unless index 0 is proven to be a collection name. It also said the two arms it added each keep their non-secret neighbour visible, "so it doubles as the over-masking control". That holds for the Kafka arm, whose positional neighbour 'clickhouse' stays visible. It does not hold for the jdbc one: that neighbour sits at index 0 and is hidden whole by the untouched markSecretArgument(0, false). The over-masking control for that branch is d1 to d3, which keep their schema and table arguments visible. Two arms are added to 04648_mysql_tls_credentials, the test that already owns these keys, one per code path, through the formatter that both share. --- src/Parsers/FunctionSecretArgumentsFinder.cpp | 4 ++-- .../0_stateless/04648_mysql_tls_credentials.reference | 4 ++++ tests/queries/0_stateless/04648_mysql_tls_credentials.sh | 7 +++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 4038ac7ed5d0f..698086605c734 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -392,7 +392,7 @@ void FunctionSecretArgumentsFinder::findMySQLFunctionSecretArguments() { /// mysql('host:port', 'database', 'table', 'user', 'password', ...) markSecretArgument(4); - findTLSCredentialsSecretArguments(5); + findTLSCredentialsSecretArguments(0); } } @@ -1321,7 +1321,7 @@ void FunctionSecretArgumentsFinder::findMySQLDatabaseSecretArguments() { /// MySQL('host:port', 'database', 'user', 'password') markSecretArgument(3); - findTLSCredentialsSecretArguments(4); + findTLSCredentialsSecretArguments(0); } } diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference index cd9aa6da303ce..f82c14fb17fa1 100644 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference @@ -12,6 +12,10 @@ SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', 'u', '[HIDDEN]', ssl_ca_pem = ' CREATE TABLE t (`x` Int32) ENGINE = MySQL('127.0.0.1:3306', 'db', 't', 'u', '[HIDDEN]', ssl_cert_pem = '[HIDDEN]') --- MySQL database engine, positional arguments CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '[HIDDEN]', ssl_ca_pem = '[HIDDEN]') +--- mysql table function, credentials before the positional arguments +SELECT * FROM mysql(ssl_key_pem = '[HIDDEN]', '127.0.0.1:3306', 'db', 't', '[HIDDEN]') +--- MySQL database engine, credentials before the positional arguments +CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '[HIDDEN]', '127.0.0.1:3306', 'db', '[HIDDEN]') --- key given as a constant expression SELECT * FROM mysql(creds, concat('ssl_ca', '_pem') = '[HIDDEN]', `table` = 't') --- key given as a constant expression, positional arguments diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh index 0fcb5994fd45e..a6f214d3a93aa 100755 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh @@ -49,6 +49,13 @@ format "MySQL table engine, positional arguments" \ format "MySQL database engine, positional arguments" \ "CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '${SECRET}', ssl_ca_pem = '${SECRET}')" +# The same credentials written before the positional arguments: the call is invalid, but it is +# formatted for logging before it is rejected, so the scan must not start past the first argument. +format "mysql table function, credentials before the positional arguments" \ + "SELECT * FROM mysql(ssl_key_pem = '${SECRET}', '127.0.0.1:3306', 'db', 't', 'u')" +format "MySQL database engine, credentials before the positional arguments" \ + "CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '${SECRET}', '127.0.0.1:3306', 'db', 'u')" + # The key of a named argument is not required to be a plain identifier or literal: the named # collection parser evaluates it as a constant expression, so `concat('ssl_ca', '_pem')` names a TLS # credential too. The formatter cannot evaluate it, so it hides the value of every argument whose key From ab89730870ac17c9483afe2fe2fff43f84fc323b Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:36:16 +0000 Subject: [PATCH 144/296] Locate the positional MySQL password among the positional arguments The positional branches of findMySQLFunctionSecretArguments and findMySQLDatabaseSecretArguments got the password wrong in one way with two faces: a positional slot is not a raw argument index, and a positional branch owes the same named-argument scan its named-collection sibling runs. A `key = value` argument written before the positional block shifts every positional one index along, so markSecretArgument(4) and markSecretArgument(3) hid the wrong argument and echoed the password; and because the branch scanned named arguments only for the TLS keys, a `password =` override written there was masked nowhere at all: mysql(ssl_key_pem = '', '127.0.0.1:3306', 'db', 't', 'u', '') mysql('127.0.0.1:3306', 'db', 't', password = '', 'u') CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '', '127.0.0.1:3306', 'db', 'u', '') CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', password = '', 'u') Such calls are invalid, but ASTFunction::formatImpl runs the finder whenever show_secrets is false, which is before any argument is validated, so the statement reaches system.query_log.query and the server log in cleartext first. Both faces are fixed together. classifyPositionalArguments returns the raw indexes of the arguments that are not `key = value` pairs, so the branch reads its slot out of that list instead of counting raw arguments, and a positional argument written after the first named one is hidden rather than listed: the parsers reject that mix, the statement is formatted before they do, and the slot it was meant to fill is then unknowable. classifyS3Arguments already applies that rule to the s3 signatures, and the helper reuses its shape rather than its body, which carries S3-specific key handling. The branch then runs findSecretNamedArgument("password", 0), the call its named-collection sibling already makes. A well-formed call is unchanged byte for byte: its named arguments come after the positional block, so the returned list is [0, 1, 2, ...] and the slot lookup yields the same index as before. Twenty of the twenty-two formatting assertions of 04648_mysql_tls_credentials are untouched; the two that move are the arms that previously recorded a credential as visible, and they now carry the password so the file's own leak oracle can see it. The invariant this closes is narrow, and only the part I enumerated is claimed. A named-argument scan in this file starts past index 0 in twenty-four places. It is safe where index 0 is proven to be a collection name, and also where index 0 is hidden whole: findXDBCSecretArguments's else branch scans from 1 with isNamedCollectionName(0) false, and is safe because markSecretArgument(0, false) above it hides that argument entirely. No claim is made for the file as a whole. A scan that starts past index 0 behind isNamedCollectionName(N) for N greater than zero still skips a secret named argument written first, which ExternalDistributed, azureBlobStorage and remote were measured to echo. Those are untouched here. --- src/Parsers/FunctionSecretArgumentsFinder.cpp | 35 +++++++++++++++++-- src/Parsers/FunctionSecretArgumentsFinder.h | 7 ++++ .../04648_mysql_tls_credentials.reference | 4 +-- .../04648_mysql_tls_credentials.sh | 8 +++-- ...oker_and_xdbc_credential_masking.reference | 2 +- ...5233_broker_and_xdbc_credential_masking.sh | 2 +- 6 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 698086605c734..b0642fd2cae5d 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -391,11 +391,39 @@ void FunctionSecretArgumentsFinder::findMySQLFunctionSecretArguments() else { /// mysql('host:port', 'database', 'table', 'user', 'password', ...) - markSecretArgument(4); + const auto positional = classifyPositionalArguments(); + if (positional.size() > 4) + markSecretArgument(positional[4]); + findSecretNamedArgument("password", 0); findTLSCredentialsSecretArguments(0); } } +std::vector FunctionSecretArgumentsFinder::classifyPositionalArguments(size_t start) +{ + std::vector positional; + bool seen_named = false; + for (size_t i = start; i < function->arguments->size(); ++i) + { + const auto equals_func = function->arguments->at(i)->getFunction(); + if (equals_func && equals_func->name() == "equals" && equals_func->hasArguments() + && equals_func->arguments->size() == 2) + { + seen_named = true; + continue; + } + + if (seen_named) + { + markSecretArgument(i); + continue; + } + + positional.push_back(i); + } + return positional; +} + void FunctionSecretArgumentsFinder::markNamedArgumentsWithUnreadableKeys(size_t start) { /// The named-collection parser does not require the key of a `key = value` argument to be a plain @@ -1320,7 +1348,10 @@ void FunctionSecretArgumentsFinder::findMySQLDatabaseSecretArguments() else { /// MySQL('host:port', 'database', 'user', 'password') - markSecretArgument(3); + const auto positional = classifyPositionalArguments(); + if (positional.size() > 3) + markSecretArgument(positional[3]); + findSecretNamedArgument("password", 0); findTLSCredentialsSecretArguments(0); } } diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 9aee325aafd37..2f72472f5f175 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -267,6 +267,13 @@ class FunctionSecretArgumentsFinder /// expression, so it can name a secret argument that `findNamedArgument` never sees. void markNamedArgumentsWithUnreadableKeys(size_t start); + /// The raw indexes of the arguments from `start` on that are not `key = value` pairs, in order, so a + /// branch that knows a secret's positional slot can find the argument that actually holds it. A + /// positional argument after the first named one is hidden instead of listed: the parsers reject that + /// mix, the statement is formatted for logging before they do, and the slot it was meant to fill is + /// then unknowable. `classifyS3Arguments` applies the same rule to the `s3` signatures. + std::vector classifyPositionalArguments(size_t start = 0); + /// Masks the secrets of an S3 named-collection form: the secret named overrides (every occurrence, /// in any order; the span covering them may hide a non-secret argument in between, which is safe) /// and the `headers(...)` / `extra_credentials(...)` map overrides. diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference index f82c14fb17fa1..237a8929e9563 100644 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference @@ -13,9 +13,9 @@ CREATE TABLE t (`x` Int32) ENGINE = MySQL('127.0.0.1:3306', 'db', 't', 'u', '[HI --- MySQL database engine, positional arguments CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '[HIDDEN]', ssl_ca_pem = '[HIDDEN]') --- mysql table function, credentials before the positional arguments -SELECT * FROM mysql(ssl_key_pem = '[HIDDEN]', '127.0.0.1:3306', 'db', 't', '[HIDDEN]') +SELECT * FROM mysql(ssl_key_pem = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') --- MySQL database engine, credentials before the positional arguments -CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '[HIDDEN]', '127.0.0.1:3306', 'db', '[HIDDEN]') +CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') --- key given as a constant expression SELECT * FROM mysql(creds, concat('ssl_ca', '_pem') = '[HIDDEN]', `table` = 't') --- key given as a constant expression, positional arguments diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh index a6f214d3a93aa..4cb5e6ac04cf4 100755 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh @@ -50,11 +50,13 @@ format "MySQL database engine, positional arguments" \ "CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '${SECRET}', ssl_ca_pem = '${SECRET}')" # The same credentials written before the positional arguments: the call is invalid, but it is -# formatted for logging before it is rejected, so the scan must not start past the first argument. +# formatted for logging before it is rejected, so the scan must not start past the first argument, +# and the positional password must be located among the positional arguments rather than at a fixed +# argument index, which a named argument written first moves. format "mysql table function, credentials before the positional arguments" \ - "SELECT * FROM mysql(ssl_key_pem = '${SECRET}', '127.0.0.1:3306', 'db', 't', 'u')" + "SELECT * FROM mysql(ssl_key_pem = '${SECRET}', '127.0.0.1:3306', 'db', 't', 'u', '${SECRET}')" format "MySQL database engine, credentials before the positional arguments" \ - "CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '${SECRET}', '127.0.0.1:3306', 'db', 'u')" + "CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '${SECRET}', '127.0.0.1:3306', 'db', 'u', '${SECRET}')" # The key of a named argument is not required to be a plain identifier or literal: the named # collection parser evaluates it as a constant expression, so `concat('ssl_ca', '_pem')` names a TLS diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference index 104821b8bd643..126d8989eee7a 100644 --- a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference @@ -32,6 +32,6 @@ f1_rabbitmq_engine_argument CREATE TABLE t05233 (`x` UInt8) ENGINE = RabbitMQ(nc f2_rabbitmq_engine_password CREATE TABLE t05233 (`x` UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_password = '[HIDDEN]') f3_rabbitmq_engine_control CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = 'amqp://h:5672/v') f4_kafka_engine_password CREATE TABLE t05233 (`x` UInt8) ENGINE = Kafka(nc05233, kafka_sasl_password = '[HIDDEN]') -f5_kafka_positional_control CREATE TABLE t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow') +f5_kafka_positional_control CREATE TABLE db05233absent.t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow') f6_kafka_first_argument_secret CREATE TABLE t05233 (`x` UInt8) ENGINE = Kafka(kafka_sasl_password = '[HIDDEN]', 'clickhouse') 0 diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh index 556dfbbd3a481..85ff60ae41da7 100755 --- a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.sh @@ -42,7 +42,7 @@ arm f1_rabbitmq_engine_argument "CREATE TABLE t05233 (x UInt8) ENGINE = RabbitM arm f2_rabbitmq_engine_password "CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_password = 'leak05233rmqpw')" arm f3_rabbitmq_engine_control "CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = 'amqp://h:5672/v')" arm f4_kafka_engine_password "CREATE TABLE t05233 (x UInt8) ENGINE = Kafka(nc05233, kafka_sasl_password = 'leak05233kafkapw')" -arm f5_kafka_positional_control "CREATE TABLE t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow')" +arm f5_kafka_positional_control "CREATE TABLE db05233absent.t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow')" # That positional form also makes the collection name optional, so a named override can be the first # argument. The positional beside it stays visible, so this arm is its own over-masking control. arm f6_kafka_first_argument_secret "CREATE TABLE t05233 (x UInt8) ENGINE = Kafka(kafka_sasl_password = 'leak05233kafkafirst', 'clickhouse')" From 4c6ceaa199766baa777776c9c893b303738fb7c2 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:29:20 +0000 Subject: [PATCH 145/296] Do not echo the rejected url_base or s3_base value StorageURL::resolveURLBase rejects a base with no "://" by interpolating the raw value into its BAD_ARGUMENTS message. The value is a setting that can carry a credential, so the credential is disclosed in full through the client error, the exception column of the query log and the server log: SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'user:@example.invalid/def/' Code: 36. The `url_base` setting must contain a scheme (e.g. https://), got: user:@example.invalid/def/ Nothing gates that. url_base is settable in a settings profile, so the value belongs to whoever wrote the profile while the error goes to whichever user trips it, and the message is not behind format_display_secrets_in_show_and_select. The same message carries s3_base and format_avro_schema_registry_url: the callers that reach it with a raw user value are TableFunctionURL.cpp for url_base and ObjectStorage/S3/Configuration.cpp in two places for s3_base. The value is dropped rather than masked. A masker only finds a credential it can locate: maskURLCredentials fires on a value containing '@', so s3_base = 'DSN=x;Pwd=' would still be echoed whole. The setting name stays in the message, which is what a user needs in order to find their own typo. The sibling carrier already has this treatment. DatabaseURL refuses a scheme-less base without echoing it, and 04627_url_database_engine asserts that with a leak count. The url_base and s3_base arms of the same invariant grep only the message prefix, which is why nothing in tree caught this. They now carry the same pair of assertions, so all three carriers are pinned the same way. The text before the removed ", got:" is byte identical, so those pre-existing prefix arms and their reference lines do not move. The new arms run clickhouse-local rather than clickhouse-client: the client appends the query it was given to an exception, and that echo is the caller's own input rather than anything the message disclosed, so a leak count taken through the client can never reach zero. 04648_mysql_tls_credentials gains two arms as well. The two findSecretNamedArgument("password", 0) calls added to the positional MySQL branches in the previous commit were exercised by no test: every existing named password arm goes through the collection branch, and the arms that carry a named credential beside a positional block use a TLS key. The new arms are the collection-less named password shape for each branch, and deleting either call reddens exactly one of them. --- src/Storages/StorageURL.cpp | 3 ++- tests/queries/0_stateless/04070_url_base_setting.reference | 2 ++ tests/queries/0_stateless/04070_url_base_setting.sh | 7 +++++++ tests/queries/0_stateless/04626_s3_base_setting.reference | 2 ++ tests/queries/0_stateless/04626_s3_base_setting.sh | 7 +++++++ .../0_stateless/04648_mysql_tls_credentials.reference | 4 ++++ tests/queries/0_stateless/04648_mysql_tls_credentials.sh | 7 +++++++ 7 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 7f3e7f6dca7a7..5ebb3cec411e5 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1882,8 +1882,9 @@ String StorageURL::resolveURLBase(const String & url, const String & base, const } auto scheme_end = base.find("://"); + /// Not echoed back: the value can carry a credential, and password masking anchors on the `://` it lacks. if (scheme_end == String::npos) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The `{}` setting must contain a scheme (e.g. https://), got: {}", base_setting_name, base); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The `{}` setting must contain a scheme (e.g. https://)", base_setting_name); /// Find the boundary of the path component in the base URL (before '?' or '#'). auto authority_start = scheme_end + 3; /// skip "://" diff --git a/tests/queries/0_stateless/04070_url_base_setting.reference b/tests/queries/0_stateless/04070_url_base_setting.reference index 8c23601338af0..958e5fdfc6c78 100644 --- a/tests/queries/0_stateless/04070_url_base_setting.reference +++ b/tests/queries/0_stateless/04070_url_base_setting.reference @@ -28,3 +28,5 @@ url = 'http://base.invalid/dir/persist.csv' 0 0 must contain a scheme +must contain a scheme +0 diff --git a/tests/queries/0_stateless/04070_url_base_setting.sh b/tests/queries/0_stateless/04070_url_base_setting.sh index 46289ca8b2c14..9127cb87d0660 100755 --- a/tests/queries/0_stateless/04070_url_base_setting.sh +++ b/tests/queries/0_stateless/04070_url_base_setting.sh @@ -172,3 +172,10 @@ DROP TABLE IF EXISTS ${CLICKHOUSE_TEST_UNIQUE_NAME}_url_pos_creds; # Invalid url_base (no scheme) should produce an error $CLICKHOUSE_CLIENT --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'example.invalid/def/', $FAST" 2>&1 | grep -oF 'must contain a scheme' | head -1 + +# The rejected value is not echoed back: it can carry a credential, and the message reaches the client, +# the exception column of the query log and the server log, none of which the display-secrets setting +# gates. (last line counts the leaks; clickhouse-local is used because the client also prints back the +# query it was given, which is the caller's own input rather than something the message disclosed) +$CLICKHOUSE_LOCAL --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'user:SEKRIT_PW@example.invalid/def/', $FAST" 2>&1 | grep -oF 'must contain a scheme' | head -1 +$CLICKHOUSE_LOCAL --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'user:SEKRIT_PW@example.invalid/def/', $FAST" 2>&1 | grep -c SEKRIT_PW ||: diff --git a/tests/queries/0_stateless/04626_s3_base_setting.reference b/tests/queries/0_stateless/04626_s3_base_setting.reference index 6b40f3b84623a..97a52a1f9a5cb 100644 --- a/tests/queries/0_stateless/04626_s3_base_setting.reference +++ b/tests/queries/0_stateless/04626_s3_base_setting.reference @@ -20,3 +20,5 @@ 5 --- s3_base without a scheme is an error must contain a scheme +must contain a scheme +0 diff --git a/tests/queries/0_stateless/04626_s3_base_setting.sh b/tests/queries/0_stateless/04626_s3_base_setting.sh index e8793ce77cab7..88ef59858adda 100755 --- a/tests/queries/0_stateless/04626_s3_base_setting.sh +++ b/tests/queries/0_stateless/04626_s3_base_setting.sh @@ -61,3 +61,10 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE test_s3_base_nc" && ${CLICKHOUSE_CLIENT} -q echo '--- s3_base without a scheme is an error' ${CLICKHOUSE_CLIENT} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'localhost:11111/test/'" 2>&1 | grep -oF 'must contain a scheme' | head -1 + +# The rejected value is not echoed back: it can carry a credential, and the message reaches the client, +# the exception column of the query log and the server log, none of which the display-secrets setting +# gates. (last line counts the leaks; clickhouse-local is used because the client also prints back the +# query it was given, which is the caller's own input rather than something the message disclosed) +${CLICKHOUSE_LOCAL} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'user:SEKRIT_PW@localhost:11111/test/'" 2>&1 | grep -oF 'must contain a scheme' | head -1 +${CLICKHOUSE_LOCAL} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'user:SEKRIT_PW@localhost:11111/test/'" 2>&1 | grep -c SEKRIT_PW ||: diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference index 237a8929e9563..daa2aafc0de8f 100644 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference @@ -16,6 +16,10 @@ CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '[HIDDEN]', ssl_ca SELECT * FROM mysql(ssl_key_pem = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') --- MySQL database engine, credentials before the positional arguments CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +--- mysql table function, named password without a collection +SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', password = '[HIDDEN]') +--- MySQL database engine, named password without a collection +CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', password = '[HIDDEN]') --- key given as a constant expression SELECT * FROM mysql(creds, concat('ssl_ca', '_pem') = '[HIDDEN]', `table` = 't') --- key given as a constant expression, positional arguments diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh index 4cb5e6ac04cf4..0bd7978918bb6 100755 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh @@ -58,6 +58,13 @@ format "mysql table function, credentials before the positional arguments" \ format "MySQL database engine, credentials before the positional arguments" \ "CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '${SECRET}', '127.0.0.1:3306', 'db', 'u', '${SECRET}')" +# A named `password` override with no named collection: the parsers reject the mix, but the statement is +# formatted for logging first, and the key is readable, so only the named scan can hide this value. +format "mysql table function, named password without a collection" \ + "SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', password = '${SECRET}')" +format "MySQL database engine, named password without a collection" \ + "CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', password = '${SECRET}')" + # The key of a named argument is not required to be a plain identifier or literal: the named # collection parser evaluates it as a constant expression, so `concat('ssl_ca', '_pem')` names a TLS # credential too. The formatter cannot evaluate it, so it hides the value of every argument whose key From 6d6f859750146f752f56771ec09cf365f85f9b94 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:08:07 +0000 Subject: [PATCH 146/296] Trim comments Delete comments narrating this change's own design, keep the third-party masker semantics (libnats, AMQP-CPP, XDBC driver grammar) and the cross-file sync invariants as standalone properties. --- src/Common/maskURIPassword.h | 4 +- src/Core/SettingsSecrets.h | 6 +-- src/Parsers/FunctionSecretArgumentsFinder.cpp | 37 ++++++------------- src/Parsers/FunctionSecretArgumentsFinder.h | 25 +++---------- src/Storages/NATS/NATS_fwd.h | 6 +-- src/Storages/RabbitMQ/RabbitMQ_fwd.h | 6 +-- 6 files changed, 23 insertions(+), 61 deletions(-) diff --git a/src/Common/maskURIPassword.h b/src/Common/maskURIPassword.h index d1f88c5e43b7d..29038c229fb73 100644 --- a/src/Common/maskURIPassword.h +++ b/src/Common/maskURIPassword.h @@ -73,9 +73,7 @@ inline bool maskURIPassword(std::string * uri) return false; } -/** The offset just past the `://` of a value that starts with an RFC 3986 scheme, and `npos` when it - * does not start with one. A value with no scheme at its start has no authority that a URI parser - * would recognise, so nothing in it can be located as a credential by position. +/** The offset just past the `://` of a value that starts with an RFC 3986 scheme, `npos` otherwise. */ inline size_t findURIAuthority(std::string_view uri) { diff --git a/src/Core/SettingsSecrets.h b/src/Core/SettingsSecrets.h index 11c49b1f82907..41dcce0c253b9 100644 --- a/src/Core/SettingsSecrets.h +++ b/src/Core/SettingsSecrets.h @@ -27,10 +27,8 @@ using ValueMaskingFunc = std::function; /// precondition holds by construction and needs no check. inline bool maskURLCredentials(String & value) { - /// Nothing validates these settings when they are set, `StorageURL::resolveURLBase` accepts a base - /// whose `://` is anywhere rather than at the start, and a statement is masked for logging before - /// its settings are validated. So a value with no scheme in front still reaches a log, and no URI - /// parser can locate the credential inside it: hide such a value whole. + /// A statement is masked for logging before its settings are validated, so a value that no URI + /// parser can read still reaches a log. if (findURIAuthority(value) == String::npos && value.contains('@')) { value = "[HIDDEN]"; diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index b0642fd2cae5d..8fc8edcbfd84a 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -526,12 +526,11 @@ void FunctionSecretArgumentsFinder::findArrowFlightSecretArguments() void FunctionSecretArgumentsFinder::findXDBCSecretArguments() { - /// The connection string is never parsed by ClickHouse: `ITableFunctionXDBC` forwards it verbatim - /// to the bridge, so its grammar belongs to the JDBC/ODBC driver. It can carry the password in a - /// query parameter (`jdbc('mysql://host:3306/?user=root&password=root', ...)`, from this function's - /// own documentation) or as `Pwd=` in a `KEY=value;` list, neither of which a URI scan locates. - /// There is no extent that can be kept visible, so the value is hidden whole; - /// `format_display_secrets_in_show_and_select` still shows the real one to a privileged reader. + /// The connection string goes verbatim to the bridge, so its grammar is the JDBC/ODBC driver's: the + /// password can sit in a query parameter (`?password=`) or as `Pwd=` in a `KEY=value;` list. + /// An invalid call is formatted for logging before validation rejects it, so both branches below + /// fail closed: after a collection name a positional argument can be the connection string, and a + /// named argument means the call is not the positional form at all. if (isNamedCollectionName(0)) { /// jdbc(named_collection, ..., datasource = 'DSN', ...) @@ -554,9 +553,6 @@ void FunctionSecretArgumentsFinder::findXDBCSecretArguments() findSecretNamedArgument("connection_settings", 1); markNamedArgumentsWithUnreadableKeys(1); - /// After the collection name every argument must be a named override. A positional one is - /// invalid, and the statement is formatted for logging before validation rejects it, so it - /// can carry the connection string itself: hide it whole (fail closed). for (size_t i = 1; i < function->arguments->size(); ++i) { const auto equals_func = function->arguments->at(i)->getFunction(); @@ -572,9 +568,6 @@ void FunctionSecretArgumentsFinder::findXDBCSecretArguments() /// JDBC('DSN', database, table) / ODBC('DSN', database, table) markSecretArgument(0, false); - /// A named argument means this is not the positional form at all, and the connection string - /// can then sit at any index under either alias. Validation rejects such a call only after the - /// statement has been formatted for logging, so hide those values too (fail closed). findSecretNamedArgument("datasource", 1); findSecretNamedArgument("connection_settings", 1); markNamedArgumentsWithUnreadableKeys(1); @@ -1025,12 +1018,8 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() } else if (engine_name == "Kafka") { - /// Kafka(named_collection, kafka_sasl_password = '...') - `registerStorageKafka` reads named - /// overrides of the collection, so an override carries the secret the `SETTINGS` clause form - /// hides through `Kafka::SETTINGS_TO_HIDE`. The legacy positional form - /// (`Kafka('brokers', 'topics', 'group', 'format', ...)`) carries no secret and stays visible, - /// so this form does not fail closed on a positional argument. That form also makes the - /// collection name optional, so the scan starts at index 0: a named argument can be the first. + /// Kafka(named_collection, kafka_sasl_password = '...'); the legacy positional form carries no + /// secret and makes the collection name optional, so a named argument can be the first one. findSecretNamedArgument("kafka_sasl_password", 0); markNamedArgumentsWithUnreadableKeys(0); } @@ -1050,14 +1039,11 @@ void FunctionSecretArgumentsFinder::findBrokerTableEngineSecretArguments( /// [, nats_credential_file = '/path'] [, nats_credentials = 'user JWT and seed'] /// [, nats_url = 'nats://user:password@host:4222'] /// [, nats_server_list = 'nats://user:password@host:4222,...'], ...) - /// RabbitMQ(named_collection [, rabbitmq_password = 'password'] - /// [, rabbitmq_address = 'amqp://user:password@host:5672/vhost'], ...) + /// RabbitMQ(named_collection [, rabbitmq_password = '...'] [, rabbitmq_address = 'amqp://user:pass@host'], ...) /// The only positional argument these engines accept is the name of a named collection, so the /// credentials can only appear as named overrides. The `SETTINGS` clause form is masked - /// separately by the engine's own `SETTINGS_TO_HIDE`, and this function masks the same keys the - /// same way: the secrets are hidden whole, and so is an address that carries an '@'. - /// A key list can hold a destination (`nats_server_list`), which is hidden whole because each - /// list entry can carry userinfo credentials. + /// separately by the engine's own `SETTINGS_TO_HIDE`, which this function must stay in sync with. + /// A destination key (`nats_server_list`) is hidden whole: each list entry can carry userinfo. /// Fail closed on a key we cannot read as a plain literal: it can name a secret setting. for (size_t i = 0; i < function->arguments->size(); ++i) { @@ -1084,8 +1070,7 @@ void FunctionSecretArgumentsFinder::findBrokerTableEngineSecretArguments( String url; if (equals_func->arguments->at(1)->tryGetString(&url, /* allow_identifier= */ false)) { - /// An '@' is the only reliable sign of a credential here, and there is no extent to - /// keep visible; see the address rule in the engine's `_fwd.h` for why. + /// An '@' is the only reliable sign of a credential here; see the engine's `_fwd.h`. if (url.contains('@')) markSecretArgument(i, /* argument_is_named= */ true); } diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 2f72472f5f175..79894454c833b 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -121,15 +121,12 @@ class FunctionSecretArgumentsFinder /// Named arguments carrying NATS credentials. They are the setting names, because the `NATS` engine /// takes its arguments as overrides of a named collection (`NATS(collection, nats_token = '...')`). /// `nats_server_list` is a destination and can carry URI userinfo credentials, so hide it whole. - /// `nats_url` is not here because the documented form (`'localhost:4222'`) carries no credential and - /// stays visible; it is handled separately, by the presence of an '@'. + /// `nats_url` is not here: it is hidden only when its value carries an '@'. /// Keep in sync with `NATS::SETTINGS_TO_HIDE`, which masks the same secrets in the `SETTINGS` clause. static constexpr std::string_view nats_secret_keys[] = {"nats_password", "nats_token", "nats_credential_file", "nats_credentials", "nats_server_list"}; - /// Named arguments carrying RabbitMQ credentials, as the setting names, for the same reason as - /// `nats_secret_keys`. `rabbitmq_address` is not here because the documented form carries no - /// credential and stays visible; it is handled by the presence of an '@'. + /// As `nats_secret_keys`, for RabbitMQ; `rabbitmq_address` is hidden only when it carries an '@'. /// Keep in sync with `RabbitMQ::SETTINGS_TO_HIDE`. static constexpr std::string_view rabbitmq_secret_keys[] = {"rabbitmq_password"}; @@ -228,13 +225,6 @@ class FunctionSecretArgumentsFinder void findRedisFunctionSecretArguments(); void findYTsaurusStorageTableEngineSecretArguments(); void findBigQuerySecretArguments(); - /// Masks the named-collection override form of a broker engine - /// (`NATS(collection, nats_token = '...')`, `RabbitMQ(collection, rabbitmq_address = '...')`). - /// `secret_keys` are hidden whole; `address_key` is hidden only when its value carries an '@', - /// because the documented address form carries no credential. Fails closed on a positional - /// argument and on a key this finder cannot read as a plain literal. - /// Keep the key lists in sync with the engine's own `SETTINGS_TO_HIDE`, which masks the same - /// settings in the `SETTINGS` clause. void findBrokerTableEngineSecretArguments( std::span secret_keys, std::string_view address_key); void findNATSTableEngineSecretArguments(); @@ -262,16 +252,11 @@ class FunctionSecretArgumentsFinder /// duplicate-key validation runs, so `session_token = 'a', session_token = 'b'` must hide both. bool findSecretNamedArgument(std::string_view key, size_t start = 0); - /// Hides the value of every `key = value` argument from `start` on whose key this finder cannot read - /// as a plain literal or identifier. The named-collection parser evaluates such a key as a constant - /// expression, so it can name a secret argument that `findNamedArgument` never sees. + /// Hides the value of every `key = value` argument from `start` on whose key is not a plain literal. void markNamedArgumentsWithUnreadableKeys(size_t start); - /// The raw indexes of the arguments from `start` on that are not `key = value` pairs, in order, so a - /// branch that knows a secret's positional slot can find the argument that actually holds it. A - /// positional argument after the first named one is hidden instead of listed: the parsers reject that - /// mix, the statement is formatted for logging before they do, and the slot it was meant to fill is - /// then unknowable. `classifyS3Arguments` applies the same rule to the `s3` signatures. + /// The raw indexes of the arguments from `start` on that are not `key = value` pairs, in order. A + /// positional argument after the first named one is hidden instead of listed: its slot is unknowable. std::vector classifyPositionalArguments(size_t start = 0); /// Masks the secrets of an S3 named-collection form: the secret named overrides (every occurrence, diff --git a/src/Storages/NATS/NATS_fwd.h b/src/Storages/NATS/NATS_fwd.h index a74a9d3687c45..a471bffbbd1dc 100644 --- a/src/Storages/NATS/NATS_fwd.h +++ b/src/Storages/NATS/NATS_fwd.h @@ -25,10 +25,8 @@ static inline std::unordered_map SETTINGS_TO_HIDE = std::string masked_value; if (!value.tryGet(masked_value)) return {}; - /// libnats trims the value, takes the scheme as optional, and ends the userinfo at the LAST - /// '@' of the whole value (`contrib/nats-io/src/url.c`, `natsUrl_Create`), so its credential - /// can run past anything an RFC 3986 authority covers. An '@' is therefore the only reliable - /// sign that this address carries one, and there is no extent to keep visible. + /// libnats takes the scheme as optional and ends the userinfo at the LAST '@' of the whole + /// value (`contrib/nats-io/src/url.c`, `natsUrl_Create`), so no URI authority bounds it. if (masked_value.contains('@')) masked_value = "[HIDDEN]"; return fmt::format("'{}'", masked_value); diff --git a/src/Storages/RabbitMQ/RabbitMQ_fwd.h b/src/Storages/RabbitMQ/RabbitMQ_fwd.h index d0f5ccf96090c..6dc017ae1e250 100644 --- a/src/Storages/RabbitMQ/RabbitMQ_fwd.h +++ b/src/Storages/RabbitMQ/RabbitMQ_fwd.h @@ -20,10 +20,8 @@ static inline std::unordered_map SETTINGS_TO_HIDE = std::string masked_value; if (!value.tryGet(masked_value)) return {}; - /// AMQP-CPP ends the login at the FIRST '@' after the scheme, unbounded by the `/?#` that - /// closes an RFC 3986 authority (`contrib/AMQP-CPP/include/amqpcpp/address.h`, `Address`), so - /// its credential can run past anything a URI masker covers. An '@' is therefore the only - /// reliable sign that this address carries one, and there is no extent to keep visible. + /// AMQP-CPP ends the login at the FIRST '@' after the scheme, unbounded by the `/?#` that closes + /// an RFC 3986 authority (`contrib/AMQP-CPP/include/amqpcpp/address.h`), so no URI masker bounds it. if (masked_value.contains('@')) masked_value = "[HIDDEN]"; return fmt::format("'{}'", masked_value); From f5fd959ecd20c72268a41f7c63c66a177f1b341a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 12:48:04 +0000 Subject: [PATCH 147/296] Backport #122345 to 26.8: Fix LOGICAL_ERROR on lambda over an `Array(Tuple(...))` --- src/Analyzer/Resolve/IdentifierResolver.cpp | 11 ++++ .../evaluateScalarSubqueryIfNeeded.cpp | 4 +- src/Analyzer/Resolve/resolveFunction.cpp | 5 +- src/Interpreters/Context.cpp | 6 +++ src/Interpreters/Context.h | 1 + .../InterpreterSelectQueryAnalyzer.cpp | 37 +++++++++++++- .../QueryPlan/DistributedCreateLocalPlan.cpp | 5 ++ src/Storages/StorageView.cpp | 3 ++ ...lter_source_column_during_insert.reference | 1 + ...nsert_alter_source_column_during_insert.sh | 43 ++++++++++++++++ ...ary_view_over_source_reads_table.reference | 5 ++ ..._ordinary_view_over_source_reads_table.sql | 51 +++++++++++++++++++ ...292_mv_source_substitution_scope.reference | 16 ++++++ .../05292_mv_source_substitution_scope.sql | 51 +++++++++++++++++++ ...rdinary_view_over_source_inlined.reference | 4 ++ ...3_mv_ordinary_view_over_source_inlined.sql | 37 ++++++++++++++ ...e_remote_local_shard_reads_table.reference | 2 + ...v_source_remote_local_shard_reads_table.sh | 45 ++++++++++++++++ 18 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05241_mv_insert_alter_source_column_during_insert.reference create mode 100755 tests/queries/0_stateless/05241_mv_insert_alter_source_column_during_insert.sh create mode 100644 tests/queries/0_stateless/05291_mv_ordinary_view_over_source_reads_table.reference create mode 100644 tests/queries/0_stateless/05291_mv_ordinary_view_over_source_reads_table.sql create mode 100644 tests/queries/0_stateless/05292_mv_source_substitution_scope.reference create mode 100644 tests/queries/0_stateless/05292_mv_source_substitution_scope.sql create mode 100644 tests/queries/0_stateless/05293_mv_ordinary_view_over_source_inlined.reference create mode 100644 tests/queries/0_stateless/05293_mv_ordinary_view_over_source_inlined.sql create mode 100644 tests/queries/0_stateless/05294_mv_source_remote_local_shard_reads_table.reference create mode 100755 tests/queries/0_stateless/05294_mv_source_remote_local_shard_reads_table.sh diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index c14879e0afb25..73cd1c3c40a99 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -305,6 +305,17 @@ std::shared_ptr IdentifierResolver::tryResolveTableIdentifier(const I StorageID storage_id(database_name, table_name); storage_id = context->resolveStorageID(storage_id); + + /// The view source carries the inserted block and its types. For a MV, return this source + /// directly as a table node instead of swapping it later for the storage from the catalog + /// which may have been changed by a concurrent ALTER (the MV types must match the + /// snapshot at the start of the INSERT, not the current types). + /// For an inner query of an ordinary view, keep the normal flow that resolves from the catalog. + if (auto view_source = context->getViewSource(); + view_source && !context->isViewInnerQuery() + && view_source->getStorageID().getFullNameNotQuoted() == storage_id.getFullNameNotQuoted()) + return std::make_shared(view_source, context); + bool is_temporary_table = storage_id.getDatabaseName() == DatabaseCatalog::TEMPORARY_DATABASE; StoragePtr storage; diff --git a/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp b/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp index 2b9c6355a3300..93d4dfb653cb7 100644 --- a/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp +++ b/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp @@ -215,7 +215,9 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden addQueryTreePasses(query_tree_pass_manager, options.only_analyze); query_tree_pass_manager.run(query_tree); - if (auto storage = subquery_context->getViewSource()) + /// The inner query of an ordinary view referenced by the view query reads the table itself, + /// not the inserted block. + if (auto storage = subquery_context->getViewSource(); storage && !subquery_context->isViewInnerQuery()) replaceStorageInQueryTree(query_tree, subquery_context, storage); auto interpreter = std::make_unique(query_tree, subquery_context, options); diff --git a/src/Analyzer/Resolve/resolveFunction.cpp b/src/Analyzer/Resolve/resolveFunction.cpp index bb3e74a48993d..aacf493888b7c 100644 --- a/src/Analyzer/Resolve/resolveFunction.cpp +++ b/src/Analyzer/Resolve/resolveFunction.cpp @@ -2141,8 +2141,9 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi } else { - /// Replace storage with values storage of insertion block - if (StoragePtr storage = scope.context->getViewSource()) + /// Replace storage with values storage of insertion block. + /// The inner query of an ordinary view referenced by the view query reads the table itself. + if (StoragePtr storage = scope.context->getViewSource(); storage && !scope.context->isViewInnerQuery()) { QueryTreeNodePtr table_expression = in_second_argument; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 4bcc3b6bc7672..54cc24c6f638f 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -3388,6 +3388,12 @@ StoragePtr Context::getViewSource() const return view_source; } + +void Context::clearViewSource() +{ + view_source.reset(); +} + bool Context::displaySecretsInShowAndSelect() const { return shared->server_settings[ServerSetting::display_secrets_in_show_and_select]; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6c2dcdcff8f59..522eef1fa017b 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1156,6 +1156,7 @@ class Context: public ContextData, public std::enable_shared_from_this void addViewSource(const StoragePtr & storage); StoragePtr getViewSource() const; + void clearViewSource(); String getCurrentDatabase() const; String getCurrentQueryId() const { return client_info.current_query_id; } diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 4f39711e5f35f..d8355d065b165 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -219,9 +219,44 @@ QueryPlanPtr buildQueryPlanForAutomaticParallelReplicas( } } +/// Like `extractAllTableReferences`, but does not descend into the inner queries of views inlined +/// by the analyzer (`analyzer_inline_views`) into a query that is not itself inside a view: +/// they read their own tables, just like a view that is not inlined. +static bool isViewInnerQueryNode(const QueryTreeNodePtr & node) +{ + if (const auto * query_node = node->as()) + return query_node->getContext()->isViewInnerQuery(); + if (const auto * union_node = node->as()) + return union_node->getContext()->isViewInnerQuery(); + return false; +} + +static void extractTableReferencesOutsideViews(const QueryTreeNodePtr & node, bool outer_is_view_inner, QueryTreeNodes & result) +{ + bool is_view_inner = isViewInnerQueryNode(node); + if (is_view_inner && !outer_is_view_inner) + return; + + if (node->getNodeType() == QueryTreeNodeType::TABLE) + { + result.push_back(node); + } + else if (const auto * query_node = node->as()) + { + for (const auto & table_expression : extractTableExpressions(query_node->getJoinTreeNodeTyped(), /*add_array_join=*/ false, /*recursive=*/ false)) + extractTableReferencesOutsideViews(table_expression, is_view_inner, result); + } + else if (const auto * union_node = node->as()) + { + for (const auto & query : union_node->getQueries().getNodes()) + extractTableReferencesOutsideViews(query, is_view_inner, result); + } +} + void replaceStorageInQueryTree(QueryTreeNodePtr & query_tree, const ContextPtr & context, const StoragePtr & storage) { - auto nodes = extractAllTableReferences(query_tree); + QueryTreeNodes nodes; + extractTableReferencesOutsideViews(query_tree, isViewInnerQueryNode(query_tree), nodes); IQueryTreeNode::ReplacementMap replacement_map; for (auto & node : nodes) diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index 7697b0e5ed0c0..e98c055c074ae 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -29,6 +29,11 @@ std::unique_ptr createLocalPlan( auto query_plan = std::make_unique(); auto new_context = Context::createCopy(context); + /// The local shard reads the table itself, like the remote shards do: the inserted block + /// of a materialized view (whose query contains this `remote` or `Distributed` read) must not + /// replace the source table here. + new_context->clearViewSource(); + if (build_logical_plan && !default_database.empty()) new_context->setCurrentDatabase(default_database); diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index b98dfce1db651..1d27a13b7d6e3 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -771,6 +771,9 @@ ContextPtr StorageView::getViewSubqueryContext(ContextPtr context, const Storage view_settings[Setting::max_result_bytes] = 0; view_settings[Setting::extremes] = false; view_context->setSettings(view_settings); + /// The inlined view body is the inner query of the view, just like in `getViewContext`: + /// e.g. it must read the table itself, not the inserted block of a materialized view. + view_context->setIsViewInnerQuery(true); return view_context; } diff --git a/tests/queries/0_stateless/05241_mv_insert_alter_source_column_during_insert.reference b/tests/queries/0_stateless/05241_mv_insert_alter_source_column_during_insert.reference new file mode 100644 index 0000000000000..64bb6b746dcea --- /dev/null +++ b/tests/queries/0_stateless/05241_mv_insert_alter_source_column_during_insert.reference @@ -0,0 +1 @@ +30 diff --git a/tests/queries/0_stateless/05241_mv_insert_alter_source_column_during_insert.sh b/tests/queries/0_stateless/05241_mv_insert_alter_source_column_during_insert.sh new file mode 100755 index 0000000000000..ca4d82b589af1 --- /dev/null +++ b/tests/queries/0_stateless/05241_mv_insert_alter_source_column_during_insert.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# ALTER of the source column type between two blocks of one INSERT. The view query must be +# resolved against the pushed block, not against the table's current metadata. + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS src; + DROP TABLE IF EXISTS dst; + DROP TABLE IF EXISTS mv; + CREATE TABLE src (arr Array(Tuple(a UInt32))) ENGINE = Null; + CREATE TABLE dst (cnt UInt64) ENGINE = MergeTree ORDER BY tuple(); + CREATE MATERIALIZED VIEW mv TO dst AS SELECT count() AS cnt FROM src WHERE arrayExists(x -> x.a >= 0, arr); +" + +# One row per block, 0.2 s apart, so the INSERT lasts about 6 s. +$CLICKHOUSE_CLIENT -q " + INSERT INTO src + SELECT [tuple(toUInt32(number))] + FROM numbers(30) + WHERE NOT ignore(sleepEachRow(0.2)) + SETTINGS max_block_size = 1, min_insert_block_size_rows = 1, min_insert_block_size_bytes = 1, max_threads = 1, max_insert_threads = 1 +" & + +# Wait until the first block has passed through the view, so the INSERT surely started with the old type. +while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM dst") -lt 1 ]] +do + sleep 0.1 +done + +$CLICKHOUSE_CLIENT -q "ALTER TABLE src MODIFY COLUMN arr Array(Tuple(a UInt32, b Nullable(UInt32)))" +wait + +$CLICKHOUSE_CLIENT -q "SELECT sum(cnt) FROM dst" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE mv; + DROP TABLE dst; + DROP TABLE src; +" diff --git a/tests/queries/0_stateless/05291_mv_ordinary_view_over_source_reads_table.reference b/tests/queries/0_stateless/05291_mv_ordinary_view_over_source_reads_table.reference new file mode 100644 index 0000000000000..09e49688a648c --- /dev/null +++ b/tests/queries/0_stateless/05291_mv_ordinary_view_over_source_reads_table.reference @@ -0,0 +1,5 @@ +3 1 +1 1 +2 1 +3 1 +3 1 1 diff --git a/tests/queries/0_stateless/05291_mv_ordinary_view_over_source_reads_table.sql b/tests/queries/0_stateless/05291_mv_ordinary_view_over_source_reads_table.sql new file mode 100644 index 0000000000000..52aecd93f385f --- /dev/null +++ b/tests/queries/0_stateless/05291_mv_ordinary_view_over_source_reads_table.sql @@ -0,0 +1,51 @@ +-- The query of a materialized view sees only the inserted block of its source table, +-- but an ordinary view over the same table referenced from that query reads the whole table. + +DROP TABLE IF EXISTS mv_scalar; +DROP TABLE IF EXISTS mv_join; +DROP TABLE IF EXISTS mv_inner; +DROP TABLE IF EXISTS dst_scalar; +DROP TABLE IF EXISTS dst_join; +DROP TABLE IF EXISTS dst_inner; +DROP VIEW IF EXISTS v; +DROP VIEW IF EXISTS v_in; +DROP VIEW IF EXISTS v_scalar; +DROP TABLE IF EXISTS src; + +CREATE TABLE src (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE VIEW v AS SELECT x FROM src; +-- The subqueries inside an ordinary view read the table as well. +CREATE VIEW v_in AS SELECT count() AS c FROM numbers(200) WHERE number IN (SELECT x FROM src); +CREATE VIEW v_scalar AS SELECT (SELECT count() FROM src) AS c; +-- Rows inserted before the materialized views exist: the view sees them, the inserted block does not contain them. +INSERT INTO src SELECT number + 100 FROM numbers(10); + +CREATE TABLE dst_scalar (block_rows UInt64, view_rows UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE MATERIALIZED VIEW mv_scalar TO dst_scalar AS + SELECT count() AS block_rows, (SELECT count() FROM v) AS view_rows FROM src; + +CREATE TABLE dst_join (x UInt64, view_rows UInt64) ENGINE = MergeTree ORDER BY x; +CREATE MATERIALIZED VIEW mv_join TO dst_join AS + SELECT s.x AS x, w.c AS view_rows FROM src AS s CROSS JOIN (SELECT count() AS c FROM v) AS w; + +CREATE TABLE dst_inner (block_rows UInt64, in_rows Nullable(UInt64), scalar_rows Nullable(UInt64)) ENGINE = MergeTree ORDER BY tuple(); +CREATE MATERIALIZED VIEW mv_inner TO dst_inner AS + SELECT count() AS block_rows, (SELECT c FROM v_in) AS in_rows, (SELECT c FROM v_scalar) AS scalar_rows FROM src; + +INSERT INTO src VALUES (1), (2), (3); + +-- Whether the inserted block itself is already visible through the view depends on when the part is committed. +SELECT block_rows, view_rows >= 10 FROM dst_scalar; +SELECT x, view_rows >= 10 FROM dst_join ORDER BY x; +SELECT block_rows, in_rows >= 10, scalar_rows >= 10 FROM dst_inner; + +DROP TABLE mv_scalar; +DROP TABLE mv_join; +DROP TABLE mv_inner; +DROP TABLE dst_scalar; +DROP TABLE dst_join; +DROP TABLE dst_inner; +DROP VIEW v; +DROP VIEW v_in; +DROP VIEW v_scalar; +DROP TABLE src; diff --git a/tests/queries/0_stateless/05292_mv_source_substitution_scope.reference b/tests/queries/0_stateless/05292_mv_source_substitution_scope.reference new file mode 100644 index 0000000000000..bf0a7a17cc333 --- /dev/null +++ b/tests/queries/0_stateless/05292_mv_source_substitution_scope.reference @@ -0,0 +1,16 @@ +view in scalar subquery +1 2 +2 2 +3 3 +view in join +1 2 +2 2 +3 3 +source table in subquery +1 2 +2 2 +3 1 +source table in IN +1 2 +2 2 +3 1 diff --git a/tests/queries/0_stateless/05292_mv_source_substitution_scope.sql b/tests/queries/0_stateless/05292_mv_source_substitution_scope.sql new file mode 100644 index 0000000000000..81523cd807b89 --- /dev/null +++ b/tests/queries/0_stateless/05292_mv_source_substitution_scope.sql @@ -0,0 +1,51 @@ +-- The pushed block stands in for the source table in the view query, including its subqueries. +-- A view over the source table keeps reading the whole table. + +DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS v; +DROP TABLE IF EXISTS dst_scalar; +DROP TABLE IF EXISTS dst_join; +DROP TABLE IF EXISTS dst_subquery; +DROP TABLE IF EXISTS dst_in; +DROP TABLE IF EXISTS mv_scalar; +DROP TABLE IF EXISTS mv_join; +DROP TABLE IF EXISTS mv_subquery; +DROP TABLE IF EXISTS mv_in; + +CREATE TABLE src (k UInt32) ENGINE = MergeTree ORDER BY k; +CREATE VIEW v AS SELECT count() AS c FROM src; + +CREATE TABLE dst_scalar (k UInt32, c UInt64) ENGINE = MergeTree ORDER BY k; +CREATE MATERIALIZED VIEW mv_scalar TO dst_scalar AS SELECT k, (SELECT c FROM v) AS c FROM src; + +CREATE TABLE dst_join (k UInt32, c UInt64) ENGINE = MergeTree ORDER BY k; +CREATE MATERIALIZED VIEW mv_join TO dst_join AS SELECT k, c FROM src CROSS JOIN v; + +CREATE TABLE dst_subquery (k UInt32, c UInt64) ENGINE = MergeTree ORDER BY k; +CREATE MATERIALIZED VIEW mv_subquery TO dst_subquery AS SELECT k, c FROM src CROSS JOIN (SELECT count() AS c FROM src) AS t; + +CREATE TABLE dst_in (k UInt32, c UInt64) ENGINE = MergeTree ORDER BY k; +CREATE MATERIALIZED VIEW mv_in TO dst_in AS SELECT k, c FROM src CROSS JOIN (SELECT count() AS c FROM numbers(10) WHERE number IN (SELECT k FROM src)) AS t; + +INSERT INTO src VALUES (1), (2); +INSERT INTO src VALUES (3); + +SELECT 'view in scalar subquery'; +SELECT * FROM dst_scalar ORDER BY k; +SELECT 'view in join'; +SELECT * FROM dst_join ORDER BY k; +SELECT 'source table in subquery'; +SELECT * FROM dst_subquery ORDER BY k; +SELECT 'source table in IN'; +SELECT * FROM dst_in ORDER BY k; + +DROP TABLE mv_scalar; +DROP TABLE mv_join; +DROP TABLE mv_subquery; +DROP TABLE mv_in; +DROP TABLE dst_scalar; +DROP TABLE dst_join; +DROP TABLE dst_subquery; +DROP TABLE dst_in; +DROP TABLE v; +DROP TABLE src; diff --git a/tests/queries/0_stateless/05293_mv_ordinary_view_over_source_inlined.reference b/tests/queries/0_stateless/05293_mv_ordinary_view_over_source_inlined.reference new file mode 100644 index 0000000000000..f551dab923cb0 --- /dev/null +++ b/tests/queries/0_stateless/05293_mv_ordinary_view_over_source_inlined.reference @@ -0,0 +1,4 @@ +3 1 +1 1 +2 1 +3 1 diff --git a/tests/queries/0_stateless/05293_mv_ordinary_view_over_source_inlined.sql b/tests/queries/0_stateless/05293_mv_ordinary_view_over_source_inlined.sql new file mode 100644 index 0000000000000..0641478a90aec --- /dev/null +++ b/tests/queries/0_stateless/05293_mv_ordinary_view_over_source_inlined.sql @@ -0,0 +1,37 @@ +-- An ordinary view over the source table of a materialized view reads the whole table +-- also when the analyzer inlines the view body into the view query. + +SET analyzer_inline_views = 1; + +DROP TABLE IF EXISTS mv_scalar; +DROP TABLE IF EXISTS mv_join; +DROP TABLE IF EXISTS dst_scalar; +DROP TABLE IF EXISTS dst_join; +DROP VIEW IF EXISTS v; +DROP TABLE IF EXISTS src; + +CREATE TABLE src (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE VIEW v AS SELECT x FROM src; +-- Rows inserted before the materialized views exist: the view sees them, the inserted block does not contain them. +INSERT INTO src SELECT number + 100 FROM numbers(10); + +CREATE TABLE dst_scalar (block_rows UInt64, view_rows UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE MATERIALIZED VIEW mv_scalar TO dst_scalar AS + SELECT count() AS block_rows, (SELECT count() FROM v) AS view_rows FROM src; + +CREATE TABLE dst_join (x UInt64, view_rows UInt64) ENGINE = MergeTree ORDER BY x; +CREATE MATERIALIZED VIEW mv_join TO dst_join AS + SELECT s.x AS x, w.c AS view_rows FROM src AS s CROSS JOIN (SELECT count() AS c FROM v) AS w; + +INSERT INTO src VALUES (1), (2), (3); + +-- Whether the inserted block itself is already visible through the view depends on when the part is committed. +SELECT block_rows, view_rows >= 10 FROM dst_scalar; +SELECT x, view_rows >= 10 FROM dst_join ORDER BY x; + +DROP TABLE mv_scalar; +DROP TABLE mv_join; +DROP TABLE dst_scalar; +DROP TABLE dst_join; +DROP VIEW v; +DROP TABLE src; diff --git a/tests/queries/0_stateless/05294_mv_source_remote_local_shard_reads_table.reference b/tests/queries/0_stateless/05294_mv_source_remote_local_shard_reads_table.reference new file mode 100644 index 0000000000000..ae2506b46c843 --- /dev/null +++ b/tests/queries/0_stateless/05294_mv_source_remote_local_shard_reads_table.reference @@ -0,0 +1,2 @@ +distributed 3 10 +remote 3 10 diff --git a/tests/queries/0_stateless/05294_mv_source_remote_local_shard_reads_table.sh b/tests/queries/0_stateless/05294_mv_source_remote_local_shard_reads_table.sh new file mode 100755 index 0000000000000..c1e513627cacd --- /dev/null +++ b/tests/queries/0_stateless/05294_mv_source_remote_local_shard_reads_table.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# The local shard of a `remote` or `Distributed` read inside a materialized view query +# reads the source table itself, not the inserted block, like the remote shards do. +# The database name is spelled out: `currentDatabase()` in the view query does not resolve +# to the test database while the block is pushed to the view. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -m -q " +DROP TABLE IF EXISTS mv; +DROP TABLE IF EXISTS mv_dist; +DROP TABLE IF EXISTS dst; +DROP TABLE IF EXISTS dist; +DROP TABLE IF EXISTS src; + +CREATE TABLE src (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO src SELECT number + 100 FROM numbers(10); + +CREATE TABLE dist AS src ENGINE = Distributed(test_shard_localhost, ${CLICKHOUSE_DATABASE}, src); + +CREATE TABLE dst (name String, block_rows UInt64, table_rows UInt64) ENGINE = MergeTree ORDER BY name; + +CREATE MATERIALIZED VIEW mv TO dst AS + SELECT 'remote' AS name, count() AS block_rows, + (SELECT count() FROM remote('127.0.0.1', ${CLICKHOUSE_DATABASE}, src) WHERE x >= 100) AS table_rows + FROM src; + +CREATE MATERIALIZED VIEW mv_dist TO dst AS + SELECT 'distributed' AS name, count() AS block_rows, + (SELECT count() FROM ${CLICKHOUSE_DATABASE}.dist WHERE x >= 100) AS table_rows + FROM src; + +INSERT INTO src VALUES (1), (2), (3); + +SELECT * FROM dst ORDER BY name; + +DROP TABLE mv; +DROP TABLE mv_dist; +DROP TABLE dst; +DROP TABLE dist; +DROP TABLE src; +" From 306e3a25edb2a2693ea8aebe1d1f1428aae025c7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 12:54:46 +0000 Subject: [PATCH 148/296] Backport #122673 to 26.8: Hide HTTP headers of `HTTP` dictionary sources --- src/Dictionaries/HTTPDictionarySource.cpp | 2 + .../ASTFunctionWithKeyValueArguments.cpp | 12 +++- ...tp_dictionary_hide_header_values.reference | 9 +++ ...141_http_dictionary_hide_header_values.sql | 68 +++++++++++++++++++ ...ctionary_hide_header_values_json.reference | 3 + ...http_dictionary_hide_header_values_json.sh | 46 +++++++++++++ 6 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference create mode 100644 tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql create mode 100644 tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference create mode 100755 tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 3e70047a89061..6d7970523bef0 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -413,6 +413,8 @@ Setting fields: | `value` | Value set for a specific identifier name. | When creating a dictionary using the DDL command (`CREATE DICTIONARY ...`) remote hosts for HTTP dictionaries are checked against the contents of `remote_url_allow_hosts` section from config to prevent database users to access arbitrary HTTP server. + +The headers, including their names, are shown as `HEADERS ('[HIDDEN]')` in the output of `SHOW CREATE DICTIONARY`, in `system.tables` and in the query logs, the same way as the password. As with the password, a query that cannot be parsed is logged as is, with only [`query_masking_rules`](/reference/settings/server-settings/settings/query#query_masking_rules) applied. To display the headers in `SHOW CREATE DICTIONARY` and `system.tables`, enable the server setting [`display_secrets_in_show_and_select`](/reference/settings/server-settings/settings/other#display_secrets_in_show_and_select) and the format setting [`format_display_secrets_in_show_and_select`](/reference/settings/formats/format#format_display_secrets_in_show_and_select); the user also needs the `displaySecretsInShowAndSelect` privilege. These settings do not affect the query logs. )DOCS_MD", .syntax = "SOURCE(HTTP(url 'https://host/path' format 'CSV'))", .related = {"file"}}); diff --git a/src/Parsers/ASTFunctionWithKeyValueArguments.cpp b/src/Parsers/ASTFunctionWithKeyValueArguments.cpp index ea1b63f82b41c..17156bf476eff 100644 --- a/src/Parsers/ASTFunctionWithKeyValueArguments.cpp +++ b/src/Parsers/ASTFunctionWithKeyValueArguments.cpp @@ -20,12 +20,16 @@ namespace { /// Keys of a dictionary source whose value must not be shown. Besides the password, this covers /// the TLS credentials that are given as the contents of a certificate or a key file (a path is - /// not accepted from a `CREATE DICTIONARY` query in the first place). + /// not accepted from a `CREATE DICTIONARY` query in the first place), and the custom HTTP headers + /// of the `HTTP` source, whose values often carry API tokens. The headers are hidden as a whole, + /// names included: the query is logged before the dictionary source validates its structure, + /// so a malformed definition must not leak either. bool isSecretKey(const String & key) { return key == "password" || key == "ssl_ca_pem" || key == "ssl_cert_pem" || key == "ssl_key_pem" - || key == "sslrootcert_pem" || key == "sslcert_pem" || key == "sslkey_pem"; + || key == "sslrootcert_pem" || key == "sslcert_pem" || key == "sslkey_pem" + || key == "headers" || key == "header"; } } @@ -56,7 +60,9 @@ void ASTPair::readJSON(const Poco::JSON::Object & json) { JSONObjectReader r(json); - first = r.getString("first"); + /// The SQL parser lower-cases the key (see `ParserKeyValuePair`), and the checks for secret keys in + /// `formatImpl` and `hasSecretParts` rely on it, so canonicalize it the same way here. + first = Poco::toLower(r.getString("first")); if (first.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Missing or empty 'first' in ASTPair during AST JSON deserialization"); diff --git a/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference new file mode 100644 index 0000000000000..ca36e0892c1a6 --- /dev/null +++ b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference @@ -0,0 +1,9 @@ +CREATE DICTIONARY default.d_05141\n(\n `id` UInt64,\n `v` String\n)\nPRIMARY KEY id\nSOURCE(HTTP(URL \'http://localhost:11111/x.tsv\' FORMAT \'TabSeparated\' CREDENTIALS (USER \'user\' PASSWORD \'[HIDDEN]\') HEADERS (\'[HIDDEN]\')))\nLIFETIME(MIN 0 MAX 0)\nLAYOUT(FLAT()) +0 0 1 +d_05141_flat HEADERS \'[HIDDEN]\')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_foo HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_key HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_nested HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_nobr HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_typo HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +2 14 0 diff --git a/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql new file mode 100644 index 0000000000000..0e1468fdee93c --- /dev/null +++ b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql @@ -0,0 +1,68 @@ +-- The values of custom HTTP headers of an `HTTP` dictionary source often carry credentials, +-- so they must be hidden in `SHOW CREATE DICTIONARY`, `system.tables` and `system.query_log`, +-- the same way as the password. They are hidden as a whole, header names included. + +SET format_display_secrets_in_show_and_select = 0; + +DROP DICTIONARY IF EXISTS d_05141; +CREATE DICTIONARY d_05141 (id UInt64, v String) +PRIMARY KEY id +SOURCE(HTTP( + url 'http://localhost:11111/x.tsv' + format 'TabSeparated' + credentials(user 'user' password 'SEKRIT_PW') + headers( + header(name 'API-KEY' value 'SEKRIT_TOKEN_1') + header(name 'X-Other' value 'SEKRIT_TOKEN_2') + ) +)) +LIFETIME(0) LAYOUT(FLAT()); + +SHOW CREATE DICTIONARY d_05141; + +SELECT create_table_query LIKE concat('%', 'SEKRIT', '%'), create_table_query LIKE '%API-KEY%', create_table_query LIKE '%HEADERS (\'[HIDDEN]\')%' +FROM system.tables WHERE database = currentDatabase() AND name = 'd_05141'; + +DROP DICTIONARY d_05141; + +-- The query is logged before the dictionary source validates its structure, so malformed header +-- definitions must not leak either. +CREATE DICTIONARY d_05141_typo (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name 'API-KEY' vaule 'SEKRIT_TYPO')))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_key (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(secret 'SEKRIT_KEY')))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_nested (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name(foo 'SEKRIT_NESTED'))))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_func (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name concat('X-', 'SEKRIT_FUNC') value 'SEKRIT_FUNC_VALUE')))) +LIFETIME(0) LAYOUT(FLAT()); -- { serverError INCORRECT_DICTIONARY_DEFINITION } +CREATE DICTIONARY d_05141_array (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name ['SEKRIT_ARRAY'])))) +LIFETIME(0) LAYOUT(FLAT()); -- { serverError BAD_ARGUMENTS } +CREATE DICTIONARY d_05141_nobr (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header 'SEKRIT_NOBR'))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_foo (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(foo 'SEKRIT_FOO'))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_flat (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers 'SEKRIT_HEADERS')) +LIFETIME(0) LAYOUT(FLAT()); + +SELECT name, extract(create_table_query, 'HEADERS.*$') +FROM system.tables WHERE database = currentDatabase() AND name LIKE 'd\\_05141\\_%' ORDER BY name; + +SYSTEM FLUSH LOGS query_log; +SELECT countIf(query LIKE '%d\\_05141 %'), countIf(query LIKE '%d\\_05141\\_%'), countIf(query LIKE concat('%', 'SEKRIT', '%')) +FROM system.query_log +WHERE current_database = currentDatabase() AND query_kind = 'Create' AND event_date >= yesterday(); + +DROP DICTIONARY d_05141_typo; +DROP DICTIONARY d_05141_key; +DROP DICTIONARY d_05141_nested; +DROP DICTIONARY d_05141_nobr; +DROP DICTIONARY d_05141_foo; +DROP DICTIONARY d_05141_flat; diff --git a/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference new file mode 100644 index 0000000000000..f7c7e2e153250 --- /dev/null +++ b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference @@ -0,0 +1,3 @@ +HEADERS (\'[HIDDEN]\'))) +CREDENTIALS (USER \'user\' PASSWORD \'[HIDDEN]\') HEADERS (\'[HIDDEN]\'))) +1 0 diff --git a/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh new file mode 100755 index 0000000000000..11eae7fc24b49 --- /dev/null +++ b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +# The HTTP headers of an `HTTP` dictionary source must also be hidden when the query is submitted +# as a JSON AST (`dialect = 'clickhouse_json'`). + +CLICKHOUSE_CLIENT_JSON="${CLICKHOUSE_CLIENT} --enable_json_ast_dialect 1 --dialect clickhouse_json" + +function to_json() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT parseQueryToJSON(\$\$$1\$\$) FORMAT TSVRaw" +} + +# A definition submitted as a JSON AST is masked like the SQL one. +JSON=$(to_json "CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.d_05142 (id UInt64, v String) PRIMARY KEY id + SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name 'API-KEY' value 'SEKRIT_JSON')))) + LIFETIME(0) LAYOUT(FLAT())") +${CLICKHOUSE_CLIENT_JSON} --query "$JSON" +${CLICKHOUSE_CLIENT} --query "SELECT extract(create_table_query, 'HEADERS.*\\)\\)\\)') FROM system.tables WHERE database = currentDatabase() AND name = 'd_05142' SETTINGS format_display_secrets_in_show_and_select = 0" + +# The SQL parser lower-cases the keys, but a JSON AST can spell them in any case; the secret keys must +# be recognized anyway. +JSON=$(to_json "CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.d_05142_case (id UInt64, v String) PRIMARY KEY id + SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' credentials(user 'user' password 'SEKRIT_JSON_CASE_PW') + headers(header(name 'API-KEY' value 'SEKRIT_JSON_CASE')))) + LIFETIME(0) LAYOUT(FLAT())") +for key in headers header value password +do + JSON=${JSON//\"first\":\"$key\"/\"first\":\"${key^^}\"} +done +${CLICKHOUSE_CLIENT_JSON} --query "$JSON" +${CLICKHOUSE_CLIENT} --query "SELECT extract(create_table_query, 'CREDENTIALS.*\\)\\)\\)') FROM system.tables WHERE database = currentDatabase() AND name = 'd_05142_case' SETTINGS format_display_secrets_in_show_and_select = 0" + +# The JSON queries are logged without the headers and the password. +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" +${CLICKHOUSE_CLIENT} --query " + SELECT count() > 0, countIf(query LIKE concat('%', 'SEKRIT', '%')) + FROM system.query_log + WHERE current_database = currentDatabase() AND query_kind = 'Create' AND query LIKE '%d_05142%' AND event_date >= yesterday()" + +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY d_05142" +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY d_05142_case" From e02ecceb99a30785413f5a74251acaae78196e84 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 1 Oct 2026 14:02:18 +0000 Subject: [PATCH 149/296] Keep only the legacy analyzer part of #121398 A right table with a row policy is joined through its query plan instead of its special storage, so the `FilterStep` carrying the policy is kept. The `ACCESS_DENIED` for `JOIN` and `joinGet` on a `Join` table and its docs are not backported. This also drops the `registerStatementRowPolicy` block that the conflict resolution pulled into the parser: `StatementFactory` does not exist in 26.8. Co-Authored-By: Claude Fable 5.1 --- src/Functions/FunctionJoinGet.cpp | 12 +- src/Interpreters/JoinedTables.cpp | 20 +- .../Access/ParserCreateRowPolicyQuery.cpp | 302 ------------------ src/Planner/PlannerJoinTree.cpp | 20 +- src/Storages/StorageJoin.cpp | 2 - .../05233_direct_join_row_policy.reference | 24 +- .../05233_direct_join_row_policy.sql | 40 +-- .../05234_row_policy_storage_join.reference | 12 - .../05234_row_policy_storage_join.sql | 31 -- 9 files changed, 33 insertions(+), 430 deletions(-) delete mode 100644 tests/queries/0_stateless/05234_row_policy_storage_join.reference delete mode 100644 tests/queries/0_stateless/05234_row_policy_storage_join.sql diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 3c71c00821c1e..e3d7b87f97d8e 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -11,8 +11,6 @@ #include #include #include -#include -#include namespace DB { @@ -23,7 +21,6 @@ namespace Setting namespace ErrorCodes { - extern const int ACCESS_DENIED; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -149,14 +146,7 @@ ExecutableFunctionPtr FunctionJoinGet::prepare(const ColumnsWithTypeAndName &) c Names column_names = storage_join->getKeyNames(); column_names.push_back(attr_name); - const auto storage_id = storage_join->getStorageID(); - context->checkAccess(AccessType::SELECT, storage_id, column_names); - - /// The hash table is read as is, so a row policy on the table cannot be applied here any more than in a JOIN. - auto row_policy_filter = context->getRowPolicyFilter(storage_id.getDatabaseName(), storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); - if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) - throw Exception(ErrorCodes::ACCESS_DENIED, - "Cannot use {} because a row policy is applied on table {} with the Join engine", function_name, storage_id.getNameForLogs()); + context->checkAccess(AccessType::SELECT, storage_join->getStorageID(), column_names); return std::make_unique(function_name, context, table_lock, storage_join, result_columns); } diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index c64b2735268ab..b1abb5e4a4785 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -42,7 +42,6 @@ namespace Setting namespace ErrorCodes { - extern const int ACCESS_DENIED; extern const int ALIAS_REQUIRED; extern const int AMBIGUOUS_COLUMN_NAME; extern const int LOGICAL_ERROR; @@ -349,20 +348,11 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se StoragePtr storage = DatabaseCatalog::instance().tryGetTable(joined_table_id, context); /// A special storage replaces the right-side plan, and with it the `FilterStep` carrying the - /// table's row policy, so such a table has to be joined as an ordinary stream. A `Join` table - /// is a prebuilt hash table read as is, so it cannot be filtered at all. - if (storage) - { - auto row_policy_filter = context->getRowPolicyFilter( - joined_table_id.getDatabaseName(), joined_table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); - if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) - { - if (typeid_cast(storage.get())) - throw Exception(ErrorCodes::ACCESS_DENIED, - "Cannot join table {} with the Join engine because a row policy is applied on it", joined_table_id.getNameForLogs()); - storage = nullptr; - } - } + /// table's row policy, so such a table has to be joined as an ordinary stream. + auto joined_table_row_policy = context->getRowPolicyFilter( + joined_table_id.getDatabaseName(), joined_table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + if (joined_table_row_policy && !joined_table_row_policy->isAlwaysTrue()) + storage = nullptr; if (storage) { diff --git a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp index e6e8bea7217d9..63dcf97699614 100644 --- a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp @@ -315,305 +315,3 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & return true; } } - -namespace DB -{ - -void registerStatementRowPolicy(StatementFactory & factory) -{ - factory.registerStatement("CREATE ROW POLICY", - { - .description = R"DOCS_MD( -Creates a [row policy](/concepts/features/security/access-rights#row-policy-management), i.e. a filter used to determine which rows a user can read from a table. - - -Row policies make sense only for users with readonly access. If a user can modify a table or copy partitions between tables, it defeats the restrictions of row policies. - - -Syntax: - -```sql --- Multiple names on one table target -CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [, ...] - [ON CLUSTER cluster_name] - ON { [db.]table | db.* } - [IN access_storage_type] - [[FOR SELECT] USING {condition | NONE}] - [AS {PERMISSIVE | RESTRICTIVE}] - [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] - --- One name on multiple table targets -CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name - [ON CLUSTER cluster_name] - ON { [db.]table | db.* } [, ...] - [IN access_storage_type] - [[FOR SELECT] USING {condition | NONE}] - [AS {PERMISSIVE | RESTRICTIVE}] - [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] - --- Mixed packing: each name paired with its own table target -CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] - policy_name ON { [db.]table | db.* } [, policy_name ON { [db.]table | db.* } ...] - [ON CLUSTER cluster_name] - [IN access_storage_type] - [[FOR SELECT] USING {condition | NONE}] - [AS {PERMISSIVE | RESTRICTIVE}] - [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] -``` - -`ParserRowPolicyNames` accepts **three** packing forms (not a full Cartesian product): - -1. **Multiple names, one target** — `pol1, pol2 ON table1` creates each listed name on that single table (or `db.*`). -2. **One name, multiple targets** — `pol1 ON table1, table2` creates the same short name on each listed target. -3. **Mixed pairs** — `p1 ON t1, p2 ON t2` creates each name only on its paired target. - -A multi-name list **cannot** be combined with a multi-table `ON` list in one group: `p1, p2 ON t1, t2` is rejected. After a multi-name group, you also cannot append another comma-separated `name ON target` group in the same statement. - -Optional `ON CLUSTER` applies to the whole statement (one cluster name). ClickHouse does **not** accept a different `ON CLUSTER` per policy name packed into a single create — run separate `CREATE ROW POLICY` statements when policies must be created on different clusters. - -`CREATE ROW POLICY` requires the [CREATE ROW POLICY](/reference/statements/grant#access-management) privilege on the table the policy is created on. `OR REPLACE` throws away an existing policy of the same name, including which roles it applies to, so it additionally requires the [DROP ROW POLICY](/reference/statements/grant#access-management) privilege on that table. The `DROP ROW POLICY` privilege is required whether or not the policy already exists, so the statement cannot be used to find out which policies exist. - -## Multiple names and tables {#multiple-names-and-tables} - -Valid: - -```sql --- Several policy names, one table -CREATE ROW POLICY pol1, pol2, pol3 ON table1 - FOR SELECT USING id = 1 - TO accountant; - --- One policy name, several tables -CREATE ROW POLICY IF NOT EXISTS pol1 ON table1, table2, table3 - FOR SELECT USING id = 1 - TO accountant; - --- Mixed packing: different name per table -CREATE ROW POLICY p4 ON db.table, p5 ON db2.table2 - USING a = b; - --- Same policy on several tables, on a cluster -CREATE ROW POLICY IF NOT EXISTS pol1 ON CLUSTER replicated_cluster ON table1, table2 - FOR SELECT USING id = 1 - TO accountant; -``` - -Invalid: - -```sql --- Multi-name × multi-table in one ON-group (not a Cartesian product) -CREATE ROW POLICY p1, p2 ON t1, t2 - FOR SELECT USING id = 1 - TO accountant; - --- Different clusters per name in one statement -CREATE ROW POLICY pol1 ON CLUSTER cluster1 ON table1, pol2 ON CLUSTER cluster2 ON table2 -``` - -## USING clause {#using-clause} - -Defines a filter condition for a table. A user can only see rows for which the condition is true (evaluates to a non-zero value). This is similar to adding an extra `WHERE` condition to every query the user runs against the table. - -For example, the following policy limits `analyst_role` to rows from the EU: - -```sql -CREATE ROW POLICY region_filter ON db.orders -USING region = 'EU' -TO analyst_role; -``` - -With this policy, `SELECT * FROM db.orders` returns the same rows as `SELECT * FROM db.orders WHERE region = 'EU'` would. - -## TO Clause {#to-clause} - -In the `TO` section you can provide a list of users and roles this policy should work for. For example, `CREATE ROW POLICY ... TO accountant, john@localhost`. - -Keyword `ALL` means all the ClickHouse users, including current user. Keyword `ALL EXCEPT` allows excluding some users from the all users list, for example, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` - -Roles named in the `TO` section, including those after `ALL EXCEPT`, are matched against the current user's enabled roles ([`system.enabled_roles`](/reference/system-tables/enabled_roles)), not against every role granted to the user, so [`SET ROLE`](/reference/statements/set-role) can change which policies apply. - -## AS Clause {#as-clause} - -It's allowed to have more than one policy enabled on the same table for the same user at one time. So we need a way to combine the conditions from multiple policies. - -By default, policies are combined using the boolean `OR` operator. For example, the following policies: - -```sql -CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter -CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio -``` - -enable the user `peter` to see rows with either `b=1` or `c=2`. - -The `AS` clause specifies how policies should be combined with other policies. Policies can be either permissive or restrictive. By default, policies are permissive, which means they are combined using the boolean `OR` operator. - -A policy can be defined as restrictive as an alternative. Restrictive policies are combined using the boolean `AND` operator. - -Here is the general formula: - -```text -row_is_visible = (one or more of the conditions from the permissive policies that apply to the current user and their enabled roles are non-zero) AND - (all of the conditions from the restrictive policies that apply to the current user and their enabled roles are non-zero) -``` - -If no permissive condition applies, the first condition has no effect and only the restrictive policies decide, because `access_control_improvements.users_without_row_policies_can_read_rows` is enabled by default. A user to whom no condition applies therefore sees every row, and `access_control_improvements.throw_on_unmatched_row_policies`, disabled by default, raises an exception instead when the table does have conditions and none of them apply. - -For example, the following policies: - -```sql -CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter -CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio -``` - -enable the user `peter` to see rows only if both `b=1` AND `c=2`. - -Database policies are combined with table policies. - -For example, the following policies: - -```sql -CREATE ROW POLICY pol1 ON mydb.* USING b=1 TO mira, peter -CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio -``` - -enable the user `peter` to see table1 rows only if both `b=1` AND `c=2`, although -any other table in mydb would have only `b=1` policy applied for the user. - -## Tables that read from other tables {#tables-that-read-from-other-tables} - -A row policy filters rows where the data is actually read. An `Alias` table returns the rows of its target table as its own, so the row policies of the target apply to reads through the alias as well, combined with the policies of the alias itself using a logical `AND`. A `Merge` table applies the policies of the tables it reads from. One exception: when a matched table reads remotely, such as a `Distributed` table, the remote server processes the query before the policy is applied, because the policy runs above that table's read rather than at the read. Such a query can fail, when it aggregates without selecting the policy's columns, or return fewer rows than the policy allows, when the remote server applies an `ORDER BY ... LIMIT` to rows the policy would have hidden. Define the policy on the underlying local tables of each remote server instead. - -This does not extend to every table that reads from another table. A `Buffer` table and a materialized view read through their destination or target table do **not** inherit that table's row policies: the policy is written against the target's schema and, for a view with `SQL SECURITY DEFINER`, is evaluated for a different user than the one running the read. Define the policy on the table users actually query in those cases. - -## Distributed and remote-backed tables {#distributed-and-remote-backed-tables} - -A row policy filters rows where the table data is actually read. A table that delegates reading to remote servers, such as a [Distributed](/reference/engines/table-engines/special/distributed) table or a wrapper over one (for example, a materialized view with a `Distributed` target), only ships the query text to the remote servers and cannot apply the policy filter to the remote read. To keep the filter from being silently dropped, queries to such a table by users the policy applies to are rejected with an `ILLEGAL_PREWHERE` error. - -Instead, define the policy on the underlying local tables on each remote server; it is applied there when the shipped query reads them: - -```sql --- Filters reads of local_table on this server, including reads shipped by a Distributed table over it. -CREATE ROW POLICY filter ON mydb.local_table USING a < 1000 TO john; -``` - - -This works while the query is shipped as text, which is the default. With [`serialize_query_plan = 1`](/reference/settings/session-settings/serialize#serialize_query_plan) the initiator ships an already-built read plan instead, and a remote server executing such a plan does not apply its own row policies, so a read of a `Distributed` table over `local_table` returns unfiltered rows. Keep `serialize_query_plan = 0` for users whose row policies must be enforced. See [issue #112891](https://github.com/ClickHouse/ClickHouse/issues/112891). - - -## Join tables {#join-tables} - -A [Join](/reference/engines/table-engines/special/join) table is a prepared hash table that a `JOIN` or `joinGet` reads as is, so its rows cannot be filtered there. A policy on such a table, including a database-wide `ON db.*` policy, filters a plain `SELECT` from the table, but while it applies, `JOIN` and `joinGet` queries against the table fail with `ACCESS_DENIED`. - -## ON CLUSTER Clause {#on-cluster-clause} - -Allows creating row policies on a cluster, see [Distributed DDL](/reference/statements/distributed-ddl). This is also the convenient way to create the policy on the local tables of every server of the cluster. - -## Examples {#examples} - -`CREATE ROW POLICY filter1 ON mydb.mytable USING a<1000 TO accountant, john@localhost` - -`CREATE ROW POLICY filter2 ON mydb.mytable USING a<1000 AND b=5 TO ALL EXCEPT mira` - -`CREATE ROW POLICY filter3 ON mydb.mytable USING 1 TO admin` - -`CREATE ROW POLICY filter4 ON mydb.* USING 1 TO admin` -)DOCS_MD", - .syntax = R"( -CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [, ...] - [ON CLUSTER cluster_name] - ON { [db.]table | db.* } [, ...] - [IN access_storage_type] - [[FOR SELECT] USING {condition | NONE}] - [AS {PERMISSIVE | RESTRICTIVE}] - [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] -)", - .parent = "CREATE", - .related = {"ALTER ROW POLICY", "CREATE MASKING POLICY", "CREATE ROLE", "DROP", "SHOW"}, - }); - - factory.registerStatement("ALTER ROW POLICY", - { - .description = R"DOCS_MD( -Changes row policy. - -Syntax: - -```sql --- Rename: exactly one fully qualified policy (one name on one target). --- RENAME TO may be combined with the same optional alteration clauses as below. -ALTER [ROW] POLICY [IF EXISTS] name - ON { [database.]table | database.* } - RENAME TO new_name - [ON CLUSTER cluster_name] - [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING {condition | NONE}][,...] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] - --- Multiple names on one table target (no RENAME) -ALTER [ROW] POLICY [IF EXISTS] name [, ...] - [ON CLUSTER cluster_name] - ON { [database.]table | database.* } - [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING {condition | NONE}][,...] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] - --- One name on multiple table targets (no RENAME) -ALTER [ROW] POLICY [IF EXISTS] name - [ON CLUSTER cluster_name] - ON { [database.]table | database.* } [, ...] - [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING {condition | NONE}][,...] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] - --- Mixed packing: each name paired with its own table target (no RENAME) -ALTER [ROW] POLICY [IF EXISTS] - name ON { [database.]table | database.* } [, name ON { [database.]table | database.* } ...] - [ON CLUSTER cluster_name] - [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING {condition | NONE}][,...] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] -``` - -`RENAME TO` is only accepted when the statement names **exactly one** policy on **one** table target. Packed multi-name or multi-table lists cannot include `RENAME TO` — rename those policies in separate statements. On that single-policy form, `RENAME TO` can still be combined with other alterations such as `AS`, `USING`, and `TO` in the same statement. - -Without `RENAME TO`, packing matches `ParserRowPolicyNames`: multiple names on **one** target, one name on **multiple** targets, or mixed `name ON target` pairs. Multi-name × multi-table (`p1, p2 ON t1, t2`) is **not** accepted. Optional `ON CLUSTER` applies once to the whole statement; run separate `ALTER ROW POLICY` statements when different clusters are required. - -Examples: - -```sql --- Single-policy rename -ALTER ROW POLICY p1 ON db.table RENAME TO p1_new; - --- Rename plus other alterations on the same single policy -ALTER POLICY old_name ON db.table RENAME TO new_name USING id > 10; - --- Multiple names, one table -ALTER POLICY p1, p2 ON db.table TO ALL; - --- One name, multiple tables -ALTER POLICY p1 ON db.table, db.table2 USING NONE; - --- Mixed targets without rename -ALTER POLICY p1 ON db.table, p2 ON db2.table2 TO ALL; -``` -)DOCS_MD", - .syntax = R"( -ALTER [ROW] POLICY [IF EXISTS] name [, ...] - ON { [database.]table | database.* } [, ...] - [RENAME TO new_name] - [ON CLUSTER cluster_name] - [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING {condition | NONE}][,...] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] -)", - .parent = "ALTER", - .related = {"CREATE ROW POLICY", "ALTER", "SHOW"}, - }); -} - -} diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 2e0ee3dbd986c..67ca7fb5945a6 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -185,7 +184,6 @@ namespace ErrorCodes extern const int PARAMETER_OUT_OF_BOUND; extern const int TOO_MANY_COLUMNS; extern const int UNSUPPORTED_METHOD; - extern const int ACCESS_DENIED; } namespace @@ -3123,22 +3121,12 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode( join_node, planner_context); - /// A prepared storage replaces the right-side plan, and with it the `FilterStep` applying the - /// table's row policy, so such a table is joined as a stream. A `Join` table is a prebuilt - /// hash table read as is, so it cannot be filtered at all. - bool right_table_has_row_policy = !right_join_tree_query_plan.used_row_policies.empty(); - - PreparedJoinStorage prepared_join = tryGetStorageInTableJoin(join_node.getRightTableExpressionNode(), planner_context); - if (prepared_join.storage_join && right_table_has_row_policy) - throw Exception(ErrorCodes::ACCESS_DENIED, - "Cannot join table {} with the Join engine because a row policy is applied on it", - prepared_join.storage_join->getStorageID().getNameForLogs()); - - bool allow_storage_join = !right_table_has_row_policy + PreparedJoinStorage prepared_join; + bool allow_storage_join = right_join_tree_query_plan.used_row_policies.empty() && right_join_tree_query_plan.stage == QueryProcessingStage::FetchColumns && right_join_tree_query_plan.useful_sets.empty(); - if (!allow_storage_join) - prepared_join = {}; + if (allow_storage_join) + prepared_join = tryGetStorageInTableJoin(join_node.getRightTableExpressionNode(), planner_context); if (prepared_join) { bool use_nulls = settings[Setting::join_use_nulls] && isLeftOrFull(join_node.getKind()); diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index cdc7a5c14c2f6..cbbb62afde5a5 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -641,8 +641,6 @@ Default value: `1`. The `Join`-engine tables can't be used in `GLOBAL JOIN` operations. -A [row policy](/reference/statements/create/row-policy) on a `Join`-engine table filters a plain `SELECT` from it, but a `JOIN` or `joinGet` reads the prepared hash table as is and cannot filter its rows, so while a policy applies to the table such queries fail with `ACCESS_DENIED`. - The `Join`-engine allows to specify [join_use_nulls](/reference/settings/session-settings/join#join_use_nulls) setting in the `CREATE TABLE` statement. [SELECT](/reference/statements/select/index) query should have the same `join_use_nulls` value. ## Usage examples {#example} diff --git a/tests/queries/0_stateless/05233_direct_join_row_policy.reference b/tests/queries/0_stateless/05233_direct_join_row_policy.reference index 523709ccc7546..f196664e9f2c7 100644 --- a/tests/queries/0_stateless/05233_direct_join_row_policy.reference +++ b/tests/queries/0_stateless/05233_direct_join_row_policy.reference @@ -4,9 +4,6 @@ -- inner 1 public public-1 3 public public-3 --- inner, policy column not selected -1 public-1 -3 public-3 -- left 1 1 public-1 2 0 @@ -17,27 +14,22 @@ 2 \N \N 3 3 public-3 5 \N \N --- left any -1 public-1 -2 -3 public-3 -5 -- left semi 1 3 -- left anti 2 5 --- key space enumeration -1 public-1 -3 public-3 --- using +-- join table 1 public-1 +2 3 public-3 --- no direct join under a row policy -0 --- direct join without a policy +5 +-- without a policy 1 public public-1 2 hidden hidden-2 3 public public-3 -Algorithm: DirectKeyValueJoin +1 public-1 +2 hidden-2 +3 public-3 +5 diff --git a/tests/queries/0_stateless/05233_direct_join_row_policy.sql b/tests/queries/0_stateless/05233_direct_join_row_policy.sql index e2a073ca13a26..38db3f227d2d7 100644 --- a/tests/queries/0_stateless/05233_direct_join_row_policy.sql +++ b/tests/queries/0_stateless/05233_direct_join_row_policy.sql @@ -1,18 +1,24 @@ -- Tags: use-rocksdb +SET enable_analyzer = 0; +SET join_algorithm = 'direct,hash'; +SET join_use_nulls = 0; + DROP TABLE IF EXISTS kv_rls; +DROP TABLE IF EXISTS join_rls; DROP TABLE IF EXISTS probe_rls; CREATE TABLE kv_rls (key UInt64, tenant String, secret String) ENGINE = EmbeddedRocksDB PRIMARY KEY key; INSERT INTO kv_rls VALUES (1, 'public', 'public-1'), (2, 'hidden', 'hidden-2'), (3, 'public', 'public-3'), (4, 'hidden', 'hidden-4'); +CREATE TABLE join_rls (key UInt64, tenant String, secret String) ENGINE = Join(ANY, LEFT, key); +INSERT INTO join_rls VALUES (1, 'public', 'public-1'), (2, 'hidden', 'hidden-2'), (3, 'public', 'public-3'), (4, 'hidden', 'hidden-4'); + CREATE TABLE probe_rls (key UInt64) ENGINE = TinyLog; INSERT INTO probe_rls VALUES (1), (2), (3), (5); CREATE ROW POLICY kv_rls_public ON kv_rls FOR SELECT USING tenant = 'public' TO CURRENT_USER; - -SET join_algorithm = 'direct,hash'; -SET join_use_nulls = 0; +CREATE ROW POLICY join_rls_public ON join_rls FOR SELECT USING tenant = 'public' TO CURRENT_USER; SELECT '-- plain select'; SELECT key, tenant, secret FROM kv_rls ORDER BY key; @@ -20,44 +26,28 @@ SELECT key, tenant, secret FROM kv_rls ORDER BY key; SELECT '-- inner'; SELECT p.key, kv.tenant, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; -SELECT '-- inner, policy column not selected'; -SELECT p.key, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; - SELECT '-- left'; SELECT p.key, kv.key, kv.secret FROM probe_rls AS p LEFT JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; SELECT '-- left, join_use_nulls'; SELECT p.key, kv.key, kv.secret FROM probe_rls AS p LEFT JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key SETTINGS join_use_nulls = 1; -SELECT '-- left any'; -SELECT p.key, kv.secret FROM probe_rls AS p LEFT ANY JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; - SELECT '-- left semi'; SELECT p.key FROM probe_rls AS p LEFT SEMI JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; SELECT '-- left anti'; SELECT p.key FROM probe_rls AS p LEFT ANTI JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; -SELECT '-- key space enumeration'; -SELECT kv.key, kv.secret FROM numbers(10) AS n INNER JOIN kv_rls AS kv ON kv.key = n.number ORDER BY kv.key; - -SELECT '-- using'; -SELECT key, secret FROM probe_rls INNER JOIN kv_rls USING (key) ORDER BY key; - -SELECT '-- no direct join under a row policy'; -SELECT count() FROM ( - EXPLAIN actions = 1 - SELECT p.key, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key -) WHERE explain LIKE '%DirectKeyValueJoin%'; +SELECT '-- join table'; +SELECT p.key, j.secret FROM probe_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; DROP ROW POLICY kv_rls_public ON kv_rls; +DROP ROW POLICY join_rls_public ON join_rls; -SELECT '-- direct join without a policy'; +SELECT '-- without a policy'; SELECT p.key, kv.tenant, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; -SELECT extract(explain, 'Algorithm: \\w+') FROM ( - EXPLAIN actions = 1 - SELECT p.key, kv.secret FROM probe_rls AS p INNER JOIN kv_rls AS kv ON kv.key = p.key -) WHERE explain LIKE '%Algorithm:%'; +SELECT p.key, j.secret FROM probe_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; DROP TABLE kv_rls; +DROP TABLE join_rls; DROP TABLE probe_rls; diff --git a/tests/queries/0_stateless/05234_row_policy_storage_join.reference b/tests/queries/0_stateless/05234_row_policy_storage_join.reference deleted file mode 100644 index 2ac892e8f5052..0000000000000 --- a/tests/queries/0_stateless/05234_row_policy_storage_join.reference +++ /dev/null @@ -1,12 +0,0 @@ --- without a policy -1 a -2 b -3 -b \N --- with a policy -1 a --- after dropping the policy -1 a -2 b -3 -b \N diff --git a/tests/queries/0_stateless/05234_row_policy_storage_join.sql b/tests/queries/0_stateless/05234_row_policy_storage_join.sql deleted file mode 100644 index 91d6f948686d2..0000000000000 --- a/tests/queries/0_stateless/05234_row_policy_storage_join.sql +++ /dev/null @@ -1,31 +0,0 @@ -DROP TABLE IF EXISTS join_rls; -DROP TABLE IF EXISTS probe_join_rls; -DROP ROW POLICY IF EXISTS join_rls_policy ON join_rls; - -CREATE TABLE join_rls (key UInt64, value String) ENGINE = Join(ANY, LEFT, key); -INSERT INTO join_rls VALUES (1, 'a'), (2, 'b'); - -CREATE TABLE probe_join_rls (key UInt64) ENGINE = TinyLog; -INSERT INTO probe_join_rls VALUES (1), (2), (3); - -SELECT '-- without a policy'; -SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; -SELECT joinGet(join_rls, 'value', toUInt64(2)), joinGetOrNull(join_rls, 'value', toUInt64(3)); - -CREATE ROW POLICY join_rls_policy ON join_rls FOR SELECT USING value = 'a' TO CURRENT_USER; - -SELECT '-- with a policy'; -SELECT key, value FROM join_rls ORDER BY key; -SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; -- { serverError ACCESS_DENIED } -SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key SETTINGS join_algorithm = 'hash'; -- { serverError ACCESS_DENIED } -SELECT joinGet(join_rls, 'value', toUInt64(2)); -- { serverError ACCESS_DENIED } -SELECT joinGetOrNull(join_rls, 'value', toUInt64(2)); -- { serverError ACCESS_DENIED } - -DROP ROW POLICY join_rls_policy ON join_rls; - -SELECT '-- after dropping the policy'; -SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; -SELECT joinGet(join_rls, 'value', toUInt64(2)), joinGetOrNull(join_rls, 'value', toUInt64(3)); - -DROP TABLE join_rls; -DROP TABLE probe_join_rls; From 2d4cececd7280f8578672149aa471f95b595633b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 19:02:35 +0000 Subject: [PATCH 150/296] Update autogenerated version to 26.8.15.10 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 4e3df68592efd..9789a0bd275ee 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54527) +SET(VERSION_REVISION 54528) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 15) -SET(VERSION_GITHASH f1d4a0d36450556c9e3c4f8b3244770b20345516) -SET(VERSION_DESCRIBE v26.8.15.1-lts) -SET(VERSION_STRING 26.8.15.1) +SET(VERSION_PATCH 16) +SET(VERSION_GITHASH 3c6755634c252e1031f2db8bfdfed39f86038237) +SET(VERSION_DESCRIBE v26.8.16.1-lts) +SET(VERSION_STRING 26.8.16.1) # end of autochange From f358962f416eea773e8bb5d39a6220270321c59d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 22:31:13 +0000 Subject: [PATCH 151/296] Backport #120220 to 26.8: Look through lossless conversions of the indexed column and of constants in the text index --- .../optimizeDirectReadFromTextIndex.cpp | 6 +- .../MergeTreeIndexBloomFilterText.cpp | 13 +- .../MergeTree/MergeTreeIndexConditionText.cpp | 149 ++++++++++++----- .../MergeTree/MergeTreeIndexConditionText.h | 6 +- src/Storages/MergeTree/RPNBuilder.cpp | 33 ++-- src/Storages/MergeTree/RPNBuilder.h | 13 +- ...ap_lowcardinality_subcolumn_cast.reference | 80 +++++++++ ...ndex_map_lowcardinality_subcolumn_cast.sql | 156 ++++++++++++++++++ ...t_index_map_key_wrapped_constant.reference | 30 ++++ ...20_text_index_map_key_wrapped_constant.sql | 66 ++++++++ ...ap_absent_key_fixedstring_needle.reference | 14 ++ ...enbf_map_absent_key_fixedstring_needle.sql | 40 +++++ ...ow_cardinality_nullable_constant.reference | 20 +++ ...alue_low_cardinality_nullable_constant.sql | 40 +++++ ...sor_lossless_conversion_haystack.reference | 40 +++++ ...processor_lossless_conversion_haystack.sql | 73 ++++++++ 16 files changed, 709 insertions(+), 70 deletions(-) create mode 100644 tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference create mode 100644 tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql create mode 100644 tests/queries/0_stateless/05220_text_index_map_key_wrapped_constant.reference create mode 100644 tests/queries/0_stateless/05220_text_index_map_key_wrapped_constant.sql create mode 100644 tests/queries/0_stateless/05221_tokenbf_map_absent_key_fixedstring_needle.reference create mode 100644 tests/queries/0_stateless/05221_tokenbf_map_absent_key_fixedstring_needle.sql create mode 100644 tests/queries/0_stateless/05241_bloom_filter_map_default_value_low_cardinality_nullable_constant.reference create mode 100644 tests/queries/0_stateless/05241_bloom_filter_map_default_value_low_cardinality_nullable_constant.sql create mode 100644 tests/queries/0_stateless/05242_text_index_preprocessor_lossless_conversion_haystack.reference create mode 100644 tests/queries/0_stateless/05242_text_index_preprocessor_lossless_conversion_haystack.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp index a1f99567b6565..765a0ffde1dca 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp @@ -703,11 +703,15 @@ class TextIndexDAGReplacer const auto & preprocessor_dag = preprocessor->getOriginalActionsDAG(); chassert(preprocessor_dag.getOutputs().size() == 1); const auto & preprocessor_output = preprocessor_dag.getOutputs().front(); - auto haystack_name = getNameWithoutAliases(arg_haystack); + /// The index was analyzed on the expression under lossless conversions, e.g. `s` in `hasToken(toNullable(s), 'Foo')`. + const auto * haystack = unwrapLosslessConversion(arg_haystack); + auto haystack_name = getNameWithoutAliases(haystack); /// Check that preprocessor contains current expression as its argument. if (hasSubexpression(preprocessor_output, haystack_name)) { + new_children[0] = haystack; + if (apply_postprocessor) { preprocessor_source_ast = preprocessor->getExpressionAST(new_children[0]->result_name); diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp index e8e299d1a6995..4a257d3e8c56f 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp @@ -464,6 +464,15 @@ bool MergeTreeConditionBloomFilterText::traverseTreeEquals( Field const_value = value_field; + /// An absent map key reads ''. `String = FixedString(N)` ignores the zero padding of the constant, while + /// the default value of `FixedString(N)` is reported as '', so the constant is compared without its padding. + Field value_without_padding = value_field; + if (isFixedString(value_type) && value_field.getType() == Field::Types::String) + { + auto & value = value_without_padding.safeGet(); + value.resize(value.find_last_not_of('\0') + 1); + } + const auto column_name = key_node.getColumnName(); auto key_index = getKeyIndex(column_name); const auto map_key_index = getKeyIndex(fmt::format("mapKeys({})", column_name)); @@ -483,7 +492,7 @@ bool MergeTreeConditionBloomFilterText::traverseTreeEquals( * We cannot skip keys that does not exist in map if comparison is with default type value because * that way we skip necessary granules where map key does not exist. */ - if (value_field == value_type->getDefault()) + if (value_without_padding == value_type->getDefault()) return false; auto first_argument = key_function_node.getArgumentAt(0); @@ -526,7 +535,7 @@ bool MergeTreeConditionBloomFilterText::traverseTreeEquals( /// Same as arrayElement: skip when comparing with default value because /// the subcolumn returns default for keys that don't exist in the map. - if (value_field == value_type->getDefault()) + if (value_without_padding == value_type->getDefault()) return false; if (const auto map_keys_index = getKeyIndex(fmt::format("mapKeys({})", map_column_name))) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index 1640e2b731c19..331256a048a42 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -898,6 +898,83 @@ static void validateRegexpPatterns(const Array & patterns, const Settings & sett #endif } +namespace +{ + +/// Whether converting a value of type `from` to type `to` never changes it and never throws. +/// `LowCardinality` may be added or dropped and `Nullable` may be added, at any depth of `Array`. +/// `Nullable` cannot be dropped, because it may throw on NULL. +bool isLosslessConversion(const DataTypePtr & from, const DataTypePtr & to) +{ + auto from_type = removeLowCardinality(from); + auto to_type = removeLowCardinality(to); + + if (to_type->isNullable()) + { + from_type = removeNullable(from_type); + to_type = removeNullable(to_type); + } + else if (from_type->isNullable()) + { + return false; + } + + if (from_type->equals(*to_type)) + return true; + + const auto * from_array = typeid_cast(from_type.get()); + const auto * to_array = typeid_cast(to_type.get()); + return from_array && to_array && isLosslessConversion(from_array->getNestedType(), to_array->getNestedType()); +} + +/// Whether the node is `CAST`, `_CAST`, `toNullable` or `toLowCardinality` with a lossless conversion (see above). +bool isLosslessConversionFunction(const ActionsDAG::Node & node) +{ + if (node.type != ActionsDAG::ActionType::FUNCTION || !node.function_base) + return false; + + const auto function_name = node.function_base->getName(); + const size_t arguments_size = node.children.size(); + + const bool is_cast = (function_name == "CAST" || function_name == "_CAST") && arguments_size == 2; + const bool is_wrapper = (function_name == "toNullable" || function_name == "toLowCardinality") && arguments_size == 1; + + if (!is_cast && !is_wrapper) + return false; + + return isLosslessConversion(node.children.front()->result_type, node.result_type); +} + +/// Strips lossless conversions from the node (see above). +RPNBuilderTreeNode unwrapLosslessConversion(const RPNBuilderTreeNode & node) +{ + if (!node.isFunction()) + return node; + + /// Only the DAG form carries the types; the AST form is left as is. + const auto function = node.toFunctionNode(); + const auto * function_dag_node = function.getDAGNode(); + + if (!function_dag_node || !isLosslessConversionFunction(*function_dag_node)) + return node; + + return unwrapLosslessConversion(function.getArgumentAt(0)); +} + +} + +const ActionsDAG::Node * unwrapLosslessConversion(const ActionsDAG::Node * node) +{ + const auto * node_without_alias = node; + while (node_without_alias->type == ActionsDAG::ActionType::ALIAS) + node_without_alias = node_without_alias->children.front(); + + if (!isLosslessConversionFunction(*node_without_alias)) + return node; + + return unwrapLosslessConversion(node_without_alias->children.front()); +} + /// The value an absent map key reads: `''`, or all NUL when the value type is `FixedString`. /// `mapValues` stores neither. static bool isMapValueDefault(std::string_view value, const Block & header) @@ -913,7 +990,7 @@ static bool isMapValueDefault(std::string_view value, const Block & header) bool MergeTreeIndexConditionText::traverseFunctionNode( const RPNBuilderFunctionTreeNode & function_node, - const RPNBuilderTreeNode & index_column_node, + const RPNBuilderTreeNode & argument_node, DataTypePtr value_type, Field value_field, RPNElement & out) const @@ -921,6 +998,9 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( const String function_name = function_node.getFunctionName(); auto direct_read_mode = getDirectReadMode(function_name); + /// The index knows the expression under the conversion, e.g. `m.key_` in `equals(_CAST(m.key_, 'String'), 'value')`. + const auto index_column_node = unwrapLosslessConversion(argument_node); + auto index_column_name = index_column_node.getColumnName(); bool has_index_column = hasIndexForColumn(index_column_name); bool has_map_keys_column = hasIndexForColumn(fmt::format("mapKeys({})", index_column_name)); @@ -1157,50 +1237,30 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( } if (function_name == "hasToken" || function_name == "hasTokenOrNull") { - // hasToken and hasTokenOrNull are legacy functions which assume splitByNonAlpha as - /// tokenizer. The text index can answer it only correctly if this is the index tokenizer. - /// In all other cases, bypass the index. + /// `hasToken` splits by non-alphanumeric characters, so only an index with the same tokenizer can answer it. if (tokenizer->getType() != ITokenizer::Type::SplitByNonAlpha) return false; - /// Unlike hasToken, hasTokenOrNull is never rewritten to direct-read, so the pre/postprocessor - /// is also not applied to its needle. Using the index here (where stringToTokens does apply them, - /// e.g. mapping a dropped token to the empty sentinel that prunes every granule) would disagree - /// with the scan result. Bail out so the index is not used for hasTokenOrNull when a - /// pre/postprocessor is configured; the plain index path is unaffected. + /// `hasTokenOrNull` is never rewritten to a direct read, so its needle never goes through the + /// pre/postprocessor, while `stringToTokens` applies them. The index would disagree with the scan. if (function_name == "hasTokenOrNull" && (has_preprocessor || has_postprocessor)) return false; - /// A needle containing a token separator is invalid for `hasToken` and the brute-force scan raises - /// BAD_ARGUMENTS for this. hasToken uses Exact direct read, so the index would tokenize the needle and - /// silently replace the predicate (or prune the granule that would have thrown), hiding the exception. - /// Therefore bypass the index and do a brute-force scan. hasTokenOrNull is not affected: it returns NULL - /// - /// A separator is any ASCII non-alphanumeric character. - if (function_name == "hasToken" - && std::ranges::any_of(value_field.safeGet(), [](unsigned char c) { return isASCII(c) && !isAlphaNumericASCII(c); })) + /// The scan raises BAD_ARGUMENTS for a needle with a separator, and the exact direct read of `hasToken` + /// would replace the predicate and hide it. `hasTokenOrNull` returns NULL for such a needle instead. + if (function_name == "hasToken" && std::ranges::any_of(value_field.safeGet(), isTokenSeparator)) return false; auto tokens = stringToTokens(value_field); if (tokens.empty()) { + /// A needle without a word character is invalid: leave it to the scan, which raises or returns NULL. + /// Otherwise the pre/postprocessor dropped the needle (e.g. a stop word), so it is not in the index: + /// push the empty sentinel to prune every granule. const String & string_needle = value_field.safeGet(); - if (!string_needle.empty()) - { - /// hasToken uses splitByNonAlpha as its tokenizer, so: - /// - A needle without any word character (alphanumeric or non-ASCII) is invalid. - /// - Bypass the index in that case so the row-level evaluation throws BAD_ARGUMENTS (or returns NULL for hasTokenOrNull) - /// -- Consistent with the no-index behaviour. - /// If the needle does contain word characters (e.g. "abc" with ngrams(4)): - /// - It is valid but too short for the index's tokenizer: - /// -- Fall through to push "" so all granules are pruned and the query returns 0 rows. - /// If the postprocessor filters the needle (e.g. stop-word): - /// -- The needle is not in the index; push "" sentinel so the condition evaluates to false. - if (std::ranges::none_of(string_needle, [](unsigned char c) { return !isASCII(c) || isAlphaNumericASCII(c); })) - return false; - } - /// - If the needle does contain word characters (e.g. "abc" with ngrams(4)), it is valid but too short for the index's tokenizer: - /// Fall through but push "" so all granules are pruned and the query returns 0 rows. + if (!string_needle.empty() && std::ranges::all_of(string_needle, isTokenSeparator)) + return false; + tokens.push_back(""); } @@ -1523,9 +1583,13 @@ bool MergeTreeIndexConditionText::traverseMapElementKeyNode(const RPNBuilderFunc /// It can be an arbitrary function that returns 0 for the default value of the map value type. /// It is true because `arrayElement` (and the equivalent subcolumn access) returns default value if key doesn't exist in the map, /// therefore we can use index to skip granules and use direct read as a hint for the original condition. + /// The result may be `Nullable` or `LowCardinality`, e.g. for a `Nullable` key: NULL reads as false below, as in WHERE. const auto * dag_node = function_node.getDAGNode(); - if (!dag_node || !dag_node->function_base || !dag_node->isDeterministic() || !WhichDataType(dag_node->result_type).isUInt8()) + if (!dag_node + || !dag_node->function_base + || !dag_node->isDeterministic() + || !WhichDataType(removeLowCardinalityAndNullable(dag_node->result_type)).isUInt8()) return false; auto subdag = ActionsDAG::cloneSubDAG({dag_node}, true); @@ -1565,10 +1629,14 @@ bool MergeTreeIndexConditionText::traverseMapElementKeyNode(const RPNBuilderFunc if (map_argument->type != ActionsDAG::ActionType::INPUT || map_argument->result_name != required_column.name) return false; - if (const_key_argument->type != ActionsDAG::ActionType::COLUMN || !isStringOrFixedString(const_key_argument->result_type)) + /// A NULL key is declined: `arrayElement` returns NULL for it, not the default value. + Field key_field; + DataTypePtr key_type; + if (!RPNBuilderTreeNode(const_key_argument, function_node.getTreeContext()).tryGetConstant(key_field, key_type) + || key_field.getType() != Field::Types::String) return false; - key_const_value = std::string{const_key_argument->column->getDataAt(0)}; + key_const_value = key_field.safeGet(); } else { @@ -1672,8 +1740,10 @@ bool MergeTreeIndexConditionText::traverseJSONSubcolumnKeyNode( /// Similar to traverseMapElementKeyNode but for JSON subcolumns. const auto * dag_node = function_node.getDAGNode(); - if (!dag_node || !dag_node->function_base || !dag_node->isDeterministic() - || !WhichDataType(removeNullable(dag_node->result_type)).isUInt8()) + if (!dag_node + || !dag_node->function_base + || !dag_node->isDeterministic() + || !WhichDataType(removeLowCardinalityAndNullable(dag_node->result_type)).isUInt8()) return false; auto subdag = ActionsDAG::cloneSubDAG({dag_node}, true); @@ -1741,8 +1811,9 @@ bool MergeTreeIndexConditionText::tryPrepareSetForTextSearch( /// `m['key']` answered by a `mapValues(m)` index: an absent key reads the value type's default. bool has_index_for_map_element_value = false; - auto has_index = [&](const RPNBuilderTreeNode & node) + auto has_index = [&](const RPNBuilderTreeNode & argument) { + const auto node = unwrapLosslessConversion(argument); if (hasIndexForMapElementValue(node)) { has_index_for_map_element_value = true; diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.h b/src/Storages/MergeTree/MergeTreeIndexConditionText.h index 5ac4da009fc11..49f37d4ece26e 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.h +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.h @@ -163,7 +163,7 @@ class MergeTreeIndexConditionText final : public IMergeTreeIndexCondition, publi bool traverseFunctionNode( const RPNBuilderFunctionTreeNode & function_node, - const RPNBuilderTreeNode & index_column_node, + const RPNBuilderTreeNode & argument_node, DataTypePtr value_type, Field value_field, RPNElement & out) const; @@ -237,4 +237,8 @@ class MergeTreeIndexConditionText final : public IMergeTreeIndexCondition, publi static constexpr std::string_view TEXT_INDEX_VIRTUAL_COLUMN_PREFIX = "__text_index_"; bool isTextIndexVirtualColumn(const String & column_name); +/// Strips `CAST`, `_CAST`, `toNullable` and `toLowCardinality` from the node while the conversion never +/// changes the value and never throws. The index is analyzed on the expression under such conversions. +const ActionsDAG::Node * unwrapLosslessConversion(const ActionsDAG::Node * node); + } diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 47ed74cc8ff1c..dfacb0126fc14 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -248,14 +248,14 @@ const Settings & RPNBuilderTreeContext::getSettings() const return query_context->getSettingsRef(); } -RPNBuilderTreeNode::RPNBuilderTreeNode(const ActionsDAG::Node * dag_node_, RPNBuilderTreeContext & tree_context_) +RPNBuilderTreeNode::RPNBuilderTreeNode(const ActionsDAG::Node * dag_node_, const RPNBuilderTreeContext & tree_context_) : dag_node(dag_node_) , tree_context(tree_context_) { chassert(dag_node); } -RPNBuilderTreeNode::RPNBuilderTreeNode(const IAST * ast_node_, RPNBuilderTreeContext & tree_context_) +RPNBuilderTreeNode::RPNBuilderTreeNode(const IAST * ast_node_, const RPNBuilderTreeContext & tree_context_) : ast_node(ast_node_) , tree_context(tree_context_) { @@ -371,6 +371,16 @@ ColumnWithTypeAndName RPNBuilderTreeNode::getConstantColumn() const return result; } +/// A `Field` is the plain value of a constant: `LowCardinality` is only an encoding of the column, +/// and a value that is not NULL has no `Nullable` type. +static DataTypePtr getTypeOfConstantValue(const Field & value, const DataTypePtr & type) +{ + auto value_type = removeLowCardinality(type); + if (!value.isNull()) + value_type = removeNullable(value_type); + return value_type; +} + bool RPNBuilderTreeNode::tryGetConstant(Field & output_value, DataTypePtr & output_type) const { if (ast_node) @@ -389,12 +399,7 @@ bool RPNBuilderTreeNode::tryGetConstant(Field & output_value, DataTypePtr & outp /// Simple literal output_value = literal->value; - output_type = block_with_constants.getByName(column_name).type; - - /// If constant is not Null, we can assume it's type is not Nullable as well. - if (!output_value.isNull()) - output_type = removeNullable(output_type); - + output_type = getTypeOfConstantValue(output_value, block_with_constants.getByName(column_name).type); return true; } if (block_with_constants.has(column_name) && isColumnConst(*block_with_constants.getByName(column_name).column)) @@ -402,11 +407,7 @@ bool RPNBuilderTreeNode::tryGetConstant(Field & output_value, DataTypePtr & outp /// An expression which is dependent on constants only const auto & constant_column = block_with_constants.getByName(column_name); output_value = (*constant_column.column)[0]; - output_type = constant_column.type; - - if (!output_value.isNull()) - output_type = removeNullable(output_type); - + output_type = getTypeOfConstantValue(output_value, constant_column.type); return true; } } @@ -417,11 +418,7 @@ bool RPNBuilderTreeNode::tryGetConstant(Field & output_value, DataTypePtr & outp if (node_without_alias->column) { output_value = node_without_alias->column->getField(); - output_type = node_without_alias->result_type; - - if (!output_value.isNull()) - output_type = removeNullable(output_type); - + output_type = getTypeOfConstantValue(output_value, node_without_alias->result_type); return true; } } diff --git a/src/Storages/MergeTree/RPNBuilder.h b/src/Storages/MergeTree/RPNBuilder.h index a785294769b27..1a38ec1e3496c 100644 --- a/src/Storages/MergeTree/RPNBuilder.h +++ b/src/Storages/MergeTree/RPNBuilder.h @@ -76,10 +76,10 @@ class RPNBuilderTreeNode { public: /// Construct RPNBuilderTreeNode with non null dag node and tree context - explicit RPNBuilderTreeNode(const ActionsDAG::Node * dag_node_, RPNBuilderTreeContext & tree_context_); + explicit RPNBuilderTreeNode(const ActionsDAG::Node * dag_node_, const RPNBuilderTreeContext & tree_context_); /// Construct RPNBuilderTreeNode with non null ast node and tree context - explicit RPNBuilderTreeNode(const IAST * ast_node_, RPNBuilderTreeContext & tree_context_); + explicit RPNBuilderTreeNode(const IAST * ast_node_, const RPNBuilderTreeContext & tree_context_); /// Get AST node const IAST * getASTNode() const { return ast_node; } @@ -113,6 +113,7 @@ class RPNBuilderTreeNode /** Try get constant from node. If node is constant returns true, and constant value and constant type output parameters are set. * Otherwise false is returned. + * The output type is the type of the value: `LowCardinality` is removed, and `Nullable` is removed when the value is not NULL. */ bool tryGetConstant(Field & output_value, DataTypePtr & output_type) const; @@ -141,16 +142,10 @@ class RPNBuilderTreeNode return tree_context; } - /// Get tree context - RPNBuilderTreeContext & getTreeContext() - { - return tree_context; - } - protected: const IAST * ast_node = nullptr; const ActionsDAG::Node * dag_node = nullptr; - RPNBuilderTreeContext & tree_context; + const RPNBuilderTreeContext & tree_context; }; /** RPNBuilderFunctionTreeNode is wrapper around RPNBuilderTreeNode with function type. diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference new file mode 100644 index 0000000000000..97d96022f0203 --- /dev/null +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference @@ -0,0 +1,80 @@ +-- the analyzer wraps the subcolumn into a cast to String +1 +-- mapValues: the index prunes granules through the cast +Granules: 2/2 +Name: idx +Granules: 1/2 +Granules: 2/2 +Name: idx +Granules: 1/2 +-- mapValues: results match the scan +idx 1 +idx 3 +scan 1 +scan 3 +idx 1 +idx 2 +scan 1 +scan 2 +idx 3 +scan 3 +-- mapValues is only a hint: a value that belongs to another key does not match +idx 0 +-- mapValues on LowCardinality(Nullable) values: the cast to Nullable(String) is looked through +Granules: 2/2 +Name: idx +Granules: 1/2 +idx 3 +scan 3 +-- String column: conversions that only add Nullable or LowCardinality are looked through +CAST Nullable Granules: 2/2 +CAST Nullable Name: idx +CAST Nullable Granules: 1/2 +CAST LowCardinality Granules: 2/2 +CAST LowCardinality Name: idx +CAST LowCardinality Granules: 1/2 +toNullable Granules: 2/2 +toNullable Name: idx +toNullable Granules: 1/2 +toLowCardinality Granules: 2/2 +toLowCardinality Name: idx +toLowCardinality Granules: 1/2 +nested Granules: 2/2 +nested Name: idx +nested Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 +not idx 3 +not idx 4 +not scan 3 +not scan 4 +in idx 1 +in idx 3 +in scan 1 +in scan 3 +-- join_use_nulls pushes the filter down to the outer side as toNullable(column) +1 +3 +-- Nullable column: adding LowCardinality is looked through +Granules: 2/2 +Name: idx +Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 +-- Nullable column: dropping Nullable is not looked through, the cast still throws on the NULL row +0 +-- Array column: a cast that only wraps the elements is looked through +Array(Nullable) Granules: 2/2 +Array(Nullable) Name: idx +Array(Nullable) Granules: 1/2 +Array(LowCardinality) Granules: 2/2 +Array(LowCardinality) Name: idx +Array(LowCardinality) Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql new file mode 100644 index 0000000000000..9c2f4b4409a6e --- /dev/null +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql @@ -0,0 +1,156 @@ +-- Tags: no-parallel-replicas +-- Tag no-parallel-replicas -- direct read is not compatible with parallel replicas + +-- `arrayElement` on a `Map(K, LowCardinality(V))` returns `V`, while the subcolumn `m.key_` is `LowCardinality(V)`. +-- Hence `optimize_functions_to_subcolumns` rewrites `m['key'] = 'value'` into `_CAST(m.key_, 'V') = 'value'`, +-- and the text index has to look through the cast: for an index on `mapValues(m)`. +-- The same applies to every conversion that cannot change a value or throw: adding or dropping `LowCardinality`, +-- adding `Nullable` (also as `toNullable`, which `join_use_nulls` emits in pushed-down filters), at any depth of `Array`. + +SET explain_query_plan_default = 'legacy'; +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +SET use_skip_indexes = 1; +SET query_plan_direct_read_from_text_index = 1; + +DROP TABLE IF EXISTS tab_values; + +CREATE TABLE tab_values +( + id UInt32, + m Map(String, LowCardinality(String)), + INDEX idx mapValues(m) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_values VALUES (1, {'level':'error','msg':'disk is full'}), (2, {'level':'warn','msg':'disk is slow'}), (3, {'level':'error','msg':'network is down'}), (4, {}); + +SELECT '-- the analyzer wraps the subcolumn into a cast to String'; +SELECT count() FROM (EXPLAIN QUERY TREE run_passes = 1 SELECT id FROM tab_values WHERE m['level'] = 'warn') WHERE explain LIKE '%function_name: _CAST%'; + +SELECT '-- mapValues: the index prunes granules through the cast'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values WHERE m['level'] = 'warn') WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values WHERE hasToken(m['msg'], 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; + +SELECT '-- mapValues: results match the scan'; +SELECT 'idx', id FROM tab_values WHERE m['level'] = 'error' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE m['level'] = 'error' ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'idx', id FROM tab_values WHERE hasToken(m['msg'], 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE hasToken(m['msg'], 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'idx', id FROM tab_values WHERE m['msg'] LIKE '%down%' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE m['msg'] LIKE '%down%' ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT '-- mapValues is only a hint: a value that belongs to another key does not match'; +SELECT 'idx', count() FROM tab_values WHERE m['level'] = 'disk' SETTINGS force_data_skipping_indices = 'idx'; + +DROP TABLE tab_values; + +-- A `mapValues` index on `LowCardinality(Nullable(String))` values: the analyzer casts to `Nullable(String)`. +DROP TABLE IF EXISTS tab_values_n; + +CREATE TABLE tab_values_n +( + id UInt32, + m Map(String, LowCardinality(Nullable(String))), + INDEX idx mapValues(m) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_values_n VALUES (1, {'level':'error'}), (2, {'level':'error'}), (3, {'level':'warn'}), (4, {'level':NULL}); + +SELECT '-- mapValues on LowCardinality(Nullable) values: the cast to Nullable(String) is looked through'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values_n WHERE m['level'] = 'warn') WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_values_n WHERE m['level'] = 'warn' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values_n WHERE m['level'] = 'warn' ORDER BY id SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_values_n; + +-- Plain columns with explicit conversions. +DROP TABLE IF EXISTS tab_s; + +CREATE TABLE tab_s +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_s VALUES (1, 'disk is full'), (2, 'disk is slow'), (3, 'network is down'), (4, ''); + +SELECT '-- String column: conversions that only add Nullable or LowCardinality are looked through'; +SELECT 'CAST Nullable', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(CAST(s, 'Nullable(String)'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'CAST LowCardinality', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(CAST(s, 'LowCardinality(String)'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'toNullable', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toNullable(s), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'toLowCardinality', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toLowCardinality(s), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'nested', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toNullable(toLowCardinality(s)), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_s WHERE hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_s WHERE hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'not idx', id FROM tab_s WHERE NOT hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'not scan', id FROM tab_s WHERE NOT hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'in idx', id FROM tab_s WHERE toNullable(s) IN ('disk is full', 'network is down') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'in scan', id FROM tab_s WHERE toNullable(s) IN ('disk is full', 'network is down') ORDER BY id SETTINGS use_skip_indexes = 0; + +SELECT '-- join_use_nulls pushes the filter down to the outer side as toNullable(column)'; +DROP TABLE IF EXISTS tab_ids; +CREATE TABLE tab_ids (id UInt32) ENGINE = MergeTree ORDER BY id; +INSERT INTO tab_ids SELECT number FROM numbers(1, 4); +-- `query_plan_convert_outer_join_to_inner_join` is pinned because the filter is pushed below the join only +-- when the LEFT JOIN becomes an INNER one; otherwise it stays above the join and no index is consulted. +SELECT count() FROM (EXPLAIN indexes = 1 SELECT l.id FROM tab_ids AS l LEFT JOIN tab_s AS r ON l.id = r.id WHERE hasToken(r.s, 'network') SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1) WHERE explain LIKE '%Name: idx%'; +SELECT l.id FROM tab_ids AS l LEFT JOIN tab_s AS r ON l.id = r.id WHERE hasToken(r.s, 'network') ORDER BY l.id SETTINGS join_use_nulls = 1; +DROP TABLE tab_ids; + +DROP TABLE tab_s; + +DROP TABLE IF EXISTS tab_ns; + +CREATE TABLE tab_ns +( + id UInt32, + s Nullable(String), + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_ns VALUES (1, 'disk is full'), (2, 'disk is slow'), (3, 'network is down'), (4, NULL); + +SELECT '-- Nullable column: adding LowCardinality is looked through'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; + +SELECT '-- Nullable column: dropping Nullable is not looked through, the cast still throws on the NULL row'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'network')) WHERE explain LIKE '%Name: idx%'; +SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'network'); -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } + +DROP TABLE tab_ns; + +DROP TABLE IF EXISTS tab_arr; + +CREATE TABLE tab_arr +( + id UInt32, + arr Array(String), + INDEX idx arr TYPE text(tokenizer = 'array') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_arr VALUES (1, ['disk', 'full']), (2, ['disk', 'slow']), (3, ['network', 'down']), (4, []); + +SELECT '-- Array column: a cast that only wraps the elements is looked through'; +SELECT 'Array(Nullable)', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'Array(LowCardinality)', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_arr WHERE has(CAST(arr, 'Array(LowCardinality(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_arr; diff --git a/tests/queries/0_stateless/05220_text_index_map_key_wrapped_constant.reference b/tests/queries/0_stateless/05220_text_index_map_key_wrapped_constant.reference new file mode 100644 index 0000000000000..97f89747069d0 --- /dev/null +++ b/tests/queries/0_stateless/05220_text_index_map_key_wrapped_constant.reference @@ -0,0 +1,30 @@ +-- results are correct and independent of the key type wrapper +2 +2 +2 +2 +0 +0 +0 +-- mapValues index prunes granules for a wrapped constant key, arrayElement form +1 +1 +1 +-- mapValues index prunes granules for a wrapped constant key, subcolumn form +1 +1 +1 +-- mapKeys index prunes granules for a wrapped constant key, arrayElement form +1 +1 +1 +-- mapKeys index prunes granules for a wrapped constant key, subcolumn form +1 +1 +1 +-- a NULL constant key does not use the mapKeys index: arrayElement returns NULL, not the default +0 +-- FixedString cast on the value does not use the mapValues index, Nullable cast does +2 +0 +1 diff --git a/tests/queries/0_stateless/05220_text_index_map_key_wrapped_constant.sql b/tests/queries/0_stateless/05220_text_index_map_key_wrapped_constant.sql new file mode 100644 index 0000000000000..c6fa27dd91fe3 --- /dev/null +++ b/tests/queries/0_stateless/05220_text_index_map_key_wrapped_constant.sql @@ -0,0 +1,66 @@ +-- A text index on `mapKeys(m)` / `mapValues(m)` is used for `m[key] = value` also when the constant key +-- is wrapped in `Nullable`, `LowCardinality` or `LowCardinality(Nullable)`. A non-NULL constant key +-- reads the value-type default ('') for an absent key, not NULL, so the index can still prune granules. +-- The `arrayElement(m, key)` form is exercised with `materialize` on the key, which defeats the rewrite +-- to a map subcolumn; without it the analyzer produces `_CAST(m.key_, 'Nullable(String)')`. + +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +-- Keep a `ReadFromMergeTree` step in the plan so the granule count is reported. +SET query_plan_optimize_count_from_text_index = 0; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + m Map(String, String), + INDEX idx_keys mapKeys(m) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1, + INDEX idx_vals mapValues(m) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 8; + +INSERT INTO tab SELECT map('key' || toString(number % 1000), 'value' || toString(number % 1000)) FROM numbers(1024); + +SELECT '-- results are correct and independent of the key type wrapper'; +SELECT count() FROM tab WHERE m['key5'] = 'value5'; +SELECT count() FROM tab WHERE m[CAST('key5' AS Nullable(String))] = 'value5'; +SELECT count() FROM tab WHERE m[CAST('key5' AS LowCardinality(String))] = 'value5'; +SELECT count() FROM tab WHERE m[CAST('key5' AS LowCardinality(Nullable(String)))] = 'value5'; +SELECT count() FROM tab WHERE m[CAST('missing' AS Nullable(String))] = 'value5'; +SELECT count() FROM tab WHERE m[CAST(NULL AS Nullable(String))] = 'value5'; +SELECT count() FROM tab WHERE m[CAST(NULL AS LowCardinality(Nullable(String)))] = 'value5'; + +SELECT '-- mapValues index prunes granules for a wrapped constant key, arrayElement form'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[materialize(CAST('key5' AS Nullable(String)))] = 'value5') WHERE explain ILIKE '%Granules: 2/128%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[materialize(CAST('key5' AS LowCardinality(String)))] = 'value5') WHERE explain ILIKE '%Granules: 2/128%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[materialize(CAST('key5' AS LowCardinality(Nullable(String))))] = 'value5') WHERE explain ILIKE '%Granules: 2/128%'; + +SELECT '-- mapValues index prunes granules for a wrapped constant key, subcolumn form'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[CAST('key5' AS Nullable(String))] = 'value5') WHERE explain ILIKE '%Granules: 2/128%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[CAST('key5' AS LowCardinality(String))] = 'value5') WHERE explain ILIKE '%Granules: 2/128%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[CAST('key5' AS LowCardinality(Nullable(String)))] = 'value5') WHERE explain ILIKE '%Granules: 2/128%'; + +SELECT '-- mapKeys index prunes granules for a wrapped constant key, arrayElement form'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[materialize(CAST('key5' AS Nullable(String)))] != '') WHERE explain ILIKE '%Granules: 2/128%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[materialize(CAST('key5' AS LowCardinality(String)))] != '') WHERE explain ILIKE '%Granules: 2/128%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[materialize(CAST('key5' AS LowCardinality(Nullable(String))))] != '') WHERE explain ILIKE '%Granules: 2/128%'; + +SELECT '-- mapKeys index prunes granules for a wrapped constant key, subcolumn form'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[CAST('key5' AS Nullable(String))] != '') WHERE explain ILIKE '%Granules: 2/128%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[CAST('key5' AS LowCardinality(String))] != '') WHERE explain ILIKE '%Granules: 2/128%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[CAST('key5' AS LowCardinality(Nullable(String)))] != '') WHERE explain ILIKE '%Granules: 2/128%'; + +SELECT '-- a NULL constant key does not use the mapKeys index: arrayElement returns NULL, not the default'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE m[materialize(CAST(NULL AS Nullable(String)))] != '') WHERE explain ILIKE '%idx_keys%'; + +-- A cast of the map value that changes its bytes must not reuse the mapValues index: the index stores +-- the raw String token, while `CAST(m[key], 'FixedString(N)')` pads the value with zero bytes. Only the +-- casts that keep the bytes (adding `Nullable` or `LowCardinality`) may reuse it. +SELECT '-- FixedString cast on the value does not use the mapValues index, Nullable cast does'; +SELECT count() FROM tab WHERE CAST(m['key5'], 'FixedString(6)') = toFixedString('value5', 6); +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE CAST(m[materialize('key5')], 'FixedString(6)') = toFixedString('value5', 6)) WHERE explain ILIKE '%idx_vals%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE CAST(m[materialize('key5')], 'Nullable(String)') = 'value5') WHERE explain ILIKE '%idx_vals%'; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/05221_tokenbf_map_absent_key_fixedstring_needle.reference b/tests/queries/0_stateless/05221_tokenbf_map_absent_key_fixedstring_needle.reference new file mode 100644 index 0000000000000..f0490c8105347 --- /dev/null +++ b/tests/queries/0_stateless/05221_tokenbf_map_absent_key_fixedstring_needle.reference @@ -0,0 +1,14 @@ +-- absent key, empty FixedString needle: all rows match +1024 +1024 +1024 +1024 +1024 +1024 +1024 +1024 +-- present key, padded FixedString needle: the index is used and the row is found +1 500 +1 +1 500 +1 diff --git a/tests/queries/0_stateless/05221_tokenbf_map_absent_key_fixedstring_needle.sql b/tests/queries/0_stateless/05221_tokenbf_map_absent_key_fixedstring_needle.sql new file mode 100644 index 0000000000000..306b49765f514 --- /dev/null +++ b/tests/queries/0_stateless/05221_tokenbf_map_absent_key_fixedstring_needle.sql @@ -0,0 +1,40 @@ +-- `m['missing'] = CAST('', 'FixedString(N)')` matches every row without the key: `arrayElement` returns '' for +-- an absent key, and `String = FixedString(N)` ignores the zero padding of the constant. The `mapKeys` bloom +-- filter index must not be used for such a comparison, otherwise every granule is pruned and the rows are lost. +-- The default value of `FixedString(N)` is reported as '', while the constant holds N zero bytes, so the guard +-- has to compare the constant without its padding. + +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_tokenbf; +DROP TABLE IF EXISTS t_ngrambf; + +CREATE TABLE t_tokenbf (id UInt64, attrs Map(String, String), INDEX idx mapKeys(attrs) TYPE tokenbf_v1(256, 2, 0) GRANULARITY 1) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8; +INSERT INTO t_tokenbf SELECT number, map(if(number = 500, 'entity', 'other'), 'v') FROM numbers(1024); + +CREATE TABLE t_ngrambf (id UInt64, attrs Map(String, String), INDEX idx mapKeys(attrs) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 1) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8; +INSERT INTO t_ngrambf SELECT number, map(if(number = 500, 'entityword', 'otherword'), 'v') FROM numbers(1024); + +SELECT '-- absent key, empty FixedString needle: all rows match'; +SET optimize_functions_to_subcolumns = 0; +SELECT count() FROM t_tokenbf WHERE attrs['missing'] = CAST('', 'FixedString(3)'); +SELECT count() FROM t_tokenbf WHERE attrs['missing'] = CAST('', 'LowCardinality(FixedString(3))'); +SELECT count() FROM t_tokenbf WHERE attrs['missing'] = CAST('', 'Nullable(FixedString(3))'); +SELECT count() FROM t_ngrambf WHERE attrs['missing'] = CAST('', 'FixedString(3)'); +SET optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_tokenbf WHERE attrs['missing'] = CAST('', 'FixedString(3)'); +SELECT count() FROM t_tokenbf WHERE attrs['missing'] = CAST('', 'LowCardinality(FixedString(3))'); +SELECT count() FROM t_tokenbf WHERE attrs['missing'] = CAST('', 'Nullable(FixedString(3))'); +SELECT count() FROM t_ngrambf WHERE attrs['missing'] = CAST('', 'FixedString(3)'); + +SELECT '-- present key, padded FixedString needle: the index is used and the row is found'; +SELECT count(), min(id) FROM t_tokenbf WHERE attrs['entity'] = CAST('v', 'FixedString(3)'); +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_tokenbf WHERE attrs['entity'] = CAST('v', 'FixedString(3)')) WHERE explain ILIKE '%Granules: 1/128%'; +SET optimize_functions_to_subcolumns = 0; +SELECT count(), min(id) FROM t_tokenbf WHERE attrs['entity'] = CAST('v', 'FixedString(3)'); +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_tokenbf WHERE attrs['entity'] = CAST('v', 'FixedString(3)')) WHERE explain ILIKE '%Granules: 1/128%'; + +DROP TABLE t_tokenbf; +DROP TABLE t_ngrambf; diff --git a/tests/queries/0_stateless/05241_bloom_filter_map_default_value_low_cardinality_nullable_constant.reference b/tests/queries/0_stateless/05241_bloom_filter_map_default_value_low_cardinality_nullable_constant.reference new file mode 100644 index 0000000000000..450deb365d138 --- /dev/null +++ b/tests/queries/0_stateless/05241_bloom_filter_map_default_value_low_cardinality_nullable_constant.reference @@ -0,0 +1,20 @@ +-- { echo } + +SELECT count() FROM t_map_keys WHERE m['k'] = ''; +5 +SELECT count() FROM t_map_keys WHERE m['k'] = CAST('' AS Nullable(String)); +5 +SELECT count() FROM t_map_keys WHERE m['k'] = CAST('' AS LowCardinality(String)); +5 +SELECT count() FROM t_map_keys WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))); +5 +SELECT count() FROM t_map_keys WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))) SETTINGS use_skip_indexes = 0; +5 +SELECT count() FROM t_map_values WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))); +5 +SELECT count() FROM t_map_values WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))) SETTINGS use_skip_indexes = 0; +5 +SELECT count() FROM t_map_keys_tokenbf WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))); +5 +SELECT count() FROM t_map_keys_tokenbf WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))) SETTINGS use_skip_indexes = 0; +5 diff --git a/tests/queries/0_stateless/05241_bloom_filter_map_default_value_low_cardinality_nullable_constant.sql b/tests/queries/0_stateless/05241_bloom_filter_map_default_value_low_cardinality_nullable_constant.sql new file mode 100644 index 0000000000000..0d5529aaac3ec --- /dev/null +++ b/tests/queries/0_stateless/05241_bloom_filter_map_default_value_low_cardinality_nullable_constant.sql @@ -0,0 +1,40 @@ +-- `m['k']` returns the default value for rows without the key `k`, so an index on the map cannot be used +-- when the constant equals that default. The default must be taken from the value of the constant, not from +-- its type: the default of `LowCardinality(Nullable(String))` is NULL, while the constant is ''. + +DROP TABLE IF EXISTS t_map_keys; +DROP TABLE IF EXISTS t_map_values; +DROP TABLE IF EXISTS t_map_keys_tokenbf; + +CREATE TABLE t_map_keys (id UInt64, m Map(String, String), INDEX idx mapKeys(m) TYPE bloom_filter GRANULARITY 1) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; + +CREATE TABLE t_map_values (id UInt64, m Map(String, String), INDEX idx mapValues(m) TYPE bloom_filter GRANULARITY 1) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; + +CREATE TABLE t_map_keys_tokenbf (id UInt64, m Map(String, String), INDEX idx mapKeys(m) TYPE tokenbf_v1(512, 3, 0) GRANULARITY 1) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; + +INSERT INTO t_map_keys SELECT number, if(number % 2 = 0, map('k', 'v'), map('x', 'v')) FROM numbers(10); +INSERT INTO t_map_values SELECT number, if(number % 2 = 0, map('k', 'v'), map('x', 'v')) FROM numbers(10); +INSERT INTO t_map_keys_tokenbf SELECT number, if(number % 2 = 0, map('k', 'v'), map('x', 'v')) FROM numbers(10); + +-- { echo } + +SELECT count() FROM t_map_keys WHERE m['k'] = ''; +SELECT count() FROM t_map_keys WHERE m['k'] = CAST('' AS Nullable(String)); +SELECT count() FROM t_map_keys WHERE m['k'] = CAST('' AS LowCardinality(String)); +SELECT count() FROM t_map_keys WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))); +SELECT count() FROM t_map_keys WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))) SETTINGS use_skip_indexes = 0; + +SELECT count() FROM t_map_values WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))); +SELECT count() FROM t_map_values WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))) SETTINGS use_skip_indexes = 0; + +SELECT count() FROM t_map_keys_tokenbf WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))); +SELECT count() FROM t_map_keys_tokenbf WHERE m['k'] = CAST('' AS LowCardinality(Nullable(String))) SETTINGS use_skip_indexes = 0; + +-- { echoOff } + +DROP TABLE t_map_keys; +DROP TABLE t_map_values; +DROP TABLE t_map_keys_tokenbf; diff --git a/tests/queries/0_stateless/05242_text_index_preprocessor_lossless_conversion_haystack.reference b/tests/queries/0_stateless/05242_text_index_preprocessor_lossless_conversion_haystack.reference new file mode 100644 index 0000000000000..674e55afe3ac5 --- /dev/null +++ b/tests/queries/0_stateless/05242_text_index_preprocessor_lossless_conversion_haystack.reference @@ -0,0 +1,40 @@ +-- { echo } + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(s, 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(s, 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(CAST(s, 'Nullable(String)'), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(CAST(s, 'Nullable(String)'), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toLowCardinality(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toLowCardinality(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasAnyTokens(toNullable(s), 'Foo Qux') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasAnyTokens(toNullable(s), 'Foo Qux') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasAllTokens(toNullable(s), 'Foo Bar') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasAllTokens(toNullable(s), 'Foo Bar') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasPhrase(toNullable(s), 'Foo Bar') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasPhrase(toNullable(s), 'Foo Bar') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab_postprocessor WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2] +SELECT arraySort(groupArray(id)) FROM tab_postprocessor WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2] +-- A part without the materialized index evaluates the row-level function. +INSERT INTO tab SETTINGS materialize_skip_indexes_on_insert = 0 VALUES (4, 'Foo qux'), (5, 'foo qux'); +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +[1,2,4,5] +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; +[1,2,4,5] diff --git a/tests/queries/0_stateless/05242_text_index_preprocessor_lossless_conversion_haystack.sql b/tests/queries/0_stateless/05242_text_index_preprocessor_lossless_conversion_haystack.sql new file mode 100644 index 0000000000000..beec5f63dc11d --- /dev/null +++ b/tests/queries/0_stateless/05242_text_index_preprocessor_lossless_conversion_haystack.sql @@ -0,0 +1,73 @@ +-- Tags: no-parallel-replicas +-- Tag no-parallel-replicas -- direct read is not compatible with parallel replicas + +-- The text index is analyzed on the expression under lossless conversions of the haystack, e.g. `s` in +-- `hasToken(toNullable(s), 'Foo')`, and its preprocessor is applied to the needle. The row-level function has +-- to apply the preprocessor to the unwrapped haystack as well, so the result doesn't depend on whether the +-- index is read directly or the part has the index materialized. + +SET enable_analyzer = 1; +SET use_skip_indexes = 1; +SET use_skip_indexes_on_data_read = 1; + +DROP TABLE IF EXISTS tab; +DROP TABLE IF EXISTS tab_postprocessor; + +CREATE TABLE tab +( + id UInt64, + s String, + INDEX idx(s) TYPE text(tokenizer = splitByNonAlpha, preprocessor = lower(s)) +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; + +CREATE TABLE tab_postprocessor +( + id UInt64, + s String, + INDEX idx(s) TYPE text(tokenizer = splitByNonAlpha, preprocessor = lower(s), postprocessor = if(s = 'bar', '', s)) +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; + +SYSTEM STOP MERGES tab; +SYSTEM STOP MERGES tab_postprocessor; + +INSERT INTO tab VALUES (1, 'Foo bar'), (2, 'foo bar'), (3, 'baz'); +INSERT INTO tab_postprocessor VALUES (1, 'Foo bar'), (2, 'foo bar'), (3, 'baz'); + +-- { echo } + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(s, 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(s, 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(CAST(s, 'Nullable(String)'), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(CAST(s, 'Nullable(String)'), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toLowCardinality(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toLowCardinality(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasAnyTokens(toNullable(s), 'Foo Qux') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab WHERE hasAnyTokens(toNullable(s), 'Foo Qux') SETTINGS query_plan_direct_read_from_text_index = 0; + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasAllTokens(toNullable(s), 'Foo Bar') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab WHERE hasAllTokens(toNullable(s), 'Foo Bar') SETTINGS query_plan_direct_read_from_text_index = 0; + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasPhrase(toNullable(s), 'Foo Bar') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab WHERE hasPhrase(toNullable(s), 'Foo Bar') SETTINGS query_plan_direct_read_from_text_index = 0; + +SELECT arraySort(groupArray(id)) FROM tab_postprocessor WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab_postprocessor WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; + +-- A part without the materialized index evaluates the row-level function. +INSERT INTO tab SETTINGS materialize_skip_indexes_on_insert = 0 VALUES (4, 'Foo qux'), (5, 'foo qux'); + +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT arraySort(groupArray(id)) FROM tab WHERE hasToken(toNullable(s), 'Foo') SETTINGS query_plan_direct_read_from_text_index = 0; + +-- { echoOff } + +DROP TABLE tab; +DROP TABLE tab_postprocessor; From 0914f104970876bd4093ab4dc652b3d78ebd24b7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Oct 2026 22:51:07 +0000 Subject: [PATCH 152/296] Backport #122272 to 26.8: Fix `CREATE OR REPLACE` bypassing the drop privilege of the replaced table via `Replicated` databases and `ON CLUSTER` --- src/Interpreters/InterpreterCreateQuery.cpp | 27 +++++++++++---- ...or_replace_remote_drop_privilege.reference | 4 +++ ...create_or_replace_remote_drop_privilege.sh | 33 +++++++++++++++++++ 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05238_create_or_replace_remote_drop_privilege.reference create mode 100755 tests/queries/0_stateless/05238_create_or_replace_remote_drop_privilege.sh diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index bac302032c8a6..19de34177dd74 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1668,6 +1668,16 @@ bool isReplicated(const ASTStorage & storage) return storage_name.starts_with("Replicated") || storage_name.starts_with("Shared"); } +/// The drop privilege matching the kind of an existing table. +AccessType getDropAccessType(const IStorage & table) +{ + if (table.isView()) + return AccessType::DROP_VIEW; + if (table.isDictionary()) + return AccessType::DROP_DICTIONARY; + return AccessType::DROP_TABLE; +} + } BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) @@ -2840,12 +2850,7 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, { /// The replaced table is dropped after the swap, under an internal temporary name that /// grants cannot cover, so check the drop privilege for its kind here, on its real name. - AccessType drop_access = AccessType::DROP_TABLE; - if (to_drop->isView()) - drop_access = AccessType::DROP_VIEW; - else if (to_drop->isDictionary()) - drop_access = AccessType::DROP_DICTIONARY; - current_context->checkAccess(drop_access, to_drop_id); + current_context->checkAccess(getDropAccessType(*to_drop), to_drop_id); to_drop->checkTableSizeBelowDropLimit(current_context); } }); @@ -3510,6 +3515,16 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const } } + /// Replicated and ON CLUSTER replays run with full access, so the drop privilege for the replaced + /// table's kind must be required here, on its real name, while the query still runs as the user. + if ((create.replace_table || create.create_or_replace || create.replace_view) && !create.isTemporary()) + { + String database_name = getContext()->resolveDatabase(create.getDatabase()); + if (auto database = DatabaseCatalog::instance().tryGetDatabase(database_name)) + if (auto table = database->tryGetTable(create.getTable(), getContext())) + required_access.emplace_back(getDropAccessType(*table), database_name, create.getTable()); + } + if (create.targets) { for (const auto & target : create.targets->targets) diff --git a/tests/queries/0_stateless/05238_create_or_replace_remote_drop_privilege.reference b/tests/queries/0_stateless/05238_create_or_replace_remote_drop_privilege.reference new file mode 100644 index 0000000000000..9971dc825b31a --- /dev/null +++ b/tests/queries/0_stateless/05238_create_or_replace_remote_drop_privilege.reference @@ -0,0 +1,4 @@ +ACCESS_DENIED +ACCESS_DENIED +Dictionary +Dictionary diff --git a/tests/queries/0_stateless/05238_create_or_replace_remote_drop_privilege.sh b/tests/queries/0_stateless/05238_create_or_replace_remote_drop_privilege.sh new file mode 100755 index 0000000000000..975c23566f924 --- /dev/null +++ b/tests/queries/0_stateless/05238_create_or_replace_remote_drop_privilege.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Tags: zookeeper +# `CREATE OR REPLACE` sent to a Replicated database or ON CLUSTER is replayed with full access, so the drop +# privilege for the replaced table's kind must be checked on the initiator, as the user, before it is sent. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_TEST_UNIQUE_NAME}" +atomic_db="atomic_${CLICKHOUSE_DATABASE}" +repl_db="repl_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query " +CREATE DATABASE ${atomic_db} ENGINE = Atomic; +CREATE DATABASE ${repl_db} ENGINE = Replicated('/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/repl', '1', '1'); +CREATE DICTIONARY ${atomic_db}.d (k UInt64) PRIMARY KEY k SOURCE(NULL()) LAYOUT(FLAT()) LIFETIME(0); +CREATE DICTIONARY ${repl_db}.d (k UInt64) PRIMARY KEY k SOURCE(NULL()) LAYOUT(FLAT()) LIFETIME(0); +CREATE USER ${user} IDENTIFIED WITH plaintext_password BY '${user}'; +GRANT SELECT, CREATE VIEW, DROP VIEW ON ${atomic_db}.* TO ${user}; +GRANT SELECT, CREATE VIEW, DROP VIEW ON ${repl_db}.* TO ${user}; +GRANT CLUSTER ON *.* TO ${user}; +" + +${CLICKHOUSE_CLIENT} --user "${user}" --password "${user}" --query "CREATE OR REPLACE VIEW ${repl_db}.d AS SELECT 1" 2>&1 | grep -Fo ACCESS_DENIED | uniq +${CLICKHOUSE_CLIENT} --user "${user}" --password "${user}" --query "CREATE OR REPLACE VIEW ${atomic_db}.d ON CLUSTER test_shard_localhost AS SELECT 1" 2>&1 | grep -Fo ACCESS_DENIED | uniq + +${CLICKHOUSE_CLIENT} --query " +SELECT engine FROM system.tables WHERE database IN ('atomic_$CLICKHOUSE_DATABASE', 'repl_$CLICKHOUSE_DATABASE') AND name = 'd'; +DROP DATABASE ${atomic_db}; +DROP DATABASE ${repl_db}; +DROP USER ${user}; +" From 0e9782821f827ac5925d063f812beb2f467c2c08 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 2 Oct 2026 00:26:21 +0000 Subject: [PATCH 153/296] Backport #117403 to 26.8: Fix ssl_certificate authentication bypass via embedded NUL byte in certificate CN --- src/Access/Authentication.cpp | 5 ++ src/Common/Crypto/X509Certificate.cpp | 47 ++++++++---- .../certs/client13-cert.pem | 31 ++++++++ .../certs/client13-key.pem | 51 +++++++++++++ .../certs/client14-cert.pem | 31 ++++++++ .../certs/client14-key.pem | 51 +++++++++++++ .../certs/generate_certs.sh | 73 +++++++++++++++++++ .../test_ssl_cert_authentication/test.py | 35 +++++++++ 8 files changed, 310 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client13-key.pem create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client14-key.pem diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 3b5e4424239a6..a0bcd3479becc 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -298,6 +298,11 @@ namespace for (const auto & certificate_subject : ssl_certificate_credentials->getSSLCertificateSubjects().at(type)) { + // Subjects are extracted with their exact bytes, so an embedded NUL byte survives. No valid + // hostname or URI contains one, and '*' must not match a span like "evil\0" in + // "evil\0.corp.example.com", so such a subject never matches a wildcard. + if (certificate_subject.contains('\0')) + continue; // Checked before the substr below so its length cannot underflow when prefix and suffix overlap. if (certificate_subject.size() < prefix.size() + suffix.size()) continue; diff --git a/src/Common/Crypto/X509Certificate.cpp b/src/Common/Crypto/X509Certificate.cpp index a2009ddf3f4f4..8efa49f8e5279 100644 --- a/src/Common/Crypto/X509Certificate.cpp +++ b/src/Common/Crypto/X509Certificate.cpp @@ -173,26 +173,45 @@ std::string X509Certificate::subjectName() const return buffer; } +/// Extract the value of the first entry with the given NID from an X509 name as a length-delimited +/// string. We read the ASN1_STRING bytes directly instead of X509_NAME_get_text_by_NID because that +/// function copies into a fixed C buffer and NUL-terminates: an embedded NUL byte (e.g. a CN of +/// "admin\0.evil.com") would be silently truncated to "admin", letting a certificate impersonate a +/// different subject during authentication. Preserving the exact bytes makes such a value compare +/// unequal to any NUL-free configured subject, and also avoids silent truncation of long names. +static std::string extractNameEntry(X509_NAME * name, uint nid) +{ + if (!name) + return {}; + + const int index = X509_NAME_get_index_by_NID(name, static_cast(nid), -1); + if (index < 0) + return {}; + + const X509_NAME_ENTRY * entry = X509_NAME_get_entry(name, index); + if (!entry) + return {}; + + const ASN1_STRING * data = X509_NAME_ENTRY_get_data(entry); + if (!data) + return {}; + + const unsigned char * bytes = ASN1_STRING_get0_data(data); + const int length = ASN1_STRING_length(data); + if (!bytes || length < 0) + return {}; + + return std::string(reinterpret_cast(bytes), static_cast(length)); +} + std::string X509Certificate::issuerName(uint nid) const { - if (X509_NAME * issuer = X509_get_issuer_name(certificate)) - { - char buffer[X509Certificate::NAME_BUFFER_SIZE]; - if (X509_NAME_get_text_by_NID(issuer, nid, buffer, sizeof(buffer)) >= 0) - return std::string(buffer); - } - return {}; + return extractNameEntry(X509_get_issuer_name(certificate), nid); } std::string X509Certificate::subjectName(uint nid) const { - if (X509_NAME * subj = X509_get_subject_name(certificate)) - { - char buffer[X509Certificate::NAME_BUFFER_SIZE]; - if (X509_NAME_get_text_by_NID(subj, nid, buffer, sizeof(buffer)) >= 0) - return std::string(buffer); - } - return {}; + return extractNameEntry(X509_get_subject_name(certificate), nid); } std::string X509Certificate::commonName() const diff --git a/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem new file mode 100644 index 0000000000000..82687debfcdec --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFPzCCAyegAwIBAgIUb8W/doRsWnXgJSqDgDz8Yb+cLVwwDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjAwMTAx +MDAwMDAwWhcNMjkxMjI5MDAwMDAwWjBhMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRow +GAYDVQQDDBFjbGllbnQxAC5ldmlsLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIP +ADCCAgoCggIBAOMBJ2yqK72eew557aeGID0RXFzVs6lLTN4/71C4/309gAcIoqsw +Phulwzj3IfH7v2PPntsROxGtRCVS5LWarXUTEviEaiWUp1MwoFlrhn2Dm78hIgYC +7NrdtI48tb9t9y6eTio99WdIiXGY2KNlctfnTyCxKeanZi7JZUmb9VW95gh/5dl3 +sajZPjUDrFxqUQbzsSzbgElFkjvOzN5xEoUM1qbjHcyML7aKweXJKhSswdi4K3oA +v/FbFleJQgoRPB43JDqc5YjpPARKJlt5VZqpkgiKFAm0R+o4TYeCBd0kZFtL+7Y3 +TO27ZdQiOFKR/DINPCdA3/nxtQ7GJ17g/nwXLQRKUDxXain7jo+87OHghMA8WfwQ +lMSkQ+sqJdxA/VunQEHNCljw64BHeuU6CqQDU9eEnFNq9+3Qb0INiMRn6EGj20LK +5+q/fJqGm/N8BcuoYHUymbtojAjnJmCIWwvTzo7OYaG84nRwOPkZZvcSOrzn2CbT +zhmXHiSnP7ZAL/r24ZW5oikjw6Ie40iqQ2MyknTv3Ko1LWIKkjAbMcE6fo3SlzMA +fVsm9wAyVa/UyjO1TcnPNNta1TdqSgdRsEPDvR/IMPQYJ7Be0H0SZLj2/v8ZmHmZ +KVfGKdWnFBQKiTSHVEBHBdXADcXi2grYjnOBKnQ7H2UQK9jfGMu4FPYlAgMBAAEw +DQYJKoZIhvcNAQELBQADggIBAF4piZQxvyxwfhQxmNbyc8EtUo5qBIWE026e5Hsl +4QX3glKhAo+xLEQbKAImsMdXXYNdGaBJDTEZRbGSvyEgVa8e4YF0FP+CkFDcM0oF +k7YUgQHkZvlR/lsd5idMM7JJ1WXnux6jay9hpF9GEi2QMB2QkwuRwBHkgtjEun8C +qdyvApSecY6BXQllAJyoalSVR4ZzaMGVgSGL99RKyn320icevO6WDOVocEiohJTK +KOwf8q9G6riKWsRbNNPhkt9QwZzC0/RuKV6r25QR7H+GSjmllcBnbyqW3+DGGuJL +iMtKrCmx4xHedvDXRxi8ZxsfIrWVI7f5hgcWCXFWFMeqkU9nD3TGJrAuamBBQMZn +aC3q9AyVudroJs/1ojrSD92+H3qSzhqoLZ4bd531ShPasQ/EHIiOeN794j3Rxo0d +ituD2LSboe2Zr4TNhi7ou/dSfDXPEwJSb5V1Xgp9OyhQCKnFa+RkbQVtME5dVBMR +oO4HMTqaCgYSzCKhrxIO/vid2p6NQp7ZI8V2hAhUR4aBwnWFFspmuWaXpZqXxE6+ +a46rTA4HLpQ7TKwbi4fS2zWSh64nCHmv72GOHQ86dlXDNvfqLo8vyz79chKPo+31 +fsTZJks9wBzeSsq0tqAWflN/hQQCZJT1IpIc5VwU45FLSraKBGYRpCP4FnciXUuJ +RiyF +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem new file mode 100644 index 0000000000000..452e632233838 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA4wEnbKorvZ57Dnntp4YgPRFcXNWzqUtM3j/vULj/fT2ABwii +qzA+G6XDOPch8fu/Y8+e2xE7Ea1EJVLktZqtdRMS+IRqJZSnUzCgWWuGfYObvyEi +BgLs2t20jjy1v233Lp5OKj31Z0iJcZjYo2Vy1+dPILEp5qdmLsllSZv1Vb3mCH/l +2XexqNk+NQOsXGpRBvOxLNuASUWSO87M3nEShQzWpuMdzIwvtorB5ckqFKzB2Lgr +egC/8VsWV4lCChE8HjckOpzliOk8BEomW3lVmqmSCIoUCbRH6jhNh4IF3SRkW0v7 +tjdM7btl1CI4UpH8Mg08J0Df+fG1DsYnXuD+fBctBEpQPFdqKfuOj7zs4eCEwDxZ +/BCUxKRD6yol3ED9W6dAQc0KWPDrgEd65ToKpANT14ScU2r37dBvQg2IxGfoQaPb +Qsrn6r98moab83wFy6hgdTKZu2iMCOcmYIhbC9POjs5hobzidHA4+Rlm9xI6vOfY +JtPOGZceJKc/tkAv+vbhlbmiKSPDoh7jSKpDYzKSdO/cqjUtYgqSMBsxwTp+jdKX +MwB9Wyb3ADJVr9TKM7VNyc8021rVN2pKB1GwQ8O9H8gw9BgnsF7QfRJkuPb+/xmY +eZkpV8Yp1acUFAqJNIdUQEcF1cANxeLaCtiOc4EqdDsfZRAr2N8Yy7gU9iUCAwEA +AQKCAgAhsUz9M2q866tUaY4iXxAk5ZFBvtQ6NztzAfv5YrlFCPOaLP1xOX6rvYLd +eczCLP0/Xr8IwSTKdF7uHUC7Svhzgn++fsTUdse5Bg7JXDdregphUr/JMvZIa+n9 +xzlV7t07wEerZw59Txuwo8uFFlchKtX97QhiPDa4kLAlve4AKmiHRW3DmnY2MKcA +2Qk5nuwPg0Go/fFWBBX20N8WevjmjOfFKxDZXctCeujZkW3bVg2K0ZcJC4McWx/U +oUI52qIDfWgKrQ6NL3T20VyibL64bx0GLccDQ7dS0RrXbISF9Kin5lNiEbuAzjTd +yLB3DO+TgY8L58n2lUSsKzFdGI7yFjJTr6yzmKIayS11c00TCBQ0w/l+oCWOhrvM +zn/k3mrORcRA1aqX5mcqAdzs6hYFIyFbKXNH8hlE7JnUUbcvR76WbzMl6IxPxMT+ +ikbTIQnG+T2PMgmaxVbe9YhyHT2YbMHF7igocz5mI0PHsdNaB1IBSV1G2Knq8QJi +rbYUdCShTJksrNIXghyjbCPx5VIo05fx8NleCuNND18WPs32tI2rceLlF3KSk8GF +moCnPJFTtEgk5IvzWavOGO+abkU1A9xGOhNHCMZVxOca/ZGlhHSRBCuhagpLPx9e +QprxfFVIFG8jNrN0/Mhgv3ivKmbYZuv03wdcWDtKCdMrxkah9wKCAQEA8Xp+V4zk +Ta4nqkjL4TTv+uczszGt/Kb5OYZyb1EzUseabRyOFkb2iMUf+Ol/92UMBlh/FzTa +RFHE1lRed45l8Jhgsbaq9r8gmJZ1WZM2IUByUI++z+USlEtG8O+8wlniyTn9Fc+n +y+wkZwSeJSLWZAWIzeygJs/ceKG+4TpnLZnC2rQqOoKGc+0Dimc1E5HV3mOkhwe1 +CtiYOUvgNf0rEV7E5D7smjpAeiJ8xPhYRg0XXLztN+9rQA+6AvwpbI3VnyAKbPUQ +/J+ZVg0cVMH7pTDXMj0jB2wevrIa7xj2Pn4w/jR8i/tNjFauMMvnofr6RW6KqYk/ +opXhfSE1Pk9w6wKCAQEA8KfWOQEmKlC0KJrN1/D1ZchNZf57pNFtjyWdjLUah05k +a0QLUe1yme7lTw59tsFIfrd/ozeiWKSvhum9/mUNvwBgPjnqp73pmrEEu1bgQ8Ib +cCm55GAmBWlNMO52iD+EPdsYvj+KgAWxu6W8B6XsyjoVLBYqHwNzitZlW9D1PAbY +KWBppmntVAhj7oNvk40uWEFIuTU0o0RIrvcYmX795sr2PY69ozOdiY6UziR4oRAs +DhJ+0a8Sjct0/RMq+sOpBONrpOBHqJzoCX+/alOjbJ8FYIMuc6OU2cNtcxjL3tcZ +cJKInsMl7bMYhi0SIEnttuAgh6mzD4Iox1WX3hbxLwKCAQEAySWPHmCnQCnEsqzG +38aX7DkwsKC+XEm+KnPa2O0bwmWwNhlmJhpgfBcwBciDJtHODW8kFnGZKvWt8BcB +RbehJKPZT89oY/dbMJ+MCtx8Z4BmeML0X/ph2pNF+abJZl83cQVT0xpRnKUMwZ9w +GGEHkvOlPFtSIGJfNUEOXlCm565ASKtwzaIyW1hf7acA6Fc/fmsj/rKl1O4NBxU0 +I+TKPkLh4Xqk3eeOE+6sXeq8pUV8Y2ygcUqG3Sr8eEYSP3F6M32eEZql1rkfIjXz +loqrkrO3XgrdJe6jQZfcpbP2UqINL4MLCBOCSldd7Bm9zgjg9nsZGBXSox0UYoYJ +8uh7lwKCAQA+I3gi+/L41iHOojooWeVjRJcHkPAoHJNndNT7cf/JlCpFsCokG2WN +7at0AE/hkoK/hW4FnOXkcZGJCm2udDVabiRrrNS0P0tEUBTisonxtPsUuRFwsIrg +ttHhopEkmRHyTtJSvWFrsQy1YRPt/Z/oj5rL6WUy4NdCsB032fqYZ0QFWwmsmIlZ +O1liSrnSpY+j5id6+wv+ZDFITDEj4TB6GUn/lw3MSBWTYSd3Gt+y5tQZRhlM1yG5 +TyGD/yEH4uGPi5FN30NhfDJF0aCBOdtmvqDKzNR/s2tJ1zY5k9uATJYbBRsVs1nl +yGq6qoSVpcEliTWdEepURM12utkd1VqtAoIBAFLriLD2AF0+ayheiDFgsbRo78V3 +dygUPBZi7b6nPWv3gugnwvWrTFaCAndACxNZPWNwEcING2Utjp8PTvPTzad9yibl +3xqNSKR9gaqmDYPEY1DwYGCpHRc969PIYcyK6z3dICj1ySEo0RwTa+Ab0OHeRWzS +ROnmmy9a9Vpx7KyMZcmOGhSXL3q7nr2imI+qZ4tGiEJts85d2fXAVhGeBOagzy5n +uD00wYzBc8rZ9u+d7ZOyQetgoJBnhyQNzW7IBCN+Z3EWW5lPlHQJpTXjHCdXgiZU +28pX/4J/mIcaURA0xhexuuLqTEjLlFoAnjNjWCCXv0u5uP8Ba35g4BSAz0o= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem new file mode 100644 index 0000000000000..25c2687c0edf6 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIUTgWFBpS4cJ/+Fgg5jAfOubkwokowDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjAwMTAx +MDAwMDAwWhcNMjkxMjI5MDAwMDAwWjBmMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMR8w +HQYDVQQDDBZldmlsAC5jb3JwLmV4YW1wbGUuY29tMIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAs6uGg+vk4GlmN+ZlJb9gUI6AS6TJVQvLCSw7RPHt0k8x +qMXeoHid3mmVIC7tWK6BDEbAWJB814cWzOviXy5GbuUZB5Yq1MR0bQV968PBa/jp +/X6eDAW3KDfvgsk2eBo+aZqbFh1EJqXzN04d255X2mlAGpwW97T+XhXa/ahxj1Ey +G5smL6jaybcrKj/Xce4qVgSQk0kVlun/35Rfd6kcFTdq+Laap3nO0JKBYwfcoeMU +vVfRZV0iaP7sbOpiZlv0+zFDDGX2Xi18VH2xeclICN4ar8K0h+VNE1uqf27r+3NA +iZbZphA2huN60KYP+IV09KirWhFM3Xcd5LlqbluRUwWxp4xXZNWFIa6AeSbuC+43 +hAa0vBfN9+CE03ifX6stqfUy/eiJsghuPEZOUXqJpBpz4VHpxXVqiLX11hrGw60S +rEQJ30SVahULl4v1/zhFaRgFnZE2yRdQG0h1AejA5OUykKTH5vAkfZRxPmL3hnBB +fg+iti93EFvdC2bXNMB04X+VkvF8CcqRUr2Ka7ZpfleJuO9ps2e0LBtjSJOLxYRA +N13RUMp9m/45F5iGE4QXdUFzOrasrclg8KfP0Zl+6/FQcygV4bLQywqm/zgCX1zC +SAb88JLEElIUmyMFT/cvst1uscy1fvoLseuKcz+9woAF1LA8XrZzMPRA22xSABMC +AwEAAaMlMCMwIQYDVR0RBBowGIIWZXZpbAAuY29ycC5leGFtcGxlLmNvbTANBgkq +hkiG9w0BAQsFAAOCAgEAFuyPgncpYKGYffQPoOWR2yTn6L56XxvQcFNrchyKkwAn +zsBQFLOX4T4z1FuIQTwwKiSRwz+hwuIfcK2++I1yyxgYzw8Cxw7uqpvRi6AgN9pk +LEsXbSOvqaQyFEukVFenbkIejBVexF74gk2LKJYnT7MRVgs/k8C5Lqy5lKpB/2Gk +7xnzVe6mLhdZrMRbtCmt+Pr2m5l+XMGucnoaNCK87zSIGEt8fCYNB//vQ61GeANv +hRYiWgdWcOCnl2ip+EWbq0+ZK2JvJ8Ih2d0S+Qtbhek8XQsuCoa5P3YjLyAfzhpk +/CZb+VbOH41qZ2uWi05xVE3IRbJBlRDnX/bXC4Nx2ojzOnWAqzmrJffsKal/04oP +BcA6dOzm+CvT7AlzImlRI/6ZEC6DlrXSkvm5yRNwdUhTZ5ZjQnSyZBF4lNY3u9Sa +G1ipxyR9wRcA7uac8cFv/MBAGP5godwi3WPvqC/7noGDbZqDxcsZOeVFdP+NFtFg +dilITLxlgaQimN2nxhPvAAz6EcJ2owFgJJk6J3f6ICq8gdbJzFq2uVJrKBBL35St +/aOpflDRhQkMdJwrCuI/+4NBcXG5fQywjH8H+9dvbUONIlttbwE2zZJgXtY0gb+A +kY8fmy5pWBYnYAB8lFdAZxW0Sx6ehkmLA/ExtH/7jz/GwnwS12/of6wSvQHj8so= +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem new file mode 100644 index 0000000000000..10d1b89a27011 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAs6uGg+vk4GlmN+ZlJb9gUI6AS6TJVQvLCSw7RPHt0k8xqMXe +oHid3mmVIC7tWK6BDEbAWJB814cWzOviXy5GbuUZB5Yq1MR0bQV968PBa/jp/X6e +DAW3KDfvgsk2eBo+aZqbFh1EJqXzN04d255X2mlAGpwW97T+XhXa/ahxj1EyG5sm +L6jaybcrKj/Xce4qVgSQk0kVlun/35Rfd6kcFTdq+Laap3nO0JKBYwfcoeMUvVfR +ZV0iaP7sbOpiZlv0+zFDDGX2Xi18VH2xeclICN4ar8K0h+VNE1uqf27r+3NAiZbZ +phA2huN60KYP+IV09KirWhFM3Xcd5LlqbluRUwWxp4xXZNWFIa6AeSbuC+43hAa0 +vBfN9+CE03ifX6stqfUy/eiJsghuPEZOUXqJpBpz4VHpxXVqiLX11hrGw60SrEQJ +30SVahULl4v1/zhFaRgFnZE2yRdQG0h1AejA5OUykKTH5vAkfZRxPmL3hnBBfg+i +ti93EFvdC2bXNMB04X+VkvF8CcqRUr2Ka7ZpfleJuO9ps2e0LBtjSJOLxYRAN13R +UMp9m/45F5iGE4QXdUFzOrasrclg8KfP0Zl+6/FQcygV4bLQywqm/zgCX1zCSAb8 +8JLEElIUmyMFT/cvst1uscy1fvoLseuKcz+9woAF1LA8XrZzMPRA22xSABMCAwEA +AQKCAgAEaIj8Y6VR/EQNyxFgQ7nRQC3VrU1jUM7CgttRbb4wEtFdGr3DojH9awnF +qGEac+2mp3XAtorZnu7oSEFdpH0F64kZro2OeuOAaUoVps/wHkNffOPT17AOxJCT +3OwBNmOho7F6cW1ipV+6U6hX4yK0sTBpdrr5iO9Uz6R35NIkehGIq93b/YCgwmXE +u5xFp1pSkfoaIwjskwE8Mx/Eh9mwi5OMVq6kvVBdvbp++4pmTnQL0UPKAOb/PIIA +ih+v80GniCXk//tzhBow2ISqQE4MKabt+REE5JNnjjA4wDf6C3Hh7lmYwX0VAi/Z +PrnVlzCvcBQEObhxFqMdIY+C9awzGrf0iJ1UGZms5IrcKwvEL7VsrQqO2COEv42K +KtbVNpRMYG5n+VwtylChmh2g6oX4wMLbYIykDfXZVv/FIjP3vgPxs19t/CRVUEer +x6GztwhAmKvp4WzerVH1xJ7QKcaIuy4cjhNLVUQCcjPbFbtHsHzvuScXHfppCM67 +6rpNQ5DrQ0Wj0a1kCal/57aKQQdD37k4aTTEoCbDJkbD10SoaljJYElHleB26xCV +AwbyXz6gwJ80CxbXS1uco3c86keDekrGV5f73XX53MZouRR90Ii/TJvvdGutXFkX +mpAIaOZlfeDSFMJoFaSpOj3UaY4RBwcXXDB69QpFMDe2ND0gwQKCAQEA6jw1G0dP +HwDYfQSMzTQkhle7hHqxPoyuooBe4Vl6MJ64S22tgvRLE+FuLfubR9tMBYYr/Ub4 +CmSBE/XMbNGpH31yct5I/8rc9OnRg0bcCw2dz/Q7b84LKas6fkBFaf1mMlPvH2zi +nwtS3Bw3Q6DAky1ra6rqjiZDtDCN+vQOh0zXVdOyVjApDBbngX/x8hVhFLs3SuwS +jm3P8YloEQ6aQ0wCjvsemJs+IpDlH7cHwYlgxBREqQ1wOKdo8IZ/MTTvlPOXW04F +pP995LCvtVacjhfOLaWdHTs442yuFvFBoPWFEvx2CGYTQe+dk8zmzLFdYVNpEvbV +dh/m1hMrA3lUdwKCAQEAxF1d+QuDanP39u+16u4+u61p3gzYv2f9m+bjViiYMv6N +kSDzdXrKidD1I+iEgIy1QjRfdHlQMCIpP8D4zuWCrNY57QKdV4bRKs0RSgpOXPv+ +FGeVf9taPIH/lUH8FUJ1gqfoLX+6W2RbZxcNPOSOf13o6jTjBbB6oVetljNjsnDS +us7CJe3+MDk20pO2+E49Q0TuyeALDPRSciSvJrVCVmoPs6rpu22z5zMCxd6+WztL +YBWfFM38QpIMUg1miz7hU7m7lQHlYE9/p9RSI5ePD9mgepPQN+bQg0fGQaD0tzad +MJr/BI0OvHLdwFQNLya66nWK7s85aD/UjDRO/A6kRQKCAQB1zGes21ToM6WsYeBp +xsJjqbWNb6K54UhmQwb0b+pqjzgB/xuW00L6sZGWoIW8QoZd9Ncknk9Z8qeToTb4 +twxF4PHw4Od3dM9ggEK0sasyB9wI3DwUA1xLzWgyXCJMpnqB7wJAHKNv9uLp/Wqx +oSOYIOx4DlG9wXKlKRIOVjUESFm3OSrj+355LP+qeez0oVnccjbhgA3pAULlpwPm +KCDenVhgDdyaROCfw5znMUY+R9eZZNQO7Mo2Q8Mby5gl6AhhMYw6B+gAzdjDbTRA +j1lWgJRZEoQMUl9OyLZYpWYrC66sGLlHigY/T8FAtniQEtbyfl9GgUpjCLIvkR49 +tgQLAoIBAF/WefD9D4y7QQDCifU5hmCvCIaZmogAxyR6EeaRNYdd+dYlUO27mnKd +C6gU6eabxjOjwBrmwp5bbepx0n2YQqj8fZURu51mbVwIbjHGyexUCPQIgky+0FHL +2OQOKmxt3VCBhq3+MwQ7/OhZtdpMasf7G5yDZ3H1akSouE4gkr4alp8aHmPIvlDm ++7zW32xdM0VLtYfN01blP//5q4qm2NO4PCWieyVBK5bhrK7KQfng/K7Onq/WwRH0 +mhLJ+4xmii8E7WqSXFMfOdy9ocFBTU+dFdf9oJhIDOil9Ts+xXFONHXukBy2g8Sy +A0zFORIUQxH/gGmBtjENRj2PoiUfOEkCggEBAMiBLM8L+k7ovGd7KVoeSgk7FjsV +5gq0GDwXGov9HEyeY/mmmuS5mfxZUYvRgzbpQutqy88MvbMsCS7VDtT9m0IFs8zR +qXIUQmOrYJuI37BrA16K5EDTj6wY/7DKRSM4guNqQjYnJ2bYBrXCZLWFUF/OVl3w +VF5reN7BCMOTXGc4aAeJzcmvhGU7M3WnGwL7yGkiX8BCDXo+ddLuf1SyDUttfrmo +Kbb4h4LAe2ATtRO0F9SfEetdmHUwO6iRJGQNQQWO7pWOjUStSry3Z8AM0uVDBRDG +Ol3DmKQlxbFAhLD62KkcWt30eNnNMlYaq8huKY4Y2juyi5ZQ31sDij4worA= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh index 19354a5a3e364..b8bbd34567a21 100755 --- a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh +++ b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh @@ -60,3 +60,76 @@ openssl x509 -req -days 36525 -in client_far_future-req.pem -CA ca-cert.pem -CAk # 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" + +# 7. Generate a CA-signed certificate whose CN carries an embedded NUL byte ("client1\0.evil.com"), +# to test that server-side CN extraction does not truncate at the NUL. User 'john' is configured with +# client1, so a server that truncated the CN at the NUL would extract +# "client1" and wrongly authenticate this certificate as 'john'; the full CN must be preserved so the +# match fails. openssl's CLI cannot place a NUL inside a -subj field, so we build this certificate +# with the 'cryptography' library instead. +python3 - <<'PY' +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import datetime + +with open("ca-key.pem", "rb") as f: + ca_key = serialization.load_pem_private_key(f.read(), password=None) +with open("ca-cert.pem", "rb") as f: + ca_cert = x509.load_pem_x509_certificate(f.read()) + +key = rsa.generate_private_key(public_exponent=65537, key_size=4096) +subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, "RU"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Some-State"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Internet Widgits Pty Ltd"), + x509.NameAttribute(NameOID.COMMON_NAME, "client1\x00.evil.com"), +]) +now = datetime.datetime(2020, 1, 1) +cert = (x509.CertificateBuilder() + .subject_name(subject).issuer_name(ca_cert.subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now).not_valid_after(now + datetime.timedelta(days=3650)) + .sign(ca_key, hashes.SHA256())) +with open("client13-key.pem", "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) +with open("client13-cert.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) +PY + +# 8. Generate a CA-signed certificate whose CN and DNS SAN both carry an embedded NUL byte +# ("evil\0.corp.example.com"), to test that wildcard matching rejects it. Users 'wildcard_cn' and +# 'wildcard_dns' are configured with '*.corp.example.com' and 'DNS:*.corp.example.com': without an +# explicit check, '*' would match the single "label" "evil\0" and authenticate this certificate. +python3 - <<'PY' +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import datetime + +with open("ca-key.pem", "rb") as f: + ca_key = serialization.load_pem_private_key(f.read(), password=None) +with open("ca-cert.pem", "rb") as f: + ca_cert = x509.load_pem_x509_certificate(f.read()) + +key = rsa.generate_private_key(public_exponent=65537, key_size=4096) +subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, "RU"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Some-State"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Internet Widgits Pty Ltd"), + x509.NameAttribute(NameOID.COMMON_NAME, "evil\x00.corp.example.com"), +]) +now = datetime.datetime(2020, 1, 1) +cert = (x509.CertificateBuilder() + .subject_name(subject).issuer_name(ca_cert.subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now).not_valid_after(now + datetime.timedelta(days=3650)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("evil\x00.corp.example.com")]), critical=False) + .sign(ca_key, hashes.SHA256())) +with open("client14-key.pem", "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) +with open("client14-cert.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) +PY diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 977bc82ea2766..118172905b9a9 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -147,6 +147,18 @@ def test_native_fallback_to_password(): assert "AUTHENTICATION_FAILED" in str(err.value) +def test_native_cn_nul_byte_no_bypass(): + # Authentication bypass: client13's CN is "client1\0.evil.com" and user 'john' is configured with + # client1. If server-side CN extraction truncated at the embedded NUL + # byte, the CN would collapse to "client1" and the certificate would authenticate as user 'john'. + # The full CN must be preserved, so the match must fail. + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user="john", cert_name="client13" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + + def get_ssl_context(cert_name): context = WrapSSLContextWithSNI(SSL_HOST, ssl.PROTOCOL_TLS_CLIENT) context.load_verify_locations(cafile=f"{SCRIPT_DIR}/certs/ca-cert.pem") @@ -221,6 +233,14 @@ def test_https_wrong_cert(): ) +def test_https_cn_nul_byte_no_bypass(): + # Same bypass as test_native_cn_nul_byte_no_bypass, over the HTTPS interface: client13's CN + # "client1\0.evil.com" must not be truncated to "client1" and authenticate as user 'john'. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name="client13") + assert "403" in str(err.value) + + def test_https_non_ssl_auth(): # Users with non-SSL authentication are allowed, in this case we can skip sending a client certificate at all (because "verificationMode" is set to "relaxed"). # assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False) == "peter\n" @@ -546,6 +566,21 @@ def test_x509_cn_wildcard_single_label(): assert "403" in str(err.value) +def test_x509_wildcard_nul_byte_no_bypass(): + # Authentication bypass: client14's CN and DNS SAN are both "evil\0.corp.example.com". Users + # 'wildcard_cn' and 'wildcard_dns' are configured with '*.corp.example.com' and + # 'DNS:*.corp.example.com'. The '*' must not match the span "evil\0", so both must fail. + for user in ["wildcard_cn", "wildcard_dns"]: + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user=user, cert_name="client14" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user=user, cert_name="client14") + assert "403" in str(err.value) + + def test_x509_uri_san_wildcard_dot_in_segment(): # Non-regression: '.' separates labels for DNS/CN but is NOT a separator for URI SANs, # whose separator is '/'. A wildcard URI path segment may legitimately contain dots, so From 625bcff8eac7cb0a8e8a12508dc3f7d26d8328ae Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 2 Oct 2026 01:33:14 +0000 Subject: [PATCH 154/296] Backport #123099 to 26.8: Add more path validations --- .../DataLakes/Paimon/PaimonClient.cpp | 2 + .../DataLakes/Paimon/PaimonMetadata.cpp | 1 + .../ObjectStorage/DataLakes/Paimon/Utils.cpp | 14 +++++ .../ObjectStorage/DataLakes/Paimon/Utils.h | 1 + ...1_paimon_manifest_path_traversal.reference | 5 ++ .../05301_paimon_manifest_path_traversal.sh | 63 +++++++++++++++++++ 6 files changed, 86 insertions(+) create mode 100644 tests/queries/0_stateless/05301_paimon_manifest_path_traversal.reference create mode 100755 tests/queries/0_stateless/05301_paimon_manifest_path_traversal.sh diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp index 50484b2e20bdd..08fac3e0d656a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp @@ -298,6 +298,7 @@ PaimonSnapshot PaimonTableClient::getSnapshot(const std::pair & s std::pair, size_t> PaimonTableClient::getManifestMeta(String manifest_list_path, bool disable_filesystem_cache) { /// read manifest list file + Paimon::checkPathIsRelativeToTable(manifest_list_path, "manifest list"); auto context = getContext(); RelativePathWithMetadata relative_path(std::filesystem::path(table_location) / PAIMON_MANIFEST_DIR / manifest_list_path); auto read_settings = getPaimonMetadataReadSettings(disable_filesystem_cache); @@ -320,6 +321,7 @@ std::pair, size_t> PaimonTableClient::getMan PaimonManifest PaimonTableClient::getDataManifest(String manifest_path, const PaimonTableSchema & table_schema, const String & partition_default_name, bool disable_filesystem_cache) { + Paimon::checkPathIsRelativeToTable(manifest_path, "manifest"); String manifest_file_name(manifest_path.begin() + manifest_path.find_last_of('/') + 1, manifest_path.end()); if (manifest_file_name.starts_with("index-manifest-")) return {}; diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonMetadata.cpp index b97affbdeff53..89e441c7ea7ce 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonMetadata.cpp @@ -896,6 +896,7 @@ Strings PaimonMetadata::collectDataFilesFromManifests( auto manifest = getManifest(meta.file_name, snapshot_state->schema_id); for (const auto & entry : manifest->entries) { + Paimon::checkPathIsRelativeToTable(entry.file.file_name, "data file"); String file_path = (std::filesystem::path(persistent_components.table_path) / entry.file.bucket_path / entry.file.file_name); diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.cpp index a3885f3c55f6b..bc1fef9cbf519 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -29,6 +31,7 @@ namespace ErrorCodes extern const int CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER; extern const int BAD_ARGUMENTS; extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; +extern const int PATH_ACCESS_DENIED; } } namespace Paimon @@ -371,6 +374,17 @@ String getBucketPath(const String & partition, Int32 bucket, const PaimonTableSc return bucket_path; } +void checkPathIsRelativeToTable(const String & path, std::string_view kind) +{ + const std::filesystem::path fs_path(path); + if (fs_path.has_root_path() || std::ranges::any_of(fs_path, [](const auto & component) { return component == ".."; })) + throw Exception( + ErrorCodes::PATH_ACCESS_DENIED, + "Paimon {} path `{}` must be relative to the table directory and must not contain `..`", + kind, + path); +} + String concatPath(std::initializer_list paths) { if (paths.size() == 1) diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.h b/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.h index b01d8d33a9ebb..fb1171b109be5 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.h @@ -73,6 +73,7 @@ class PathEscape DB::Row getPartitionFields(const String & partition, const PaimonTableSchema & table_schema); String getBucketPath(const String & partition, Int32 bucket, const PaimonTableSchema & table_schema, const String & partition_default_name); String concatPath(std::initializer_list paths); +void checkPathIsRelativeToTable(const String & path, std::string_view kind); template void getValueFromJSON(T & t, const Poco::JSON::Object::Ptr & json, const String & key) diff --git a/tests/queries/0_stateless/05301_paimon_manifest_path_traversal.reference b/tests/queries/0_stateless/05301_paimon_manifest_path_traversal.reference new file mode 100644 index 0000000000000..ea980fefbba0e --- /dev/null +++ b/tests/queries/0_stateless/05301_paimon_manifest_path_traversal.reference @@ -0,0 +1,5 @@ +INSIDE_TABLE +PATH_ACCESS_DENIED +PATH_ACCESS_DENIED +PATH_ACCESS_DENIED +PATH_ACCESS_DENIED diff --git a/tests/queries/0_stateless/05301_paimon_manifest_path_traversal.sh b/tests/queries/0_stateless/05301_paimon_manifest_path_traversal.sh new file mode 100755 index 0000000000000..6ec04988ef6bc --- /dev/null +++ b/tests/queries/0_stateless/05301_paimon_manifest_path_traversal.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BASE_DIR="${CLICKHOUSE_USER_FILES_UNIQUE}/paimon_path_traversal" +rm -rf "${BASE_DIR}" +mkdir -p "${BASE_DIR}/outside" +echo "SECRET_OUTSIDE_TABLE" > "${BASE_DIR}/outside/secret.txt" + +# $1 - table name, $2 - manifest file name in the manifest list, $3 - data file name in the manifest +create_table() +{ + local table_dir="${BASE_DIR}/$1" + mkdir -p "${table_dir}/schema" "${table_dir}/snapshot" "${table_dir}/manifest" "${table_dir}/bucket-0" + echo "INSIDE_TABLE" > "${table_dir}/bucket-0/data.txt" + echo '{"version":3,"id":0,"highestFieldId":0,"partitionKeys":[],"primaryKeys":[],"options":{},"timeMillis":0,"fields":[{"id":0,"name":"data","type":"STRING NOT NULL"}]}' > "${table_dir}/schema/schema-0" + echo -n '1' > "${table_dir}/snapshot/LATEST" + echo '{"id":1,"schemaId":0,"baseManifestList":"manifest-list-1","deltaManifestList":"manifest-list-1","commitUser":"test","commitIdentifier":0,"commitKind":"APPEND","timeMillis":0}' > "${table_dir}/snapshot/snapshot-1" + + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION file('${table_dir}/manifest/manifest-list-1', 'Avro', + '_FILE_NAME String, _FILE_SIZE Int64, _NUM_ADDED_FILES Int64, _NUM_DELETED_FILES Int64, + _PARTITION_STATS Tuple(_MAX_VALUES String, _MIN_VALUES String, _NULL_COUNTS Array(Int64)), _SCHEMA_ID Int64') + VALUES ('$2', 1, 1, 0, ('', '', [0]), 0)" + + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION file('${table_dir}/manifest/manifest-1', 'Avro', + '_KIND Int32, _PARTITION String, _BUCKET Int32, _TOTAL_BUCKETS Int32, + _FILE Tuple(_FILE_NAME String, _FILE_SIZE Int64, _ROW_COUNT Int64, _MIN_KEY String, _MAX_KEY String, + _KEY_STATS Tuple(_MAX_VALUES String, _MIN_VALUES String, _NULL_COUNTS Array(Int64)), + _VALUE_STATS Tuple(_MAX_VALUES String, _MIN_VALUES String, _NULL_COUNTS Array(Int64)), + _MIN_SEQUENCE_NUMBER Int64, _MAX_SEQUENCE_NUMBER Int64, _SCHEMA_ID Int64, _LEVEL Int32, + _EXTRA_FILES Array(String), _CREATION_TIME Nullable(DateTime64(6)), _DELETE_ROW_COUNT Nullable(Int64), + _EMBEDDED_FILE_INDEX Nullable(String), _FILE_SOURCE Nullable(Int8), _VALUE_STATS_COLS Array(Int64))') + VALUES (0, unhex('00000000'), 0, 1, + ('$3', 13, 1, '', '', ('', '', [0]), ('', '', [0]), 0, 0, 0, 0, [], NULL, NULL, NULL, NULL, []))" +} + +read_table() +{ + ${CLICKHOUSE_CLIENT} -q "SELECT data FROM paimonLocal('${BASE_DIR}/$1', 'RawBLOB', 'data String')" 2>&1 \ + | grep -o -m1 -E "INSIDE_TABLE|SECRET_OUTSIDE_TABLE|PATH_ACCESS_DENIED" +} + +create_table valid 'manifest-1' 'data.txt' +read_table valid + +create_table data_file_dotdot 'manifest-1' '../../outside/secret.txt' +read_table data_file_dotdot + +create_table data_file_absolute 'manifest-1' "${BASE_DIR}/outside/secret.txt" +read_table data_file_absolute + +create_table manifest_dotdot '../../valid/manifest/manifest-1' 'data.txt' +read_table manifest_dotdot + +create_table manifest_absolute "${BASE_DIR}/valid/manifest/manifest-1" 'data.txt' +read_table manifest_absolute + +rm -rf "${BASE_DIR}" From 3d750e67d81a8b0e1e957e6374b5a181985cd3cf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 2 Oct 2026 04:28:23 +0000 Subject: [PATCH 155/296] Backport #122868 to 26.8: Fix a TTL check crash on a constant `arrayMap` result, reject a constant nested column in `ColumnArray` --- src/Columns/ColumnArray.cpp | 58 ++------------- src/Columns/ColumnArray.h | 8 --- src/Columns/ColumnTuple.cpp | 7 +- src/Columns/tests/gtest_column_array.cpp | 25 +++++++ src/Columns/tests/gtest_column_tuple.cpp | 70 +++++++++++++++++++ src/Storages/TTLDescription.cpp | 3 +- ...l_array_map_constant_lambda_body.reference | 3 + ...292_ttl_array_map_constant_lambda_body.sql | 20 ++++++ 8 files changed, 129 insertions(+), 65 deletions(-) create mode 100644 tests/queries/0_stateless/05292_ttl_array_map_constant_lambda_body.reference create mode 100644 tests/queries/0_stateless/05292_ttl_array_map_constant_lambda_body.sql diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index e08fcc942e810..983e8647d420e 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -59,6 +59,9 @@ ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && if (data) { + if (isColumnConst(*data)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnArray cannot have ColumnConst as its nested column"); + Offset last_offset = offsets_data.empty() ? 0 : offsets_data.back(); /// This will also prevent possible overflow in offset. @@ -79,16 +82,14 @@ ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && "offsets_column is not monotonically increasing: the offset {} at position {} is greater than the next offset {}", *non_monotonic, non_monotonic - offsets_data.begin(), *(non_monotonic + 1)); #endif - - /** NOTE - * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). - * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. - */ } ColumnArray::ColumnArray(MutableColumnPtr && nested_column) : data(std::move(nested_column)) { + if (isColumnConst(*data)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnArray cannot have ColumnConst as its nested column"); + if (!data->empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Not empty data passed to ColumnArray, but no offsets passed"); @@ -638,13 +639,6 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const } -ColumnPtr ColumnArray::convertToFullColumnIfConst() const -{ - /// It is possible to have an array with constant data and non-constant offsets. - /// Example is the result of expression: replicate('hello', [1]) - return ColumnArray::create(data->convertToFullColumnIfConst(), offsets); -} - void ColumnArray::getExtremes(Field & min, Field & max, size_t start, size_t end) const { min = Array(); @@ -1444,8 +1438,6 @@ ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const return replicateNumber(replicate_offsets); if (typeid_cast(data.get())) return replicateString(replicate_offsets); - if (typeid_cast(data.get())) - return replicateConst(replicate_offsets); if (typeid_cast(data.get())) return replicateNullable(replicate_offsets); if (typeid_cast(data.get())) @@ -1586,44 +1578,6 @@ ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const } -ColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const -{ - size_t col_size = size(); - if (col_size != replicate_offsets.size()) - throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of offsets doesn't match size of column."); - - if (0 == col_size) - return cloneEmpty(); - - const Offsets & src_offsets = getOffsets(); - - auto res_column_offsets = ColumnOffsets::create(); - Offsets & res_offsets = res_column_offsets->getData(); - res_offsets.reserve_exact(replicate_offsets.back()); - - Offset prev_replicate_offset = 0; - Offset prev_data_offset = 0; - Offset current_new_offset = 0; - - for (size_t i = 0; i < col_size; ++i) - { - size_t size_to_replicate = replicate_offsets[i] - prev_replicate_offset; - size_t value_size = src_offsets[i] - prev_data_offset; - - for (size_t j = 0; j < size_to_replicate; ++j) - { - current_new_offset += value_size; - res_offsets.push_back(current_new_offset); - } - - prev_replicate_offset = replicate_offsets[i]; - prev_data_offset = src_offsets[i]; - } - - return ColumnArray::create(getData().cloneResized(current_new_offset), std::move(res_column_offsets)); -} - - ColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const { size_t col_size = size(); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 1d4fade3dc326..50e68b18e5515 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -133,7 +133,6 @@ class ColumnArray final : public COWHelper, ColumnArr size_t allocatedBytes() const override; void protect() override; ColumnPtr replicate(const Offsets & replicate_offsets) const override; - ColumnPtr convertToFullColumnIfConst() const override; void getExtremes(Field & min, Field & max, size_t start, size_t end) const override; bool hasEqualOffsets(const ColumnArray & other) const; @@ -244,13 +243,6 @@ class ColumnArray final : public COWHelper, ColumnArr /// Multiply the values if the nested column is ColumnString. The code is too complicated. ColumnPtr replicateString(const Offsets & replicate_offsets) const; - /** Non-constant arrays of constant values are quite rare. - * Most functions can not work with them, and does not create such columns as a result. - * An exception is the function `replicate` (see FunctionsMiscellaneous.h), which has service meaning for the implementation of lambda functions. - * Only for its sake is the implementation of the `replicate` method for ColumnArray(ColumnConst). - */ - ColumnPtr replicateConst(const Offsets & replicate_offsets) const; - /** The following is done by simply replicating of nested columns. */ ColumnPtr replicateTuple(const Offsets & replicate_offsets) const; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 7e178d368a11c..ba512dc192901 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -21,7 +21,6 @@ namespace DB namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; extern const int NOT_IMPLEMENTED; extern const int CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE; extern const int LOGICAL_ERROR; @@ -54,7 +53,7 @@ ColumnTuple::ColumnTuple(MutableColumns && mutable_columns) for (auto & column : mutable_columns) { if (isColumnConst(*column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnTuple cannot have ColumnConst as its element"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnTuple cannot have ColumnConst as its element"); columns.push_back(std::move(column)); } @@ -70,7 +69,7 @@ ColumnTuple::Ptr ColumnTuple::create(const Columns & columns) for (const auto & column : columns) if (isColumnConst(*column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnTuple cannot have ColumnConst as its element"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnTuple cannot have ColumnConst as its element"); auto column_tuple = ColumnTuple::create(columns[0]->size()); column_tuple->columns.assign(columns.begin(), columns.end()); @@ -85,7 +84,7 @@ ColumnTuple::Ptr ColumnTuple::create(const TupleColumns & columns) for (const auto & column : columns) if (isColumnConst(*column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnTuple cannot have ColumnConst as its element"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnTuple cannot have ColumnConst as its element"); auto column_tuple = ColumnTuple::create(columns[0]->size()); column_tuple->columns = columns; diff --git a/src/Columns/tests/gtest_column_array.cpp b/src/Columns/tests/gtest_column_array.cpp index a9b55a5827354..c2b95118b0187 100644 --- a/src/Columns/tests/gtest_column_array.cpp +++ b/src/Columns/tests/gtest_column_array.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -23,6 +24,14 @@ ColumnArray::MutablePtr createArray(std::vector data_values, std::vector return ColumnArray::create(std::move(data), std::move(offsets)); } +/// One array of two rows of a constant: the offsets match the nested column, so only the constant is wrong. +ColumnArray::MutablePtr createArrayOverConst() +{ + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->getData().push_back(2); + return ColumnArray::create(ColumnConst::create(ColumnUInt64::create(1, 42), 2), std::move(offsets)); +} + } /// A ColumnArray is created from an already populated nested column and offsets, so its offsets @@ -64,6 +73,12 @@ TEST(ColumnArray, InconsistentOffsetsAreRejected) EXPECT_THROW(createArray({10}, {}), Exception); } +TEST(ColumnArray, ConstNestedColumnIsRejected) +{ + EXPECT_THROW(createArrayOverConst(), Exception); + EXPECT_THROW(ColumnArray::create(ColumnConst::create(ColumnUInt64::create(1, 42), 0)), Exception); +} + #endif /// A decreasing offset makes `sizeAt` underflow to a huge value even when the last offset matches @@ -86,4 +101,14 @@ TEST(ColumnArrayDeathTest, NonMonotonicOffsetsAreRejected) EXPECT_DEATH((createArray({10, 20, 30}, {3, 0, 3})), "not monotonically increasing"); } +TEST(ColumnArrayDeathTest, ConstNestedColumnIsRejected) +{ + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + EXPECT_DEATH(createArrayOverConst(), "ColumnArray cannot have ColumnConst as its nested column"); + EXPECT_DEATH( + ColumnArray::create(ColumnConst::create(ColumnUInt64::create(1, 42), 0)), + "ColumnArray cannot have ColumnConst as its nested column"); +} + #endif diff --git a/src/Columns/tests/gtest_column_tuple.cpp b/src/Columns/tests/gtest_column_tuple.cpp index cc784d9ff53ef..bf073267ae49e 100644 --- a/src/Columns/tests/gtest_column_tuple.cpp +++ b/src/Columns/tests/gtest_column_tuple.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -18,6 +19,7 @@ namespace DB namespace ErrorCodes { extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; +extern const int LOGICAL_ERROR; } } @@ -142,3 +144,71 @@ TEST(ColumnTuple, EmptyTuplePermute) ASSERT_EQ(e.code(), ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); } } + +namespace +{ + +/// Two rows of a constant next to a full element of two rows: the sizes match, so only the constant is wrong. +ColumnPtr createTupleFromMutableColumnsWithConst() +{ + MutableColumns elements; + elements.push_back(ColumnUInt64::create(2, 1)); + elements.push_back(ColumnConst::create(ColumnUInt64::create(1, 42), 2)); + return ColumnTuple::create(std::move(elements)); +} + +ColumnPtr createTupleFromColumnsWithConst() +{ + return ColumnTuple::create(Columns{ColumnUInt64::create(2, 1), ColumnConst::create(ColumnUInt64::create(1, 42), 2)}); +} + +ColumnPtr createTupleFromTupleColumnsWithConst() +{ + VectorWithMemoryTracking elements; + elements.emplace_back(ColumnUInt64::create(2, 1)); + elements.emplace_back(ColumnConst::create(ColumnUInt64::create(1, 42), 2)); + return ColumnTuple::create(elements); +} + +} + +/// Skipped under debug/sanitizers: LOGICAL_ERROR aborts there, so the exception can't be caught. +#ifndef DEBUG_OR_SANITIZER_BUILD + +namespace +{ + +int errorCodeOf(ColumnPtr (*create)()) +{ + try + { + (void)create(); + } + catch (const Exception & e) + { + return e.code(); + } + return 0; +} + +} + +TEST(ColumnTuple, ConstElementIsRejected) +{ + EXPECT_EQ(errorCodeOf(createTupleFromMutableColumnsWithConst), ErrorCodes::LOGICAL_ERROR); + EXPECT_EQ(errorCodeOf(createTupleFromColumnsWithConst), ErrorCodes::LOGICAL_ERROR); + EXPECT_EQ(errorCodeOf(createTupleFromTupleColumnsWithConst), ErrorCodes::LOGICAL_ERROR); +} + +#else + +TEST(ColumnTupleDeathTest, ConstElementIsRejected) +{ + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + EXPECT_DEATH(createTupleFromMutableColumnsWithConst(), "ColumnTuple cannot have ColumnConst as its element"); + EXPECT_DEATH(createTupleFromColumnsWithConst(), "ColumnTuple cannot have ColumnConst as its element"); + EXPECT_DEATH(createTupleFromTupleColumnsWithConst(), "ColumnTuple cannot have ColumnConst as its element"); +} + +#endif diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 3c06a812616bd..086002a38a3e4 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -715,7 +715,8 @@ std::vector checkActionsDAGForAggregateFunctions( { auto offsets = ColumnArray::ColumnOffsets::create(); offsets->getData().push_back(1); - candidates.push_back(ColumnArray::create(element->cloneResized(1), std::move(offsets))); + candidates.push_back( + ColumnArray::create(element->convertToFullColumnIfConst()->cloneResized(1), std::move(offsets))); } } else diff --git a/tests/queries/0_stateless/05292_ttl_array_map_constant_lambda_body.reference b/tests/queries/0_stateless/05292_ttl_array_map_constant_lambda_body.reference new file mode 100644 index 0000000000000..e8183f05f5db6 --- /dev/null +++ b/tests/queries/0_stateless/05292_ttl_array_map_constant_lambda_body.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/tests/queries/0_stateless/05292_ttl_array_map_constant_lambda_body.sql b/tests/queries/0_stateless/05292_ttl_array_map_constant_lambda_body.sql new file mode 100644 index 0000000000000..85d82c894c131 --- /dev/null +++ b/tests/queries/0_stateless/05292_ttl_array_map_constant_lambda_body.sql @@ -0,0 +1,20 @@ +-- A TTL expression that reads the result of `arrayMap` whose lambda returns a constant `Dynamic` value. + +-- With `cast_keep_nullable`, `toUInt8` of a `Dynamic` value is `Nullable`, and a TTL expression cannot be `Nullable`. +SET cast_keep_nullable = 0; + +CREATE TABLE t_ttl_array_map_length (d DateTime, arr Array(UInt32)) ENGINE = MergeTree ORDER BY tuple() +TTL d + toIntervalDay(length(arrayMap(x -> CAST(1, 'Dynamic'), arr))); +INSERT INTO t_ttl_array_map_length VALUES ('2100-01-01 00:00:00', [1]); +SELECT count() FROM t_ttl_array_map_length; + +CREATE TABLE t_ttl_array_map_element (d DateTime, arr Array(UInt32)) ENGINE = MergeTree ORDER BY tuple() +TTL d + toIntervalDay(toUInt8(arrayMap(x -> CAST(1, 'Dynamic'), arr)[1])); +INSERT INTO t_ttl_array_map_element VALUES ('2100-01-01 00:00:00', [1]); +SELECT count() FROM t_ttl_array_map_element; + +ALTER TABLE t_ttl_array_map_element MODIFY TTL d + toIntervalDay(toUInt8(arrayMap(x -> CAST(2, 'Dynamic'), arr)[1])); +SELECT count() FROM t_ttl_array_map_element; + +DROP TABLE t_ttl_array_map_length; +DROP TABLE t_ttl_array_map_element; From e25638152729d67db9ef78dc0e7d1a144221ef4b Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Fri, 2 Oct 2026 09:23:37 +0200 Subject: [PATCH 156/296] Fix analyzer setting in views over `Distributed` Keep the analyzer disabled in `InterpreterSelectQuery` after applying a stored `SELECT`'s settings. This prevents `Distributed` reads from entering analyzer-only paths without a planner context. --- src/Interpreters/InterpreterSelectQuery.cpp | 9 ++++ ...nalyzer_setting_over_distributed.reference | 16 ++++++ ...view_analyzer_setting_over_distributed.sql | 49 +++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 tests/queries/0_stateless/05315_view_analyzer_setting_over_distributed.reference create mode 100644 tests/queries/0_stateless/05315_view_analyzer_setting_over_distributed.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 4de94dd46ea08..578fb66e392b6 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -139,6 +139,7 @@ namespace Setting extern const SettingsUInt64 aggregation_in_order_max_block_bytes; extern const SettingsUInt64 aggregation_memory_efficient_merge_threads; extern const SettingsBool allow_calculating_subcolumns_sizes_for_merge_tree_reading; + extern const SettingsBool allow_experimental_analyzer; extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; extern const SettingsUInt64 automatic_parallel_replicas_mode; extern const SettingsBool async_socket_for_remote; @@ -3949,8 +3950,16 @@ void InterpreterSelectQuery::initSettings() { auto & query = getSelectQuery(); if (query.settings()) + { InterpreterSetQuery(query.settings(), context).executeForCurrentContext(options.ignore_setting_constraints); + /// The old interpreter disabled the analyzer in `IInterpreterUnionOrSelectQuery`, but a `SELECT` + /// stored in a `VIEW` may enable it again here. Storages must not use analyzer-only query tree + /// paths (for example, `StorageDistributed::read`) when this interpreter built the query. + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) + context->setSetting("allow_experimental_analyzer", false); + } + const auto & client_info = context->getClientInfo(); if (client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && diff --git a/tests/queries/0_stateless/05315_view_analyzer_setting_over_distributed.reference b/tests/queries/0_stateless/05315_view_analyzer_setting_over_distributed.reference new file mode 100644 index 0000000000000..071137aa6b5d3 --- /dev/null +++ b/tests/queries/0_stateless/05315_view_analyzer_setting_over_distributed.reference @@ -0,0 +1,16 @@ +view analyzer 1, read analyzer 0 +2 +s1 2.5 +s2 7 +view analyzer 1, read analyzer 0, remote shard +s1 2.5 +s2 7 +view analyzer 1, read analyzer 1 +s1 2.5 +s2 7 +view analyzer 0, read analyzer 1 +s1 2.5 +s2 7 +view analyzer 0, read analyzer 0 +s1 2.5 +s2 7 diff --git a/tests/queries/0_stateless/05315_view_analyzer_setting_over_distributed.sql b/tests/queries/0_stateless/05315_view_analyzer_setting_over_distributed.sql new file mode 100644 index 0000000000000..a85ac964d1592 --- /dev/null +++ b/tests/queries/0_stateless/05315_view_analyzer_setting_over_distributed.sql @@ -0,0 +1,49 @@ +-- Tags: distributed + +-- The old interpreter must not use the analyzer-only `Distributed` path when a view's +-- stored `SELECT` has `SETTINGS enable_analyzer = 1`. + +DROP VIEW IF EXISTS v_new; +DROP VIEW IF EXISTS v_old; +DROP TABLE IF EXISTS dist; +DROP TABLE IF EXISTS src; + +CREATE TABLE src (trade_date Date, strategy String, pnl Float64, ts DateTime) ENGINE = Memory; +INSERT INTO src VALUES + ('2026-09-17', 's1', 1.5, '2026-09-17 10:00:00'), + ('2026-09-17', 's1', 2.5, '2026-09-17 11:00:00'), + ('2026-09-17', 's2', 7, '2026-09-17 09:00:00'), + ('2026-09-16', 's3', 9, '2026-09-16 10:00:00'); + +CREATE TABLE dist AS src ENGINE = Distributed(test_shard_localhost, currentDatabase(), src); + +CREATE VIEW v_new AS + SELECT strategy, argMax(pnl, ts) AS pnl + FROM dist WHERE trade_date = '2026-09-17' + GROUP BY strategy SETTINGS enable_analyzer = 1; + +CREATE VIEW v_old AS + SELECT strategy, argMax(pnl, ts) AS pnl + FROM dist WHERE trade_date = '2026-09-17' + GROUP BY strategy SETTINGS enable_analyzer = 0; + +SELECT 'view analyzer 1, read analyzer 0'; +SELECT count() FROM v_new SETTINGS enable_analyzer = 0; +SELECT strategy, pnl FROM v_new ORDER BY strategy SETTINGS enable_analyzer = 0; + +SELECT 'view analyzer 1, read analyzer 0, remote shard'; +SELECT strategy, pnl FROM v_new ORDER BY strategy SETTINGS enable_analyzer = 0, prefer_localhost_replica = 0; + +SELECT 'view analyzer 1, read analyzer 1'; +SELECT strategy, pnl FROM v_new ORDER BY strategy SETTINGS enable_analyzer = 1; + +SELECT 'view analyzer 0, read analyzer 1'; +SELECT strategy, pnl FROM v_old ORDER BY strategy SETTINGS enable_analyzer = 1; + +SELECT 'view analyzer 0, read analyzer 0'; +SELECT strategy, pnl FROM v_old ORDER BY strategy SETTINGS enable_analyzer = 0; + +DROP VIEW v_new; +DROP VIEW v_old; +DROP TABLE dist; +DROP TABLE src; From 42edff16bcd01db2e7b285abe47592e82e0f8844 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Fri, 2 Oct 2026 09:39:12 +0200 Subject: [PATCH 157/296] Fix `ALIAS` row policies on legacy distributed reads Expand table aliases before building row-policy filter actions on shards. Keep row-policy inputs separate from `PREWHERE` inputs so columns needed by selected aliases remain available. --- src/Interpreters/InterpreterSelectQuery.cpp | 26 +++--- ..._policy_alias_column_distributed.reference | 38 ++++++++ ...16_row_policy_alias_column_distributed.sql | 92 +++++++++++++++++++ 3 files changed, 143 insertions(+), 13 deletions(-) create mode 100644 tests/queries/0_stateless/05316_row_policy_alias_column_distributed.reference create mode 100644 tests/queries/0_stateless/05316_row_policy_alias_column_distributed.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 4de94dd46ea08..ba1062256cba6 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -303,10 +303,13 @@ try ASTs select_expressions; - /// The first column is our filter expression. - /// the row_policy_filter_expression should be cloned, because it may be changed by TreeRewriter. - /// which make it possible an invalid expression, although it may be valid in whole select. - select_expressions.push_back(row_policy_filter_expression->clone()); + /// The first column is our filter expression. Clone it because `TreeRewriter` can change the AST. + auto filter_expression = row_policy_filter_expression->clone(); + + /// `TreeRewriter` expands table aliases only on the initiator, but these filters are also + /// created on shards. An `ALIAS` used by a row policy must be evaluated before the read filter. + replaceAliasColumnsInQuery(filter_expression, metadata_snapshot->getColumns(), {}, context); + select_expressions.push_back(std::move(filter_expression)); /// Keep columns that are required after the filter actions. for (const auto & column_str : prerequisite_columns) @@ -2626,7 +2629,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() } /// Set of all (including ALIAS) required columns for PREWHERE - auto get_prewhere_columns = [&]() + auto get_prewhere_columns = [&](bool include_row_level_filter) { NameSet columns; @@ -2639,7 +2642,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() /// A row policy that will not be pushed into the storage read is applied as an ordinary /// FilterStep above it, so its columns are read normally and are not PREWHERE columns. - if (row_level_filter && shouldPushRowLevelFilterToStorage()) + if (row_level_filter && include_row_level_filter && shouldPushRowLevelFilterToStorage()) { auto row_level_required_columns = row_level_filter->actions.getRequiredColumns().getNames(); columns.insert(row_level_required_columns.begin(), row_level_required_columns.end()); @@ -2657,15 +2660,14 @@ void InterpreterSelectQuery::addPrewhereAliasActions() /// before any other executions. if (alias_columns_required) { - NameSet required_columns_from_prewhere = get_prewhere_columns(); + /// The row-level filter runs before `PREWHERE`, but its inputs are not produced by + /// `PREWHERE`. Keep them in the alias actions for queries that still need them. + NameSet required_columns_from_prewhere = get_prewhere_columns(/*include_row_level_filter=*/ false); NameSet required_aliases_from_prewhere; /// Set of ALIAS required columns for PREWHERE /// Expression, that contains all raw required columns ASTPtr required_columns_all_expr = make_intrusive(); - /// Expression, that contains raw required columns for PREWHERE - ASTPtr required_columns_from_prewhere_expr = make_intrusive(); - /// Sort out already known required columns between expressions, /// also populate `required_aliases_from_prewhere`. for (const auto & column : required_columns) @@ -2690,8 +2692,6 @@ void InterpreterSelectQuery::addPrewhereAliasActions() if (required_columns_from_prewhere.contains(column)) { - required_columns_from_prewhere_expr->children.emplace_back(std::move(column_expr)); - if (is_alias) required_aliases_from_prewhere.insert(column); } @@ -2759,7 +2759,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() const auto & supported_prewhere_columns = storage->supportedPrewhereColumns(); if (supported_prewhere_columns.has_value()) { - NameSet required_columns_from_prewhere = get_prewhere_columns(); + NameSet required_columns_from_prewhere = get_prewhere_columns(/*include_row_level_filter=*/ true); const auto & table_columns = metadata_snapshot->getColumns(); const bool include_subcolumns = storage->supportedPrewhereColumnsIncludeSubcolumns(); diff --git a/tests/queries/0_stateless/05316_row_policy_alias_column_distributed.reference b/tests/queries/0_stateless/05316_row_policy_alias_column_distributed.reference new file mode 100644 index 0000000000000..565151239d363 --- /dev/null +++ b/tests/queries/0_stateless/05316_row_policy_alias_column_distributed.reference @@ -0,0 +1,38 @@ +baseline local +1 +2 +3 +4 +baseline distributed +1 +2 +3 +4 +policy local +2 +policy distributed local replica +2 +policy distributed remote replica +2 +policy distributed with PREWHERE +2 +policy local without alias optimization +2 +policy local with aliases selected +2 20 ok20 +policy local with aliases selected and PREWHERE +2 20 ok20 +policy distributed with aliases selected +2 20 ok20 +policy distributed with aliases selected and PREWHERE +2 20 ok20 +policy distributed new analyzer +2 +policy with only an ALIAS predicate +1 2 +policy with inline expression +2 +additional filter on ALIAS +2 +3 +4 diff --git a/tests/queries/0_stateless/05316_row_policy_alias_column_distributed.sql b/tests/queries/0_stateless/05316_row_policy_alias_column_distributed.sql new file mode 100644 index 0000000000000..91e7d1360b03f --- /dev/null +++ b/tests/queries/0_stateless/05316_row_policy_alias_column_distributed.sql @@ -0,0 +1,92 @@ +-- Tags: distributed + +SET enable_analyzer = 0; + +DROP ROW POLICY IF EXISTS rp ON t; +DROP TABLE IF EXISTS t_dist; +DROP TABLE IF EXISTS t; + +CREATE TABLE t +( + k UInt64, + team String, + a UInt64 ALIAS k * 10, + a2 String ALIAS concat(team, toString(a)) +) ENGINE = MergeTree ORDER BY k; + +INSERT INTO t VALUES (1, 'ok'), (2, 'ok'), (3, 'ok'), (4, 'no'); + +CREATE TABLE t_dist AS t +ENGINE = Distributed('test_shard_localhost', currentDatabase(), 't', rand()); + +SELECT 'baseline local'; +SELECT k FROM t ORDER BY k; + +SELECT 'baseline distributed'; +SELECT k FROM t_dist ORDER BY k SETTINGS prefer_localhost_replica = 0; + +-- 1 fails the `ALIAS` condition; 3 fails the nested `ALIAS` condition; +-- 4 passes both `ALIAS` conditions but fails the physical team condition. +CREATE ROW POLICY rp ON t FOR SELECT +USING team = 'ok' AND a >= 20 AND a2 != 'ok30' TO ALL; + +SELECT 'policy local'; +SELECT k FROM t ORDER BY k; + +SELECT 'policy distributed local replica'; +SELECT k FROM t_dist ORDER BY k SETTINGS prefer_localhost_replica = 1; + +SELECT 'policy distributed remote replica'; +SELECT k FROM t_dist ORDER BY k SETTINGS prefer_localhost_replica = 0; + +SELECT 'policy distributed with PREWHERE'; +SELECT k FROM t_dist PREWHERE k >= 1 WHERE k <= 4 ORDER BY k +SETTINGS prefer_localhost_replica = 0; + +SELECT 'policy local without alias optimization'; +SELECT k FROM t ORDER BY k SETTINGS optimize_respect_aliases = 0; + +SELECT 'policy local with aliases selected'; +SELECT k, a, a2 FROM t ORDER BY k SETTINGS optimize_respect_aliases = 0; + +-- `PREWHERE` uses team, while both the policy and the selected aliases need k. +-- The alias step must preserve k even when a separate `PREWHERE` step exists. +SELECT 'policy local with aliases selected and PREWHERE'; +SELECT k, a, a2 FROM t PREWHERE team != 'no' ORDER BY k +SETTINGS optimize_respect_aliases = 0; + +SELECT 'policy distributed with aliases selected'; +SELECT k, a, a2 FROM t_dist ORDER BY k SETTINGS prefer_localhost_replica = 0; + +SELECT 'policy distributed with aliases selected and PREWHERE'; +SELECT k, a, a2 FROM t_dist PREWHERE team != 'no' ORDER BY k +SETTINGS prefer_localhost_replica = 0, optimize_respect_aliases = 0; + +SELECT 'policy distributed new analyzer'; +SELECT k FROM t_dist ORDER BY k +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 0; + +-- A policy whose only predicate references an `ALIAS` still needs its value +-- before the row-level filter, even when the `SELECT` only needs `count`. +CREATE ROW POLICY OR REPLACE rp ON t FOR SELECT +USING a2 = 'ok20' TO ALL; + +SELECT 'policy with only an ALIAS predicate'; +SELECT count(), sum(k) FROM t_dist SETTINGS prefer_localhost_replica = 0; + +-- The equivalent expression is a control for the alias substitution. +CREATE ROW POLICY OR REPLACE rp ON t FOR SELECT +USING team = 'ok' AND k * 10 >= 20 AND concat(team, toString(k * 10)) != 'ok30' TO ALL; + +SELECT 'policy with inline expression'; +SELECT k FROM t_dist ORDER BY k SETTINGS prefer_localhost_replica = 0; + +DROP ROW POLICY rp ON t; + +-- Additional table filters use the same filter-action construction path. +SELECT 'additional filter on ALIAS'; +SELECT k FROM t ORDER BY k +SETTINGS optimize_respect_aliases = 0, additional_table_filters = {'t': 'a >= 20'}; + +DROP TABLE t_dist; +DROP TABLE t; From 89532b91a3329e9e8882c6efe553066257de5699 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 2 Oct 2026 08:05:09 +0000 Subject: [PATCH 158/296] Backport #123337 to 26.8: Fix a crash when reading an ORC file whose root type is not a struct --- .../Impl/NativeORCBlockInputFormat.cpp | 15 ++++++- .../Formats/Impl/NativeORCBlockInputFormat.h | 3 ++ src/Storages/Hive/HiveFile.cpp | 2 +- .../05316_orc_non_struct_root.reference | 6 +++ .../0_stateless/05316_orc_non_struct_root.sh | 38 ++++++++++++++++++ .../data_orc/non_struct_root_array.orc | Bin 0 -> 230 bytes .../data_orc/non_struct_root_bigint.orc | Bin 0 -> 164 bytes 7 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05316_orc_non_struct_root.reference create mode 100755 tests/queries/0_stateless/05316_orc_non_struct_root.sh create mode 100644 tests/queries/0_stateless/data_orc/non_struct_root_array.orc create mode 100644 tests/queries/0_stateless/data_orc/non_struct_root_bigint.orc diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index 20096b6f91d17..aa7115021e73a 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -1015,6 +1015,17 @@ std::unique_ptr buildORCSearchArgument( return builder->build(); } +std::unique_ptr createORCReader(std::unique_ptr stream, const orc::ReaderOptions & options) +{ + auto reader = orc::createReader(std::move(stream), options); + if (reader->getType().getKind() != orc::STRUCT) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "ORC files whose root type is not a struct are not supported, the file has root type {}", + reader->getType().toString()); + return reader; +} + static void getFileReader( ReadBuffer & in, std::unique_ptr & file_reader, @@ -1036,7 +1047,7 @@ static void getFileReader( options.setCacheOptions(orc::CacheOptions{.holeSizeLimit = hole_size_limit, .rangeSizeLimit = range_size_limit}); auto input_stream = asORCInputStream(in, format_settings, use_prefetch, is_stopped); - file_reader = orc::createReader(std::move(input_stream), options); + file_reader = createORCReader(std::move(input_stream), options); } static const orc::Type * @@ -1519,7 +1530,7 @@ void ORCColumnToCHColumn::orcTableToCHChunk( { const auto * struct_batch = dynamic_cast(table); if (!struct_batch) - throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC table must be StructVectorBatch but is {}", struct_batch->toString()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC table must be StructVectorBatch but is {}", table->toString()); if (schema->getSubtypeCount() != struct_batch->fields.size()) throw Exception( diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h index 70df42d54ead2..49a29a7d85179 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h @@ -63,6 +63,9 @@ std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in /// instead of returning a null pointer that the library dereferences. orc::MemoryPool & getORCMemoryPool(); +/// Creates an ORC file reader; throws INCORRECT_DATA if the root type of the file is not a struct. +std::unique_ptr createORCReader(std::unique_ptr stream, const orc::ReaderOptions & options); + std::unique_ptr buildORCSearchArgument( const KeyCondition & key_condition, const Block & header, const orc::Type & schema, const FormatSettings & format_settings); diff --git a/src/Storages/Hive/HiveFile.cpp b/src/Storages/Hive/HiveFile.cpp index 6dfd1f78e73f4..378bb0840ef73 100644 --- a/src/Storages/Hive/HiveFile.cpp +++ b/src/Storages/Hive/HiveFile.cpp @@ -161,7 +161,7 @@ void HiveORCFile::prepareReader() std::atomic is_stopped{0}; orc::ReaderOptions options; options.setMemoryPool(getORCMemoryPool()); - reader = orc::createReader(asORCInputStream(*in, format_settings, /*use_prefetch=*/false, is_stopped), options); + reader = createORCReader(asORCInputStream(*in, format_settings, /*use_prefetch=*/false, is_stopped), options); } void HiveORCFile::prepareColumnMapping() diff --git a/tests/queries/0_stateless/05316_orc_non_struct_root.reference b/tests/queries/0_stateless/05316_orc_non_struct_root.reference new file mode 100644 index 0000000000000..dd8297685ff4d --- /dev/null +++ b/tests/queries/0_stateless/05316_orc_non_struct_root.reference @@ -0,0 +1,6 @@ +INCORRECT_DATA +INCORRECT_DATA +ORC files whose root type is not a struct are not supported +INCORRECT_DATA +the file has root type bigint +10 diff --git a/tests/queries/0_stateless/05316_orc_non_struct_root.sh b/tests/queries/0_stateless/05316_orc_non_struct_root.sh new file mode 100755 index 0000000000000..ccc6d5b8079b9 --- /dev/null +++ b/tests/queries/0_stateless/05316_orc_non_struct_root.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Reading an ORC file whose root type is not a struct must fail with an error instead of crashing. +# The fixtures have the root types `bigint` and `array`. +BIGINT_ROOT="$CUR_DIR/data_orc/non_struct_root_bigint.orc" +ARRAY_ROOT="$CUR_DIR/data_orc/non_struct_root_array.orc" + +check() +{ + local query="$1" expected="$2" + local out + out=$($CLICKHOUSE_LOCAL --query "$query" 2>&1) + if [ -z "$out" ]; then + echo "no output at all" + else + echo "$out" | grep -o -m1 "$expected" || echo "unexpected: $out" + fi +} + +check "SELECT * FROM file('$BIGINT_ROOT', ORC, 'x Int64') FORMAT Null" 'INCORRECT_DATA' +check "SELECT count() FROM file('$BIGINT_ROOT', ORC, 'x Int64')" 'INCORRECT_DATA' +check "DESCRIBE file('$ARRAY_ROOT', ORC)" 'ORC files whose root type is not a struct are not supported' +check "SELECT * FROM file('$ARRAY_ROOT', ORC, 'x Array(Int64)') FORMAT Null" 'INCORRECT_DATA' + +# The message names the root type of the file. +check "SELECT * FROM file('$BIGINT_ROOT', ORC, 'x Int64') FORMAT Null" 'the file has root type bigint' + +# A regular ORC file, whose root type is a struct, is still read. +STRUCT_ROOT="$CLICKHOUSE_TMP/05316_struct_root_${CLICKHOUSE_DATABASE}.orc" +rm -f "$STRUCT_ROOT" +$CLICKHOUSE_LOCAL --query "INSERT INTO FUNCTION file('$STRUCT_ROOT', ORC) SELECT number AS x FROM numbers(5)" +$CLICKHOUSE_LOCAL --query "SELECT sum(x) FROM file('$STRUCT_ROOT', ORC)" +rm -f "$STRUCT_ROOT" diff --git a/tests/queries/0_stateless/data_orc/non_struct_root_array.orc b/tests/queries/0_stateless/data_orc/non_struct_root_array.orc new file mode 100644 index 0000000000000000000000000000000000000000..e1f883326d83f31274dc9e7a67be5cd18a975574 GIT binary patch literal 230 zcmeYda#m(w;Ns_EW?*0t;^1HoU`S$;V9*AN2}8tqIM{^PI2Z)jB!mJOSb?gTxEUA@ zu`)0TdN43NPI{amoG?M)A=AYkAt8rZoF~#*FZDRswF_`}@GCBz;p{wz(G+Nw1Q*&diFplDJ3M~LV`g;z@rBVjT?AothKP1!7MRp zhKj_R3X2;pH!LJrZm6^{Nyw=Es5DU9DD_hE)l3Ek4nF}#iG~IiB_;+Q4Os)WrZ3FQ I{z1;-0DM?Fr2qf` literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/non_struct_root_bigint.orc b/tests/queries/0_stateless/data_orc/non_struct_root_bigint.orc new file mode 100644 index 0000000000000000000000000000000000000000..77e2dbe03c2b2149294bc57687cb2ab89a6e1ca1 GIT binary patch literal 164 zcmeYda@J;G;1cFyW?*0t;^ANwV&h;C;E)grVBln6IKaZdWDb;O<6skDkdOe8xsAs}qDZnVv(7>X^#K5B=Y{1s^g_+qu$XOf!R1Og3 literal 0 HcmV?d00001 From 5d99a3b2747fcbd6d2711aaf590776a5b5ef24a1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 2 Oct 2026 08:07:25 +0000 Subject: [PATCH 159/296] Backport #123336 to 26.8: Do not dereference a null `logs_queue` in `OwnSplitChannel::log` --- src/Loggers/OwnSplitChannel.cpp | 3 ++- ...05316_local_sync_logger_without_channels.reference | 2 ++ .../05316_local_sync_logger_without_channels.sh | 11 +++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05316_local_sync_logger_without_channels.reference create mode 100755 tests/queries/0_stateless/05316_local_sync_logger_without_channels.sh diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp index f35fbb8f8b65d..547fdd7f4f73d 100644 --- a/src/Loggers/OwnSplitChannel.cpp +++ b/src/Loggers/OwnSplitChannel.cpp @@ -65,7 +65,8 @@ void OwnSplitChannel::log(Poco::Message && msg) return; const auto & logs_queue = CurrentThread::getInternalTextLogsQueue(); - if (channels.empty() && (logs_queue == nullptr && !logs_queue->isNeeded(msg.getPriority(), msg.getSource()))) + if (channels.empty() && !text_log_max_priority.load(std::memory_order_relaxed) + && (logs_queue == nullptr || !logs_queue->isNeeded(msg.getPriority(), msg.getSource()))) return; if (const auto & masker = SensitiveDataMasker::getInstance()) diff --git a/tests/queries/0_stateless/05316_local_sync_logger_without_channels.reference b/tests/queries/0_stateless/05316_local_sync_logger_without_channels.reference new file mode 100644 index 0000000000000..6ed281c757a96 --- /dev/null +++ b/tests/queries/0_stateless/05316_local_sync_logger_without_channels.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/05316_local_sync_logger_without_channels.sh b/tests/queries/0_stateless/05316_local_sync_logger_without_channels.sh new file mode 100755 index 0000000000000..3ef19f954157f --- /dev/null +++ b/tests/queries/0_stateless/05316_local_sync_logger_without_channels.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Synchronous logger with no log destination and one logger level raised explicitly. +$CLICKHOUSE_LOCAL --query "SELECT 1" -- --logger.async=0 --logger.levels.Application=trace + +# Client logs requested with send_logs_level still arrive when the logger has no destination. +$CLICKHOUSE_LOCAL --query "SELECT 1" --send_logs_level=trace -- --logger.async=0 2>&1 >/dev/null | grep -F ' executeQuery: ' | grep -c -F 'SELECT 1' From 32a05457cefab196e2852fcb83fb1d1afcab8fcf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 2 Oct 2026 08:48:22 +0000 Subject: [PATCH 160/296] Backport #120313 to 26.8: Hide secret arguments in `EXPLAIN AST` output --- src/Interpreters/InterpreterExplainQuery.cpp | 168 ++++++++++++++++++ .../05219_explain_ast_hide_secrets.reference | 101 +++++++++++ .../05219_explain_ast_hide_secrets.sh | 64 +++++++ 3 files changed, 333 insertions(+) create mode 100644 tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference create mode 100755 tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 1f2b0e376ad39..239cf6c75a235 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -28,9 +28,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -61,10 +63,13 @@ #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -336,6 +341,161 @@ namespace } }; + /// Replace a node with a single `'[HIDDEN]'` literal, keeping its alias. + void hideWholeNode(ASTPtr & node) + { + auto hidden = make_intrusive(Field("[HIDDEN]")); + hidden->setAlias(node->tryGetAlias()); + node = std::move(hidden); + } + + /// Replace every literal inside a node with `'[HIDDEN]'`, keeping the expression structure. Only + /// for the secret arguments of `encrypt` / `HMAC`, where the shape is not a secret (a key built as + /// `leftPad('...', 16, '*')` stays readable as such); every other secret slot is hidden whole. + void hideLiteralsInSubtree(ASTPtr & node) + { + if (node->as()) + { + hideWholeNode(node); + return; + } + for (auto & child : node->children) + hideLiteralsInSubtree(child); + } + + /// Keep in sync with the names `FunctionSecretArgumentsFinder` sends to `findEncryptionFunctionSecretArguments` + /// and `findHMACSecretArguments`. A name missing here only makes the dump stricter: its span is hidden whole. + bool isEncryptionOrHMACFunction(const ASTFunction & function) + { + return function.name == "encrypt" || function.name == "decrypt" || function.name == "aes_encrypt_mysql" + || function.name == "aes_decrypt_mysql" || function.name == "tryDecrypt" || equalsCaseInsensitive(function.name, "HMAC"); + } + + bool isKeyValueArgument(const IAST & node) + { + const auto * function = node.as(); + return function && function->name == "equals" && function->arguments && function->arguments->children.size() == 2; + } + + /// The secret value of a `key = value` argument is its second child; anything else carries the + /// secret in the node itself. + ASTPtr & secretValueSlot(ASTPtr & node) + { + if (isKeyValueArgument(*node)) + return node->as()->arguments->children[1]; + return node; + } + + /// Replace an argument with the partially masked SQL the formatter prints for it: a URL with its + /// credentials removed, or the reconstructed `S3(...)` destination of a `Backup` database. The + /// finder builds the text from literals it read, so it parses. If it does not, the original node + /// must not stay in the tree; the argument is hidden whole (fail closed). + void replaceWithMaskedText(ASTPtr & node, const String & text) + { + ParserExpression parser; + const char * pos = text.data(); + String error; + ASTPtr parsed = tryParseQuery( + parser, + pos, + text.data() + text.size(), + error, + /* hilite= */ false, + "masked secret argument", + /* allow_multi_statements= */ false, + /* max_query_size= */ 0, + DBMS_DEFAULT_MAX_PARSER_DEPTH, + DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, + /* skip_insignificant= */ true); + if (!parsed) + { + hideWholeNode(node); + return; + } + parsed->setAlias(node->tryGetAlias()); + node = std::move(parsed); + } + + /// `DumpASTNode` prints a literal through `IAST::getID`, value included, so the dump cannot hide + /// secrets while formatting as `ASTFunction::formatImpl` does. Hide them in the tree instead. As in the + /// formatter, a secret slot becomes one `'[HIDDEN]'` literal. That includes the slots the finder could + /// not inspect (a url built by `concat(...)`, an identifier in a password slot): their expression is + /// part of the secret and must not be dumped node by node. Only the `encrypt` / `HMAC` span keeps its + /// structure (see `hideLiteralsInSubtree`). All values of a nested map (`headers(...)`, + /// `extra_credentials(...)`) are hidden; the formatter keeps the non-secret `extra_credentials` + /// values, so the dump is stricter. + struct HideSecretArgumentsMatcher + { + struct Data + { + }; + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data &) + { + auto * function = ast->as(); + if (!function || !function->arguments) + return; + + auto secret_arguments = FunctionSecretArgumentsFinderAST(*function).getResult(); + if (!secret_arguments.hasSecrets()) + return; + + auto & arguments = function->arguments->children; + for (size_t i = 0; i < arguments.size(); ++i) + { + if (auto * map = arguments[i]->as(); + map && map->arguments && std::ranges::contains(secret_arguments.nested_maps, map->name)) + { + for (auto & entry : map->arguments->children) + hideWholeNode(secretValueSlot(entry)); + continue; + } + + if (auto replaced = secret_arguments.replaced_arguments.find(i); replaced != secret_arguments.replaced_arguments.end()) + { + replaceWithMaskedText(arguments[i], replaced->second); + continue; + } + + /// An individually masked argument: only the named `key = value` form keeps its key. + if (auto masked = secret_arguments.masked_arguments.find(i); masked != secret_arguments.masked_arguments.end()) + { + hideWholeNode(masked->second ? secretValueSlot(arguments[i]) : arguments[i]); + continue; + } + + if (!(secret_arguments.start <= i && i < secret_arguments.start + secret_arguments.count)) + continue; + + if (!secret_arguments.replacement.empty()) + { + const auto text + = secret_arguments.quote_replacement ? quoteString(secret_arguments.replacement) : secret_arguments.replacement; + replaceWithMaskedText(secret_arguments.are_named ? secretValueSlot(arguments[i]) : arguments[i], text); + continue; + } + + if (secret_arguments.are_named) + { + hideWholeNode(secretValueSlot(arguments[i])); + continue; + } + + /// Only the span of `encrypt` / `HMAC` keeps its structure. Any other unnamed span without a + /// replacement, such as an unreadable url in `mongodb(concat(...), 'c')`, is hidden whole. So is a + /// `key = value` in the span: it is a positional secret written as a comparison. + if (isEncryptionOrHMACFunction(*function) && !isKeyValueArgument(*arguments[i])) + hideLiteralsInSubtree(arguments[i]); + else + hideWholeNode(arguments[i]); + } + } + }; + + using HideSecretArgumentsVisitor = InDepthNodeVisitor; + bool hasSecretsInActionsDAG(const ActionsDAG & dag) { for (const auto & node : dag.getNodes()) @@ -996,6 +1156,14 @@ QueryPipeline InterpreterExplainQuery::executeImpl() ExplainAnalyzedSyntaxVisitor(data).visit(query); } + /// `optimize = 1` inlines views the user may read but whose secrets they may not see. + /// Hide them under the same gate as `SHOW CREATE`. + if (!canDisplaySecrets(query_context)) + { + HideSecretArgumentsVisitor::Data data; + HideSecretArgumentsVisitor(data).visit(query); + } + if (settings.graph) dumpASTInDotFormat(*ast.getExplainedQuery(), buf); else diff --git a/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference new file mode 100644 index 0000000000000..405f09ad316dd --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference @@ -0,0 +1,101 @@ +-- the inlined view body hides the key +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Identifier encrypted_secret + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Function hex (alias encrypted_secret) (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 3) + Literal \'aes-128-ecb\' + Identifier secret + Literal \'[HIDDEN]\' + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + TableIdentifier default.private_plaintext +-- the session setting alone does not disclose it + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' +-- the graph dump hides it too +Literal \'[HIDDEN]\' +-- a secret typed into the explained query itself is hidden as well +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 4) + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' + Function leftPad (children 1) + ExpressionList (children 3) + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' +-- a positional secret written as a comparison collapses to one node, not to 'Function equals' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 3) + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' +-- a url the finder cannot read collapses to one node, not to 'Function concat' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function url (children 1) + ExpressionList (children 1) + Literal \'[HIDDEN]\' +-- a mongodb url the finder cannot read collapses to one node, not to 'Function concat' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function mongodb (children 1) + ExpressionList (children 2) + Literal \'[HIDDEN]\' + Literal \'c\' +-- the same url as a named override keeps its key, the value collapses +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function url (children 1) + ExpressionList (children 2) + Identifier creds + Function equals (children 1) + ExpressionList (children 2) + Identifier url + Literal \'[HIDDEN]\' +-- a nested map keeps its keys and hides its values + Literal \'http://x/f\' + Literal \'Authorization\' + Literal \'[HIDDEN]\' +-- the view itself stays usable for the restricted user +1 diff --git a/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh new file mode 100755 index 0000000000000..dcc3ca50f8c08 --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: the encryption functions are not available in the fast test build +# Tag no-replicated-database: SQL SECURITY DEFINER views and users are set up per-test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `EXPLAIN AST optimize = 1` inlines the body of a view the user may only SELECT from, and the dump +# prints every literal verbatim. The secret arguments must be hidden as `SHOW CREATE` hides them. +# The stateless test server keeps `display_secrets_in_show_and_select` off, so the gate always hides here. + +user="user_05219_${CLICKHOUSE_DATABASE}_$RANDOM" +db=${CLICKHOUSE_DATABASE} +key='Sixteen byte key' + +${CLICKHOUSE_CLIENT} < diff --git a/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include_source.xml b/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include_source.xml index f5a02ed7b5c69..f8f975b6777ab 100644 --- a/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include_source.xml +++ b/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include_source.xml @@ -33,4 +33,14 @@ + + +
+ local + /etc/clickhouse-server/ + 1 + 1 +
+ + diff --git a/tests/integration/test_s3_credentials_hardening/test.py b/tests/integration/test_s3_credentials_hardening/test.py index eca2cb680ea5c..6e4095215412a 100644 --- a/tests/integration/test_s3_credentials_hardening/test.py +++ b/tests/integration/test_s3_credentials_hardening/test.py @@ -591,6 +591,32 @@ def test_dynamic_disk_include_locations_child_is_restricted(): assert "ACCESS_DENIED" in error or "must provide its credentials explicitly" in error, error +def test_dynamic_disk_include_locations_child_local_path_is_fenced(): + # The same shape, for the local filesystem rather than S3: an `include` can inject a + # `locations.` child whose backend is `local` and whose `path` is outside + # `custom_local_disks_base_directory`. A disk root that names no local path of its own must not let + # that child through -- `RegisterDiskObjectStorage` builds one object storage per child, each with + # its own `path`. + # A `type = object_storage` disk with an `include` and no literal `object_storage_type` may resolve to S3, + # so the credentials restriction rejects it before the disk is created; lift it to reach the path check. + node_with_includes.query("DROP TABLE IF EXISTS t_loc_local SYNC") + error = node_with_includes.query_and_get_error( + "CREATE TABLE t_loc_local (x UInt8) ENGINE = MergeTree ORDER BY tuple() " + "SETTINGS disk = disk(type = object_storage, include = 'evil_locations_local_path')", + settings={ + "dynamic_disk_allow_include": 1, + "s3_allow_server_credentials_in_user_queries": 1, + }, + ) + assert "must be inside" in error, error + assert ( + node_with_includes.query( + "SELECT count() FROM system.disks WHERE path = '/etc/clickhouse-server/'" + ).strip() + == "0" + ) + + def test_persistent_table_does_not_reuse_credentialed_client_across_sessions(): # A persistent S3 table whose credentials are server-managed (use_environment_credentials) is non-static, so # its client is rebuilt per query. An opt-in session reading it builds a credentialed client in the shared diff --git a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference new file mode 100644 index 0000000000000..005a61cd1bdaf --- /dev/null +++ b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference @@ -0,0 +1,23 @@ +local +1 +0 +no directory +local object storage +1 +0 +no directory +metadata path +1 +0 +no directory +no settings clause +1 +0 +no directory +named backup +1 +0 +no directory +constraints +1 +unrelated change allowed diff --git a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh new file mode 100755 index 0000000000000..372dc17955715 --- /dev/null +++ b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: custom disks are not configured in fasttest +# Tag no-replicated-database: creates a user and a settings profile + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every location on the local filesystem that a disk defined in SQL names has to be inside +# `custom_local_disks_base_directory`. `CREATE TABLE` checks that; `ALTER TABLE ... MODIFY SETTING disk` +# has to check it the same way. + +OUTSIDE="${USER_FILES_PATH}/05217_outside_${CLICKHOUSE_DATABASE}" +INSIDE="${CLICKHOUSE_DISKS_FILES}/05217_${CLICKHOUSE_DATABASE}" + +# Prints 1 if the statement was rejected by the fence, then whether the disk is registered and whether +# the rejected location was created. +function check_rejected() +{ + local disk_name=$1 location=$2 query=$3 + $CLICKHOUSE_CLIENT -q "$query" 2>&1 | grep -m1 -c -F "must be inside" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM system.disks WHERE name = '${disk_name}'" + test -e "$location" && echo "the rejected disk created ${location}" || echo "no directory" +} + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_05217" +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t_05217 (x UInt64) ENGINE = MergeTree ORDER BY x + SETTINGS disk = disk(name = '05217_ok_${CLICKHOUSE_DATABASE}', type = local, path = '${INSIDE}_ok/')" + +# `ALTER TABLE ... MODIFY SETTING disk = disk(...)` used to resolve the definition as if it came from +# stored metadata, which registered the disk unchecked, before the check of the statement ran. +echo "local" +check_rejected "05217_local_${CLICKHOUSE_DATABASE}" "${OUTSIDE}_local/" " + ALTER TABLE t_05217 MODIFY SETTING disk = disk(name = '05217_local_${CLICKHOUSE_DATABASE}', type = local, path = '${OUTSIDE}_local/')" + +echo "local object storage" +check_rejected "05217_object_storage_${CLICKHOUSE_DATABASE}" "${OUTSIDE}_object_storage/" " + ALTER TABLE t_05217 MODIFY SETTING disk = disk(name = '05217_object_storage_${CLICKHOUSE_DATABASE}', + type = object_storage, object_storage_type = local, path = '${OUTSIDE}_object_storage/')" + +echo "metadata path" +check_rejected "05217_metadata_${CLICKHOUSE_DATABASE}" "${OUTSIDE}_metadata/" " + ALTER TABLE t_05217 MODIFY SETTING disk = disk(name = '05217_metadata_${CLICKHOUSE_DATABASE}', + type = object_storage, object_storage_type = local, path = '${INSIDE}_metadata/', metadata_path = '${OUTSIDE}_metadata/')" + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_05217" + +# A table created without a SETTINGS clause goes through the same check. +echo "no settings clause" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_05217_plain (x UInt64) ENGINE = MergeTree ORDER BY x" +check_rejected "05217_plain_${CLICKHOUSE_DATABASE}" "${OUTSIDE}_plain/" " + ALTER TABLE t_05217_plain MODIFY SETTING disk = disk(name = '05217_plain_${CLICKHOUSE_DATABASE}', type = local, path = '${OUTSIDE}_plain/')" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_05217_plain" + +# The disk name is chosen by the query, so it cannot be what exempts a disk from the fence. +echo "named backup" +check_rejected "backup" "${OUTSIDE}_backup/" " + CREATE TABLE t_05217_backup (x UInt64) ENGINE = MergeTree ORDER BY x + SETTINGS disk = disk(name = 'backup', type = local, path = '${OUTSIDE}_backup/')" + +# The settings constraints compare the `disk` the table has after the `ALTER`, not one left unresolved: +# a constant `merge_tree_disk` rejects a change of the disk and nothing else. +echo "constraints" +user="u_05217_${CLICKHOUSE_DATABASE}" +profile="p_05217_${CLICKHOUSE_DATABASE}" +$CLICKHOUSE_CLIENT -q " + DROP USER IF EXISTS ${user}; + DROP SETTINGS PROFILE IF EXISTS ${profile}; + CREATE USER ${user} IDENTIFIED WITH no_password; + GRANT ALTER ON ${CLICKHOUSE_DATABASE}.* TO ${user}; + CREATE SETTINGS PROFILE ${profile} SETTINGS merge_tree_disk CONST TO ${user}; + CREATE TABLE t_05217_default (x UInt64) ENGINE = MergeTree ORDER BY x; + CREATE TABLE t_05217_disk (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS disk = 'local_disk';" + +$CLICKHOUSE_CLIENT --user "${user}" -q "ALTER TABLE t_05217_default MODIFY SETTING disk = 'local_disk'" 2>&1 \ + | grep -m1 -c -F "Setting disk should not be changed" +$CLICKHOUSE_CLIENT --user "${user}" -q "ALTER TABLE t_05217_disk MODIFY SETTING min_bytes_for_wide_part = 0" && echo "unrelated change allowed" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE t_05217_default; + DROP TABLE t_05217_disk; + DROP SETTINGS PROFILE ${profile}; + DROP USER ${user};" From a104ba11dce84e4aea823dbbbe00c43719c3efde Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 07:24:51 +0000 Subject: [PATCH 193/296] Backport #124113 to 26.8: Respect `iceberg_metadata_staleness_ms` in metadata parsing --- .../DataLakes/Iceberg/IcebergMetadata.cpp | 2 +- ...ction_metadata_staleness_no_list.reference | 5 +++ ...ble_function_metadata_staleness_no_list.sh | 33 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05306_iceberg_table_function_metadata_staleness_no_list.reference create mode 100755 tests/queries/0_stateless/05306_iceberg_table_function_metadata_staleness_no_list.sh diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 2af675f6cf10a..b47dec44d1a06 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -184,7 +184,7 @@ Iceberg::PersistentTableComponents IcebergMetadata::initializePersistentTableCom LoggerPtr log) { const auto [metadata_version, metadata_file_path, compression_method] - = getLatestOrExplicitMetadataFileAndVersion(object_storage, configuration->getPathForRead().path, configuration->getDataLakeSettings(), cache_ptr, context_, log.get(), std::nullopt, CompressionMethod::None, true); + = getLatestOrExplicitMetadataFileAndVersion(object_storage, configuration->getPathForRead().path, configuration->getDataLakeSettings(), cache_ptr, context_, log.get(), std::nullopt, CompressionMethod::None, /* force_fetch_latest_metadata */ false); LOG_DEBUG(log, "Latest metadata file path is {}, version {}", metadata_file_path, metadata_version); auto metadata_object = getMetadataJSONObject(metadata_file_path, object_storage, cache_ptr, context_, log, compression_method, std::nullopt); diff --git a/tests/queries/0_stateless/05306_iceberg_table_function_metadata_staleness_no_list.reference b/tests/queries/0_stateless/05306_iceberg_table_function_metadata_staleness_no_list.reference new file mode 100644 index 0000000000000..193c56cfa3217 --- /dev/null +++ b/tests/queries/0_stateless/05306_iceberg_table_function_metadata_staleness_no_list.reference @@ -0,0 +1,5 @@ +6 +6 +6 +stale 0 +fresh 1 diff --git a/tests/queries/0_stateless/05306_iceberg_table_function_metadata_staleness_no_list.sh b/tests/queries/0_stateless/05306_iceberg_table_function_metadata_staleness_no_list.sh new file mode 100755 index 0000000000000..2677b25520bfb --- /dev/null +++ b/tests/queries/0_stateless/05306_iceberg_table_function_metadata_staleness_no_list.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="05306_iceberg_staleness/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE} (c0 Int32) ENGINE = IcebergS3(s3_conn, filename = '${TABLE_PATH}')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE} VALUES (1), (2), (3)" + +run_select() +{ + ${CLICKHOUSE_CLIENT} --iceberg_metadata_staleness_ms="$1" --log_comment="${CLICKHOUSE_TEST_UNIQUE_NAME}_$2" \ + --query "SELECT sum(c0) FROM icebergS3(s3_conn, filename = '${TABLE_PATH}')" +} + +run_select 600000 warm +run_select 600000 stale +run_select 0 fresh + +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" +${CLICKHOUSE_CLIENT} --query " + SELECT replaceOne(log_comment, '${CLICKHOUSE_TEST_UNIQUE_NAME}_', ''), ProfileEvents['S3ListObjects'] > 0 + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' + AND log_comment IN ('${CLICKHOUSE_TEST_UNIQUE_NAME}_stale', '${CLICKHOUSE_TEST_UNIQUE_NAME}_fresh') + ORDER BY event_time_microseconds +" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${TABLE}" From 2171c47906368b40f4824120323c89479c348816 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 08:15:01 +0000 Subject: [PATCH 194/296] Backport #123255 to 26.8: File cache: make reserve granularity dynamic --- src/Common/ProfileEvents.cpp | 3 +- .../IO/CachedOnDiskReadBufferFromFile.cpp | 7 +- src/Disks/IO/CachedOnDiskReadBufferFromFile.h | 2 + .../IO/CachedOnDiskWriteBufferFromFile.cpp | 5 +- .../IO/CachedOnDiskWriteBufferFromFile.h | 2 + src/IO/DiskCacheProvider.cpp | 7 +- src/IO/DiskCacheProvider.h | 2 + src/Interpreters/FileCache/FileCache.cpp | 2 + src/Interpreters/FileCache/FileCache.h | 3 + .../FileCache/FileCacheSettings.cpp | 2 +- src/Interpreters/FileCache/FileCache_fwd.h | 2 +- src/Interpreters/FileCache/FileSegment.cpp | 44 +++--- src/Interpreters/FileCache/FileSegment.h | 31 +++- src/Interpreters/FileCache/Metadata.cpp | 7 +- .../FileCache/WriteBufferToFileSegment.cpp | 4 +- .../FileCache/WriteBufferToFileSegment.h | 1 + src/Interpreters/tests/gtest_filecache.cpp | 146 ++++++++++++++++++ ...StorageSystemDashboardsFilesystemCache.cpp | 5 +- 18 files changed, 237 insertions(+), 38 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index c100f5999c0e8..3a17b07f8f6a7 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -903,7 +903,8 @@ The server successfully detected this situation and will download merged part fr M(FilesystemCacheStateLockMicroseconds, "Lock filesystem cache time for state lock", ValueType::Microseconds) \ M(FilesystemCacheReserveMicroseconds, "Filesystem cache space reservation time", ValueType::Microseconds) \ M(FilesystemCacheReserveAttempts, "Filesystem cache space reservation attempt", ValueType::Number) \ - M(FilesystemCacheFailedReserveAttempts, "Number of failed file cache space reservation attempts.", ValueType::Number) \ + M(FilesystemCacheReserveAheadRetries, "Number of times a filesystem cache reservation with reserve-ahead did not fit and was retried with exactly the requested size", ValueType::Number) \ + M(FilesystemCacheFailedReserveAttempts, "Number of failed file cache space reservation attempts, including reserve-ahead attempts that were retried with the exact size (see FilesystemCacheReserveAheadRetries).", ValueType::Number) \ M(FilesystemCacheEvictMicroseconds, "Filesystem cache eviction time", ValueType::Microseconds) \ M(FilesystemCacheGetOrSetMicroseconds, "Filesystem cache getOrSet() time", ValueType::Microseconds) \ M(FilesystemCacheGetMicroseconds, "Filesystem cache get() time", ValueType::Microseconds) \ diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 884a1243aea27..5a4c13c7c676b 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -1131,7 +1131,8 @@ bool CachedOnDiskReadBufferFromFile::predownloadForFileSegment( info.cache_settings.reserve_space_wait_lock_timeout_milliseconds, failure_reason, /* reserve_stat */nullptr, - reserve_hint); + reserve_hint, + &info.reserve_ahead); if (continue_predownload) { @@ -1634,7 +1635,8 @@ size_t CachedOnDiskReadBufferFromFile::readFromFileSegment( info.cache_settings.reserve_space_wait_lock_timeout_milliseconds, failure_reason, /* reserve_stat */nullptr, - reserve_hint); + reserve_hint, + &info.reserve_ahead); if (success) { @@ -2064,6 +2066,7 @@ off_t CachedOnDiskReadBufferFromFile::seek(off_t offset, int whence) first_offset = file_offset_of_buffer_end = new_pos; info.reset(); + info.reserve_ahead.reset(); state.reset(); initialized = false; diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index ee80a207acea1..7aff6baf52288 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -129,6 +129,8 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase, public IRe /// List of file segments which we need to read /// given initial [start_offset, read_until_position). FileSegmentsHolderPtr file_segments; + /// Kept across file segments and setReadUntilPosition(), reset on seek. + FileCacheReserveAhead reserve_ahead; void reset(); }; diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 1c036c9ef357e..538ebe5025b0c 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -197,7 +197,9 @@ bool FileSegmentRangeWriter::write(char * data, size_t size, size_t offset, File } size_t size_to_write = std::min(available_size, size); - bool reserved = file_segment->reserve(size_to_write, reserve_space_lock_wait_timeout_milliseconds, failure_reason); + bool reserved = file_segment->reserve( + size_to_write, reserve_space_lock_wait_timeout_milliseconds, failure_reason, + /* reserve_stat */nullptr, /* reserve_hint */std::nullopt, &reserve_ahead); if (!reserved) { appendFilesystemCacheLog(*file_segment); @@ -423,6 +425,7 @@ void FileSegmentRangeWriter::jumpToPosition(size_t position) expected_write_offset = position; ignore_bytes = 0; + reserve_ahead.reset(); } CachedOnDiskWriteBufferFromFile::CachedOnDiskWriteBufferFromFile( diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h index e07609a9df390..b9c7f342eae7e 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h @@ -82,6 +82,8 @@ class FileSegmentRangeWriter size_t expected_write_offset = 0; + FileCacheReserveAhead reserve_ahead; + bool finalized = false; }; diff --git a/src/IO/DiskCacheProvider.cpp b/src/IO/DiskCacheProvider.cpp index ebb06055ee9b4..1e8a5633f8ce6 100644 --- a/src/IO/DiskCacheProvider.cpp +++ b/src/IO/DiskCacheProvider.cpp @@ -280,10 +280,15 @@ size_t DiskCacheWriter::write(ChainedBuffers data, const Claim & claim) data.copyTo(flat_buf.data(), write_range); std::string failure_reason; + /// Don't reserve ahead past the end of the miss range. + const size_t reserve_hint = write_end_max - write_offset; const bool reserved = seg.reserve( contiguous, cache_settings.reserve_space_wait_lock_timeout_milliseconds, - failure_reason); + failure_reason, + /* reserve_stat */nullptr, + reserve_hint, + &reserve_ahead); if (!reserved) { LOG_TRACE(log, "DiskCacheWriter::write: reserve failed for [{}, {}]: {}", diff --git a/src/IO/DiskCacheProvider.h b/src/IO/DiskCacheProvider.h index 92eb337b68e27..f24ac534b3e74 100644 --- a/src/IO/DiskCacheProvider.h +++ b/src/IO/DiskCacheProvider.h @@ -92,6 +92,8 @@ class DiskCacheWriter : public CacheWriter /// same time, so both update this set. mutable std::mutex committed_mutex; ByteRange aligned_range; + /// One state per writer: it owns a single append-only segment. + FileCacheReserveAhead reserve_ahead; LoggerPtr log = getLogger("DiskCacheWriter"); }; diff --git a/src/Interpreters/FileCache/FileCache.cpp b/src/Interpreters/FileCache/FileCache.cpp index 74f109772c5ee..1e4d9d0140386 100644 --- a/src/Interpreters/FileCache/FileCache.cpp +++ b/src/Interpreters/FileCache/FileCache.cpp @@ -1387,6 +1387,7 @@ bool FileCache::doTryReserve( file_segment.key(), file_segment.offset(), size, query_priority->getStateInfoForLog(lock)); failure_reason = "query limit exceeded"; + reserve_stat.not_enough_space = true; return false; } query_eviction_info = query_priority->collectEvictionInfo( @@ -1434,6 +1435,7 @@ bool FileCache::doTryReserve( invalidated_entries, query_priority, failure_reason)) { chassert(!failure_reason.empty()); + reserve_stat.not_enough_space = true; return false; } diff --git a/src/Interpreters/FileCache/FileCache.h b/src/Interpreters/FileCache/FileCache.h index 02848269dcfb4..1b6915101d677 100644 --- a/src/Interpreters/FileCache/FileCache.h +++ b/src/Interpreters/FileCache/FileCache.h @@ -70,6 +70,8 @@ struct FileCacheReserveStat Stat total_stat; std::array()> stat_by_kind{}; + /// Set if the reservation failed because the cache or the query limit had no room for it. + bool not_enough_space = false; Stat & getStatByKind(FileSegmentKind kind) { return stat_by_kind[static_cast(kind)]; } const Stat & getStatByKind(FileSegmentKind kind) const { return stat_by_kind[static_cast(kind)]; } @@ -89,6 +91,7 @@ struct FileCacheReserveStat total_stat += other.total_stat; for (size_t i = 0; i < stat_by_kind.size(); ++i) stat_by_kind[i] += other.stat_by_kind[i]; + not_enough_space |= other.not_enough_space; return *this; } }; diff --git a/src/Interpreters/FileCache/FileCacheSettings.cpp b/src/Interpreters/FileCache/FileCacheSettings.cpp index 307b407178510..cbabbf15affd5 100644 --- a/src/Interpreters/FileCache/FileCacheSettings.cpp +++ b/src/Interpreters/FileCache/FileCacheSettings.cpp @@ -38,7 +38,7 @@ namespace ErrorCodes DECLARE(UInt64, max_elements, FILECACHE_DEFAULT_MAX_ELEMENTS, "Maximum number of cache elements, e.g. file segments (limits number of files on filesystem)", 0) \ DECLARE(UInt64, max_file_segment_size, FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE, "Maximum size of a single file segment", 0) \ DECLARE(UInt64, boundary_alignment, FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT, "File segment alignment", 0) \ - DECLARE(UInt64, reserve_granularity, FILECACHE_DEFAULT_RESERVE_GRANULARITY, "When reserving space for a file segment download, reserve at least this many bytes ahead of the downloaded size (capped at the file segment size). Coarser granularity reduces the rate of cache state lock acquisitions on the reservation hot path, at the cost of holding some reserved-but-not-yet-downloaded space (reclaimed on file segment completion). Value 0 disables reserve-ahead: space is reserved exactly as requested.", 0) \ + DECLARE(UInt64, reserve_granularity, FILECACHE_DEFAULT_MAX_RESERVE_GRANULARITY, "Maximum size to reserve ahead of the downloaded size of a file segment, to take the cache lock less often. Starts from 0 for each reader/writer and grows with its repeated reservations up to this value. 0 disables reserve-ahead.", 0) \ DECLARE(Bool, cache_on_write_operations, false, "Enables write-through cache (cache on INSERT and MERGE)", 0) \ DECLARE(FileCachePolicy, cache_policy, FILECACHE_DEFAULT_CACHE_POLICY, "Cache eviction policy", 0) \ DECLARE(Double, slru_size_ratio, FILECACHE_DEFAULT_SLRU_RATIO, "SLRU cache policy size ratio of protected to probationary elements", 0) \ diff --git a/src/Interpreters/FileCache/FileCache_fwd.h b/src/Interpreters/FileCache/FileCache_fwd.h index 7c8df3d4a586a..76a864a3f98b4 100644 --- a/src/Interpreters/FileCache/FileCache_fwd.h +++ b/src/Interpreters/FileCache/FileCache_fwd.h @@ -7,7 +7,7 @@ namespace DB static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 32 * 1024 * 1024; /// 32Mi static constexpr int FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT = 4 * 1024 * 1024; /// 4Mi -static constexpr int FILECACHE_DEFAULT_RESERVE_GRANULARITY = 4 * 1024 * 1024; /// 4Mi. 0 disables reserve-ahead (reserve exactly the requested size). +static constexpr int FILECACHE_DEFAULT_MAX_RESERVE_GRANULARITY = 4 * 1024 * 1024; /// 4Mi. 0 disables reserve-ahead. static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE_WITH_BACKGROUND_DOWLOAD = 4 * 1024 * 1024; /// 4Mi static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_THREADS = 5; static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_QUEUE_SIZE_LIMIT = 5000; diff --git a/src/Interpreters/FileCache/FileSegment.cpp b/src/Interpreters/FileCache/FileSegment.cpp index fa0c71d3311ba..d820d7ea15d7c 100644 --- a/src/Interpreters/FileCache/FileSegment.cpp +++ b/src/Interpreters/FileCache/FileSegment.cpp @@ -34,6 +34,7 @@ namespace ProfileEvents extern const Event FilesystemCacheHoldFileSegments; extern const Event FilesystemCacheUnusedHoldFileSegments; extern const Event FilesystemCacheBackgroundDownloadQueuePush; + extern const Event FilesystemCacheReserveAheadRetries; } namespace CurrentMetrics @@ -668,7 +669,8 @@ bool FileSegment::reserve( size_t lock_wait_timeout_milliseconds, std::string & failure_reason, FileCacheReserveStat * reserve_stat, - size_t reserve_hint) + std::optional reserve_hint, + FileCacheReserveAhead * reserve_ahead) { if (!size_to_reserve) throw Exception(ErrorCodes::LOGICAL_ERROR, "Zero space reservation is not allowed"); @@ -708,29 +710,18 @@ bool FileSegment::reserve( const size_t minimum_reserve_size = size_to_reserve; - if (!is_unbound) + if (!is_unbound && reserve_ahead) { - const auto reserve_granularity = cache->getReserveGranularity(); - if (reserve_granularity && reserve_granularity > size_to_reserve) + /// Don't reserve ahead past the segment end or the end of the read. + size_t segment_reserve_limit = range().size() - reserved_size; + if (reserve_hint) { - size_to_reserve = reserved_size + reserve_granularity > range().size() - ? range().size() - reserved_size - : reserve_granularity; - - /// `reserve_hint` is measured from the current download offset, so the read ends at - /// `read_horizon` in segment-relative terms. Don't reserve ahead past it. - const size_t read_horizon = current_downloaded_size + reserve_hint; - if (reserve_hint - && read_horizon > reserved_size - && read_horizon < reserved_size + size_to_reserve) - size_to_reserve = read_horizon - reserved_size; + const size_t read_horizon = current_downloaded_size + *reserve_hint; + segment_reserve_limit = std::min(segment_reserve_limit, read_horizon > reserved_size ? read_horizon - reserved_size : 0); } - } - /// The reserve-ahead caps above (segment range, read horizon) are only an upper bound; they - /// must never reserve less than the current write needs, otherwise the write would exceed the - /// reservation. A bare assert would not protect release builds, so clamp explicitly. - size_to_reserve = std::max(size_to_reserve, minimum_reserve_size); + size_to_reserve = reserve_ahead->getReserveSize(size_to_reserve, segment_reserve_limit, cache->getReserveGranularity()); + } /// This (resizable file segments) is allowed only for single threaded use of file segment. /// Currently it is used only for temporary files through cache. @@ -742,10 +733,23 @@ bool FileSegment::reserve( FileCacheReserveStat dummy_stat; if (!reserve_stat) reserve_stat = &dummy_stat; + reserve_stat->not_enough_space = false; bool reserved = cache->tryReserve( *this, size_to_reserve, *reserve_stat, *getKeyMetadata()->origin, lock_wait_timeout_milliseconds, failure_reason); + if (!reserved && reserve_ahead) + reserve_ahead->reset(); + + /// Reserve-ahead is best-effort: retry with the exact size if it did not fit. + if (!reserved && size_to_reserve > minimum_reserve_size && reserve_stat->not_enough_space) + { + ProfileEvents::increment(ProfileEvents::FilesystemCacheReserveAheadRetries); + *reserve_stat = FileCacheReserveStat{}; + reserved = cache->tryReserve( + *this, minimum_reserve_size, *reserve_stat, *getKeyMetadata()->origin, lock_wait_timeout_milliseconds, failure_reason); + } + if (!reserved) setDownloadFailedUnlocked(lock()); diff --git a/src/Interpreters/FileCache/FileSegment.h b/src/Interpreters/FileCache/FileSegment.h index 6a53fde3184c7..c4d3fe5b17cb0 100644 --- a/src/Interpreters/FileCache/FileSegment.h +++ b/src/Interpreters/FileCache/FileSegment.h @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #include #include #include @@ -24,6 +26,24 @@ namespace DB class ReadBufferFromFileBase; struct FileCacheReserveStat; +/// Reserve-ahead state of one reader/writer: the first reservation is exact, then the reserve-ahead +/// doubles up to `max_reserve_granularity` (the cache's `reserve_granularity`). Reset on a failed reservation. +struct FileCacheReserveAhead +{ + /// Returns the size to reserve, at least `size_to_reserve`, and grows the reserve-ahead. + /// `segment_reserve_limit` is the room left in the segment, capped by the end of the read if known. + size_t getReserveSize(size_t size_to_reserve, size_t segment_reserve_limit, size_t max_reserve_granularity) + { + const size_t result = std::max(size_to_reserve, std::min({granularity, max_reserve_granularity, segment_reserve_limit})); + granularity = std::min(max_reserve_granularity, std::max(granularity, size_to_reserve) * 2); + return result; + } + + void reset() { granularity = 0; } + + size_t granularity = 0; +}; + struct CreateFileSegmentSettings { @@ -211,18 +231,15 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). * ========== Methods for _only_ file segment's `downloader` ================== */ - /// Try to reserve exactly `size` bytes (in addition to the getDownloadedSize() bytes already downloaded). - /// Returns true if reservation was successful, false otherwise. - /// - /// `reserve_hint`, if non-zero, bounds the reserve-ahead to the bytes left to read from the - /// current download offset (e.g. up to read_until_position), so the segment is never reserved - /// ahead past what the read will consume. + /// Try to reserve `size` bytes on top of getDownloadedSize(). Reserves ahead if `reserve_ahead` is + /// set, but not past `reserve_hint` bytes from the current download offset. bool reserve( size_t size_to_reserve, size_t lock_wait_timeout_milliseconds, std::string & failure_reason, FileCacheReserveStat * reserve_stat = nullptr, - size_t reserve_hint = 0); + std::optional reserve_hint = std::nullopt, + FileCacheReserveAhead * reserve_ahead = nullptr); /// Write data into reserved space. void write(char * from, size_t size, size_t offset_in_file); diff --git a/src/Interpreters/FileCache/Metadata.cpp b/src/Interpreters/FileCache/Metadata.cpp index 7e94b38517342..8cb4d3c3b6c7c 100644 --- a/src/Interpreters/FileCache/Metadata.cpp +++ b/src/Interpreters/FileCache/Metadata.cpp @@ -995,6 +995,7 @@ void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optional(buf->getPosition())) buf->seek(offset, SEEK_SET); + FileCacheReserveAhead reserve_ahead; while (size_to_download && !buf->eof()) { const auto available = buf->available(); @@ -1004,7 +1005,11 @@ void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optionalreserve(bytes_to_write, reserve_space_lock_wait_timeout_milliseconds, failure_reason, &reserve_stat); + bool ok = file_segment->reserve( + bytes_to_write, reserve_space_lock_wait_timeout_milliseconds, failure_reason, + &reserve_stat, /* reserve_hint */std::nullopt, &reserve_ahead); if (!ok) { diff --git a/src/Interpreters/FileCache/WriteBufferToFileSegment.h b/src/Interpreters/FileCache/WriteBufferToFileSegment.h index b4d2e944f924e..d6d576044d107 100644 --- a/src/Interpreters/FileCache/WriteBufferToFileSegment.h +++ b/src/Interpreters/FileCache/WriteBufferToFileSegment.h @@ -51,6 +51,7 @@ class WriteBufferToFileSegment : public WriteBufferFromFileBase, public IReadabl FileSegmentsHolderPtr segment_holder; const size_t reserve_space_lock_wait_timeout_milliseconds; + FileCacheReserveAhead reserve_ahead; size_t written_bytes = 0; }; diff --git a/src/Interpreters/tests/gtest_filecache.cpp b/src/Interpreters/tests/gtest_filecache.cpp index fb24b54a6a9ce..9ddd0e382259e 100644 --- a/src/Interpreters/tests/gtest_filecache.cpp +++ b/src/Interpreters/tests/gtest_filecache.cpp @@ -75,6 +75,7 @@ namespace ProfileEvents { extern const Event FilesystemCacheDowngradedFileSegments; extern const Event FilesystemCacheEvictedFileSegments; + extern const Event FilesystemCacheReserveAheadRetries; } using namespace std::chrono_literals; @@ -108,6 +109,7 @@ namespace DB::FileCacheSetting extern const FileCacheSettingsUInt64 max_elements; extern const FileCacheSettingsUInt64 max_file_segment_size; extern const FileCacheSettingsUInt64 boundary_alignment; + extern const FileCacheSettingsUInt64 reserve_granularity; extern const FileCacheSettingsFileCachePolicy cache_policy; extern const FileCacheSettingsDouble slru_size_ratio; extern const FileCacheSettingsDouble keep_free_space_elements_ratio; @@ -3815,6 +3817,150 @@ TEST_F(FileCacheTest, ReserveUndoneWhenKeyDirectoryCannotBeCreated) ASSERT_EQ(cache->getUsedCacheSize(), 8u); } +TEST(FileCacheReserveAhead, GrowsUpToLimitAndResets) +{ + DB::FileCacheReserveAhead reserve_ahead; + + /// Exact first, then doubling up to the limit. + ASSERT_EQ(reserve_ahead.getReserveSize(/* size_to_reserve */ 2, /* max_reserve_size */ 100, /* limit */ 16), 2u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 4u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 8u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 16u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 16u); + + /// Capped by `max_reserve_size`, but never less than the request. + ASSERT_EQ(reserve_ahead.getReserveSize(2, 10, 16), 10u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 1, 16), 2u); + ASSERT_EQ(reserve_ahead.getReserveSize(32, 100, 16), 32u); + + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 4), 4u); + + /// Limit 0 disables reserve-ahead. + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 0), 2u); + + reserve_ahead.reset(); + ASSERT_EQ(reserve_ahead.getReserveSize(6, 100, 16), 6u); + ASSERT_EQ(reserve_ahead.getReserveSize(6, 100, 16), 12u); + + reserve_ahead.reset(); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 2u); + + /// A short first request does not keep the reserve-ahead small for later, larger requests. + reserve_ahead.reset(); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 1000, 1024), 2u); + ASSERT_EQ(reserve_ahead.getReserveSize(100, 1000, 1024), 100u); + ASSERT_EQ(reserve_ahead.getReserveSize(100, 1000, 1024), 200u); +} + +TEST_F(FileCacheTest, DynamicReserveGranularity) +{ + ServerUUID::setRandomForUnitTests(); + DB::ThreadStatus thread_status; + + DB::FileCacheSettings settings; + settings[FileCacheSetting::path] = cache_base_path; + settings[FileCacheSetting::max_size] = 1000; + settings[FileCacheSetting::max_elements] = 10; + settings[FileCacheSetting::max_file_segment_size] = 100; + settings[FileCacheSetting::boundary_alignment] = 100; + settings[FileCacheSetting::reserve_granularity] = 16; + settings[FileCacheSetting::load_metadata_asynchronously] = false; + settings[FileCacheSetting::cache_policy] = FileCachePolicy::LRU; + + auto cache = std::make_shared("dynamic_reserve_granularity", settings); + cache->initialize(); + + const auto & user = FileCache::getCommonOrigin(); + std::string failure_reason; + std::string data(100, '0'); + + auto reserve_and_write = [&](DB::FileSegment & segment, size_t size, DB::FileCacheReserveAhead * reserve_ahead) + { + EXPECT_TRUE(segment.reserve(size, 1000, failure_reason, nullptr, std::nullopt, reserve_ahead)) << failure_reason; + segment.write(data.data(), size, segment.getCurrentWriteOffset()); + return segment.getReservedSize(); + }; + + { + auto holder = cache->getOrSet(DB::FileCacheKey::fromPath("no_reserve_ahead"), 0, 100, /*file_size=*/100, {}, 0, user); + auto segment = *holder->begin(); + ASSERT_EQ(segment->getOrSetDownloader(), FileSegment::getCallerId()); + ASSERT_EQ(reserve_and_write(*segment, 2, nullptr), 2u); + ASSERT_EQ(reserve_and_write(*segment, 2, nullptr), 4u); + ASSERT_EQ(reserve_and_write(*segment, 2, nullptr), 6u); + } + + { + auto holder = cache->getOrSet(DB::FileCacheKey::fromPath("reserve_ahead"), 0, 100, /*file_size=*/100, {}, 0, user); + auto segment = *holder->begin(); + ASSERT_EQ(segment->getOrSetDownloader(), FileSegment::getCallerId()); + + DB::FileCacheReserveAhead reserve_ahead; + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 2u); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 2u + 4); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 6u); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 6u + 8); + for (size_t downloaded = 10; downloaded <= 14; downloaded += 2) + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 14u); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 14u + 16); + for (size_t downloaded = 18; downloaded <= 30; downloaded += 2) + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 30u); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 30u + 16); + } +} + +TEST_F(FileCacheTest, ReserveAheadFallsBackToExactSize) +{ + ServerUUID::setRandomForUnitTests(); + DB::ThreadStatus thread_status; + + /// The cache is smaller than the reserve-ahead limit. + DB::FileCacheSettings settings; + settings[FileCacheSetting::path] = cache_base_path; + settings[FileCacheSetting::max_size] = 12; + settings[FileCacheSetting::max_elements] = 10; + settings[FileCacheSetting::max_file_segment_size] = 20; + settings[FileCacheSetting::boundary_alignment] = 20; + settings[FileCacheSetting::reserve_granularity] = 16; + settings[FileCacheSetting::load_metadata_asynchronously] = false; + settings[FileCacheSetting::cache_policy] = FileCachePolicy::LRU; + + auto cache = std::make_shared("reserve_ahead_fallback", settings); + cache->initialize(); + + const auto & user = FileCache::getCommonOrigin(); + auto holder = cache->getOrSet(DB::FileCacheKey::fromPath("reserve_ahead_fallback"), 0, 20, /*file_size=*/20, {}, 0, user); + auto segment = *holder->begin(); + ASSERT_EQ(segment->getOrSetDownloader(), FileSegment::getCallerId()); + + std::string failure_reason; + std::string data(20, '0'); + DB::FileCacheReserveAhead reserve_ahead; + auto & events = CurrentThread::getProfileEvents(); + const auto retries_before = events[ProfileEvents::FilesystemCacheReserveAheadRetries]; + auto reserve_and_write = [&]() + { + EXPECT_TRUE(segment->reserve(2, 1000, failure_reason, nullptr, std::nullopt, &reserve_ahead)) << failure_reason; + segment->write(data.data(), 2, segment->getCurrentWriteOffset()); + return segment->getReservedSize(); + }; + + ASSERT_EQ(reserve_and_write(), 2u); + ASSERT_EQ(reserve_and_write(), 6u); + ASSERT_EQ(reserve_and_write(), 6u); + ASSERT_EQ(events[ProfileEvents::FilesystemCacheReserveAheadRetries], retries_before); + ASSERT_EQ(reserve_and_write(), 8u); /// +8 does not fit, falls back to exact +2 and resets + ASSERT_EQ(events[ProfileEvents::FilesystemCacheReserveAheadRetries], retries_before + 1); + ASSERT_EQ(reserve_and_write(), 10u); + ASSERT_EQ(reserve_and_write(), 12u); /// +4 does not fit, falls back to exact +2, the cache is full + ASSERT_EQ(events[ProfileEvents::FilesystemCacheReserveAheadRetries], retries_before + 2); + + ASSERT_FALSE(segment->reserve(2, 1000, failure_reason, nullptr, std::nullopt, &reserve_ahead)); + ASSERT_EQ(events[ProfileEvents::FilesystemCacheReserveAheadRetries], retries_before + 2); + ASSERT_EQ(segment->getReservedSize(), 12u); + ASSERT_EQ(cache->getUsedCacheSize(), 12u); +} + TEST_F(FileCacheTest, QueryLimitContextRevivedDuringRelease) { /// Regression for STID 4192-71db: a holder for some query_id decides it is the last one and diff --git a/src/Storages/System/StorageSystemDashboardsFilesystemCache.cpp b/src/Storages/System/StorageSystemDashboardsFilesystemCache.cpp index 40f8da6ac1e99..3d9578979cf1a 100644 --- a/src/Storages/System/StorageSystemDashboardsFilesystemCache.cpp +++ b/src/Storages/System/StorageSystemDashboardsFilesystemCache.cpp @@ -382,9 +382,10 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_FilesystemCacheReserveAttempts) AS Attempts, - avg(ProfileEvent_FilesystemCacheFailedReserveAttempts) AS Failed, + avg(ProfileEvent_FilesystemCacheFailedReserveAttempts - ProfileEvent_FilesystemCacheReserveAheadRetries) AS Failed, avg(ProfileEvent_FilesystemCacheFailToReserveSpaceBecauseOfLockContention) AS SkippedOnLockContention, - avg(ProfileEvent_FilesystemCacheFailToReserveSpaceBecauseOfCacheResize) AS SkippedOnCacheResize + avg(ProfileEvent_FilesystemCacheFailToReserveSpaceBecauseOfCacheResize) AS SkippedOnCacheResize, + avg(ProfileEvent_FilesystemCacheReserveAheadRetries) AS ReserveAheadRetries FROM merge('system', '^metric_log') WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to GROUP BY t From beb1e890ca4bcfca42a7641b83eee4a910805286 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 09:04:48 +0000 Subject: [PATCH 195/296] Backport #121629 to 26.8: Require SET DEFINER to rewrite the body of a view that runs as another user --- src/Interpreters/InterpreterAlterQuery.cpp | 69 +++++++++- src/Interpreters/InterpreterAlterQuery.h | 4 + ...7_alter_view_modify_query_access.reference | 19 +++ .../05237_alter_view_modify_query_access.sh | 122 ++++++++++++++++++ ...w_modify_query_ephemeral_definer.reference | 2 + ...ter_view_modify_query_ephemeral_definer.sh | 42 ++++++ 6 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05237_alter_view_modify_query_access.reference create mode 100755 tests/queries/0_stateless/05237_alter_view_modify_query_access.sh create mode 100644 tests/queries/0_stateless/05296_alter_view_modify_query_ephemeral_definer.reference create mode 100755 tests/queries/0_stateless/05296_alter_view_modify_query_ephemeral_definer.sh diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index ed545cf51acaa..d6fd232999dc0 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -2,7 +2,10 @@ #include #include +#include #include +#include +#include #include #include #include @@ -445,6 +448,14 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) visitor.substituteDatabaseInTableFunctions(*alter.command_list); } + /// The hosts run the entry without the user, unless `distributed_ddl_use_initial_user_and_roles` is on, + /// so the view's SQL security is authorized here, which needs the view on this host. + if (modify_query && !table) + throw Exception(ErrorCodes::UNKNOWN_TABLE, + "Table {}.{} does not exist on this host. `ALTER TABLE ... ON CLUSTER ... MODIFY QUERY` is authorized " + "on the initiator, so it must be run from a host that has the view", + backQuoteIfNeed(alter.getDatabase()), backQuoteIfNeed(alter.getTable())); + DDLQueryOnClusterParams params; params.access_to_check = getRequiredAccess(table); return executeDDLQueryOnCluster(query_ptr, getContext(), params); @@ -599,14 +610,70 @@ bool InterpreterAlterQuery::isRowExistsLightweightDeleteMarker(const StoragePtr return metadata_snapshot->isVirtualColumn(RowExistsColumn::name); } +/** `MODIFY QUERY` replaces the body a view executes, and for `SQL SECURITY DEFINER` or `NONE` that body does + * not run with the caller's privileges. Writing it is the same act of impersonation that `CREATE` and + * `MODIFY SQL SECURITY` gate behind `SET DEFINER` and `ALLOW SQL SECURITY NONE` in + * `processSQLSecurityOption`, so it takes the same grants. + */ +void InterpreterAlterQuery::addRequiredAccessForModifyQuerySQLSecurity( + AccessRightsElements & required_access, const StoragePtr & storage) const +{ + /// Without the view there is no body to replace: the local path fails on the missing table, and an + /// `ON CLUSTER` statement refuses to dispatch a `MODIFY QUERY` for a view this host does not have. + if (!storage) + return; + + const auto metadata_snapshot = storage->getInMemoryMetadataPtr(getContext(), /*bypass_metadata_cache=*/ false); + if (!metadata_snapshot->sql_security_type) + return; + + if (*metadata_snapshot->sql_security_type == SQLSecurityType::NONE) + { + required_access.emplace_back(AccessType::ALLOW_SQL_SECURITY_NONE); + return; + } + + if (*metadata_snapshot->sql_security_type != SQLSecurityType::DEFINER || !metadata_snapshot->definer) + return; + + /// `processSQLSecurityOption` stores an ephemeral definer as a clone named `:definer`, and the grant is + /// held on the user it was cloned from. A real user can have the same suffix, so strip it only when the base + /// user is ephemeral, which is the condition under which the clone is made. + String definer_name = *metadata_snapshot->definer; + static constexpr std::string_view ephemeral_suffix = ":definer"; + if (definer_name.ends_with(ephemeral_suffix)) + { + String base_name = definer_name.substr(0, definer_name.size() - ephemeral_suffix.size()); + const auto & access_control = getContext()->getAccessControl(); + if (auto base_id = access_control.find(base_name); base_id && access_control.isEphemeral(*base_id)) + definer_name = std::move(base_name); + } + + if (definer_name != getContext()->getUserName()) + required_access.emplace_back(AccessType::SET_DEFINER, definer_name); +} + AccessRightsElements InterpreterAlterQuery::getRequiredAccess(const StoragePtr & storage) const { AccessRightsElements required_access; const auto & alter = query_ptr->as(); const bool row_exists_is_marker = isRowExistsLightweightDeleteMarker(storage, getContext()); + /// A `MODIFY SQL SECURITY` in the same statement decides what the new body will execute as, and + /// `processSQLSecurityOption` has already authorized exactly that. The stored security then says + /// nothing about the body being written, so it must not add a requirement of its own. + const bool sql_security_is_being_replaced = std::ranges::any_of( + alter.command_list->children, + [](const auto & child) { return child->template as().sql_security != nullptr; }); + for (const auto & child : alter.command_list->children) + { + const auto & command = child->as(); required_access.append_range( - getRequiredAccessForCommand(child->as(), alter.getDatabase(), alter.getTable(), row_exists_is_marker)); + getRequiredAccessForCommand(command, alter.getDatabase(), alter.getTable(), row_exists_is_marker)); + + if (command.type == ASTAlterCommand::MODIFY_QUERY && !sql_security_is_being_replaced) + addRequiredAccessForModifyQuerySQLSecurity(required_access, storage); + } return required_access; } diff --git a/src/Interpreters/InterpreterAlterQuery.h b/src/Interpreters/InterpreterAlterQuery.h index 541c2793ab4fe..ebb9deddba453 100644 --- a/src/Interpreters/InterpreterAlterQuery.h +++ b/src/Interpreters/InterpreterAlterQuery.h @@ -42,6 +42,10 @@ class InterpreterAlterQuery : public IInterpreter, WithMutableContext private: AccessRightsElements getRequiredAccess(const StoragePtr & storage) const; + /// Adds the grants `processSQLSecurityOption` demands for the view's stored SQL security, so replacing the + /// body of a `DEFINER` or `NONE` view takes the same authority as declaring that security in the first place. + void addRequiredAccessForModifyQuerySQLSecurity(AccessRightsElements & required_access, const StoragePtr & storage) const; + BlockIO executeToTable(const ASTAlterQuery & alter); BlockIO executeToDatabase(const ASTAlterQuery & alter); diff --git a/tests/queries/0_stateless/05237_alter_view_modify_query_access.reference b/tests/queries/0_stateless/05237_alter_view_modify_query_access.reference new file mode 100644 index 0000000000000..64c23bcc0f816 --- /dev/null +++ b/tests/queries/0_stateless/05237_alter_view_modify_query_access.reference @@ -0,0 +1,19 @@ +-- the body of a view that runs as another user can be rewritten only with SET DEFINER on that user +reads secret ACCESS_DENIED +on cluster ACCESS_DENIED +reads only src ACCESS_DENIED +row policy ACCESS_DENIED +after the grant accepted +-- a SQL SECURITY NONE body runs unchecked, so rewriting it needs ALLOW SQL SECURITY NONE +security none ACCESS_DENIED +after the grant accepted +-- a MODIFY SQL SECURITY in the same statement decides what the body runs as, so the old one +-- must not be what gets authorized +take over accepted +hand to other ACCESS_DENIED +-- a definer named :definer is a real user unless is ephemeral +real, base grant ACCESS_DENIED +real, own view accepted +cluster no view UNKNOWN_TABLE on the initiator +-- no data leaked: none of the denied bodies was stored +rows, leaked 1 0 diff --git a/tests/queries/0_stateless/05237_alter_view_modify_query_access.sh b/tests/queries/0_stateless/05237_alter_view_modify_query_access.sh new file mode 100755 index 0000000000000..11f6052fd6f72 --- /dev/null +++ b/tests/queries/0_stateless/05237_alter_view_modify_query_access.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +db=${CLICKHOUSE_DATABASE} +author="author_${CLICKHOUSE_DATABASE}" +definer="definer_${CLICKHOUSE_DATABASE}" + +# The definer may read the protected table; the author may not. +${CLICKHOUSE_CLIENT} --query " +DROP USER IF EXISTS $author, $definer; +CREATE USER $author, $definer IDENTIFIED WITH no_password; + +CREATE TABLE $db.secret (id UInt64, secret String) ENGINE = MergeTree ORDER BY id; +INSERT INTO $db.secret VALUES (1, 'SECRET'); +CREATE TABLE $db.src (id UInt64) ENGINE = MergeTree ORDER BY id; + +GRANT SELECT ON $db.src TO $definer; +GRANT SELECT ON $db.secret TO $definer; +GRANT SELECT, INSERT ON $db.src TO $author; +GRANT CLUSTER ON *.* TO $author; +" + +# Prints the error code the statement failed with, or `accepted` when it succeeded. Reporting the code +# rather than matching one expected code keeps a broken setup from reading as a successful statement. +run() { + local err + err=$(${CLICKHOUSE_CLIENT} --user "${3:-$author}" --query "$2" 2>&1 | grep -o -m1 -E '\([A-Z_]+\)' | tr -d '()') + echo "$1 ${err:-accepted}" +} + +echo '-- the body of a view that runs as another user can be rewritten only with SET DEFINER on that user' +${CLICKHOUSE_CLIENT} --query " +CREATE MATERIALIZED VIEW $db.mv_foreign ENGINE = MergeTree ORDER BY id +DEFINER = $definer SQL SECURITY DEFINER +AS SELECT id, ''::String AS secret FROM $db.src; +GRANT ALTER VIEW MODIFY QUERY, SELECT ON $db.mv_foreign TO $author; +" +reads_secret="SELECT e.id AS id, j.secret AS secret FROM $db.src AS e CROSS JOIN (SELECT * FROM $db.secret) AS j" +run 'reads secret ' "ALTER TABLE $db.mv_foreign MODIFY QUERY $reads_secret" +run 'on cluster ' "ALTER TABLE $db.mv_foreign ON CLUSTER test_shard_localhost MODIFY QUERY $reads_secret" +run 'reads only src ' "ALTER TABLE $db.mv_foreign MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" + +# The author may read the table, but a row policy hides every row from it. The body would run as the +# definer, which has no such policy, so only the SQL security gate can stop it. +${CLICKHOUSE_CLIENT} --query " +GRANT SELECT ON $db.secret TO $author; +CREATE ROW POLICY policy_$db ON $db.secret USING 0 TO $author; +" +run 'row policy ' "ALTER TABLE $db.mv_foreign MODIFY QUERY $reads_secret" +${CLICKHOUSE_CLIENT} --query " +DROP ROW POLICY policy_$db ON $db.secret; +REVOKE SELECT ON $db.secret FROM $author; +" + +${CLICKHOUSE_CLIENT} --query "GRANT SET DEFINER ON $definer TO $author" +run 'after the grant ' "ALTER TABLE $db.mv_foreign MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" +${CLICKHOUSE_CLIENT} --query "REVOKE SET DEFINER ON $definer FROM $author" + +echo '-- a SQL SECURITY NONE body runs unchecked, so rewriting it needs ALLOW SQL SECURITY NONE' +${CLICKHOUSE_CLIENT} --query " +CREATE MATERIALIZED VIEW $db.mv_none ENGINE = MergeTree ORDER BY id +SQL SECURITY NONE +AS SELECT id, ''::String AS secret FROM $db.src; +GRANT ALTER VIEW MODIFY QUERY, SELECT ON $db.mv_none TO $author; +" +run 'security none ' "ALTER TABLE $db.mv_none MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" +${CLICKHOUSE_CLIENT} --query "GRANT ALLOW SQL SECURITY NONE ON *.* TO $author" +run 'after the grant ' "ALTER TABLE $db.mv_none MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" +${CLICKHOUSE_CLIENT} --query "REVOKE ALLOW SQL SECURITY NONE ON *.* FROM $author" + +echo '-- a MODIFY SQL SECURITY in the same statement decides what the body runs as, so the old one' +echo '-- must not be what gets authorized' +${CLICKHOUSE_CLIENT} --query " +CREATE MATERIALIZED VIEW $db.mv_migrate ENGINE = MergeTree ORDER BY id +DEFINER = $definer SQL SECURITY DEFINER +AS SELECT id, ''::String AS secret FROM $db.src; +GRANT ALTER VIEW MODIFY QUERY, ALTER VIEW MODIFY SQL SECURITY, SELECT ON $db.mv_migrate TO $author; +" +run 'take over ' "ALTER TABLE $db.mv_migrate MODIFY SQL SECURITY DEFINER DEFINER = CURRENT_USER, MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" +run 'hand to other ' "ALTER TABLE $db.mv_migrate MODIFY SQL SECURITY DEFINER DEFINER = $definer, MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" + +echo '-- a definer named :definer is a real user unless is ephemeral' +real="${definer}:definer" +${CLICKHOUSE_CLIENT} --query " +CREATE USER \`$real\` IDENTIFIED WITH no_password; +GRANT SELECT ON $db.src TO \`$real\`; +CREATE MATERIALIZED VIEW $db.mv_real ENGINE = MergeTree ORDER BY id +DEFINER = \`$real\` SQL SECURITY DEFINER +AS SELECT id, ''::String AS secret FROM $db.src; +GRANT ALTER VIEW MODIFY QUERY ON $db.mv_real TO $author, \`$real\`; +GRANT SET DEFINER ON $definer TO $author; +" +run 'real, base grant' "ALTER TABLE $db.mv_real MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" +run 'real, own view ' "ALTER TABLE $db.mv_real MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" "$real" +${CLICKHOUSE_CLIENT} --query "REVOKE SET DEFINER ON $definer FROM $author" + +# A view this host does not have must be refused here, before dispatch: the hosts that have it run the +# entry without the user. The host rejects it too, but its error names the host. +${CLICKHOUSE_CLIENT} --query "GRANT ALTER VIEW MODIFY QUERY ON $db.mv_nowhere TO $author" +out=$(${CLICKHOUSE_CLIENT} --user "$author" --query " +ALTER TABLE $db.mv_nowhere ON CLUSTER test_shard_localhost MODIFY QUERY SELECT id, ''::String AS secret FROM $db.src" 2>&1) +code=$(echo "$out" | grep -o -m1 -E '\([A-Z_]+\)' | tr -d '()') +if echo "$out" | grep -q 'There was an error on'; then where='elsewhere'; else where='on the initiator'; fi +echo "cluster no view ${code:-accepted} $where" + +echo '-- no data leaked: none of the denied bodies was stored' +${CLICKHOUSE_CLIENT} --user "$author" --query "INSERT INTO $db.src VALUES (1)" +echo -n 'rows, leaked ' +${CLICKHOUSE_CLIENT} --user "$author" --query "SELECT count(), countIf(secret != '') FROM $db.mv_foreign" + +${CLICKHOUSE_CLIENT} --query " +DROP TABLE $db.mv_foreign SYNC; +DROP TABLE $db.mv_none SYNC; +DROP TABLE $db.mv_migrate SYNC; +DROP TABLE $db.mv_real SYNC; +DROP USER $author, $definer; +" +# Dropping its last view also removes a definer named :definer. +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS \`$real\`" diff --git a/tests/queries/0_stateless/05296_alter_view_modify_query_ephemeral_definer.reference b/tests/queries/0_stateless/05296_alter_view_modify_query_ephemeral_definer.reference new file mode 100644 index 0000000000000..414f416cadc2e --- /dev/null +++ b/tests/queries/0_stateless/05296_alter_view_modify_query_ephemeral_definer.reference @@ -0,0 +1,2 @@ +without grant ACCESS_DENIED +with grant accepted diff --git a/tests/queries/0_stateless/05296_alter_view_modify_query_ephemeral_definer.sh b/tests/queries/0_stateless/05296_alter_view_modify_query_ephemeral_definer.sh new file mode 100755 index 0000000000000..15893cd08970f --- /dev/null +++ b/tests/queries/0_stateless/05296_alter_view_modify_query_ephemeral_definer.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Tags: no-replicated-database +# no-replicated-database: that configuration replaces the user directories, so there is no `memory` storage. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +db=${CLICKHOUSE_DATABASE} +author="author_${CLICKHOUSE_DATABASE}" +ephemeral="ephemeral_${CLICKHOUSE_DATABASE}" + +# A view whose definer is ephemeral stores a clone of it named `:definer`, and the grant that lets +# the author act as that definer is held on the ephemeral user itself. +${CLICKHOUSE_CLIENT} --query " +DROP USER IF EXISTS $author, $ephemeral, \`$ephemeral:definer\`; +CREATE USER $author IDENTIFIED WITH no_password; +CREATE USER $ephemeral IN memory; + +CREATE TABLE $db.src (id UInt64) ENGINE = MergeTree ORDER BY id; +GRANT SELECT ON $db.src TO $author, $ephemeral; + +CREATE MATERIALIZED VIEW $db.mv ENGINE = MergeTree ORDER BY id +DEFINER = $ephemeral SQL SECURITY DEFINER +AS SELECT id FROM $db.src; +GRANT ALTER VIEW MODIFY QUERY ON $db.mv TO $author; +" + +run() { + local err + err=$(${CLICKHOUSE_CLIENT} --user "$author" --query "$2" 2>&1 | grep -o -m1 -E '\([A-Z_]+\)' | tr -d '()') + echo "$1 ${err:-accepted}" +} + +run 'without grant' "ALTER TABLE $db.mv MODIFY QUERY SELECT id FROM $db.src" +${CLICKHOUSE_CLIENT} --query "GRANT SET DEFINER ON $ephemeral TO $author" +run 'with grant ' "ALTER TABLE $db.mv MODIFY QUERY SELECT id FROM $db.src" + +${CLICKHOUSE_CLIENT} --query " +DROP TABLE $db.mv SYNC; +DROP USER IF EXISTS $author, $ephemeral, \`$ephemeral:definer\`; +" From 3bd5adc5b4563a62e53ace4d4bca98243f5ee7ff Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 10:16:43 +0000 Subject: [PATCH 196/296] Backport #123543 to 26.8: Delta lake write creds for catalogs --- src/Databases/DataLake/DatabaseDataLake.cpp | 2 +- src/Databases/DataLake/ICatalog.h | 5 +++ src/Databases/DataLake/UnityCatalog.cpp | 31 ++++++++++++++----- src/Databases/DataLake/UnityCatalog.h | 6 +++- .../ObjectStorages/IObjectStorage.h | 2 ++ .../ObjectStorages/S3/S3ObjectStorage.h | 3 +- .../DeltaLakeMetadataDeltaKernel.cpp | 27 +++++++++++++--- .../ObjectStorage/S3/Configuration.cpp | 17 +++++----- 8 files changed, 70 insertions(+), 23 deletions(-) diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 38c41f1e182e9..6a276373d7ef8 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -920,7 +920,7 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con const auto is_secondary_query = context_->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; /// When we applied static credentials from database settings, they are authoritative: - /// do not let a catalog-vended refresh callback (e.g. Unity/REST `requestReadCredentials`) + /// do not let a catalog-vended refresh callback (e.g. Unity/REST `requestCredentials`) /// silently re-fetch credentials and override them. The same holds when the user disabled /// `vended_credentials` and no static credentials were applied (e.g. relying on default or /// environment S3 auth): the object storage layer invokes the refresh callback after an diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index e79ebf686f685..6077a30586ccb 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -286,6 +286,11 @@ class ICatalog return std::nullopt; } + virtual CredentialsRefreshCallback getWriteCredentialsConfigurationCallback(const DB::StorageID & /*storage_id*/) + { + return std::nullopt; + } + /// Result of `prepareSettingsChanges`: the new catalog state built off to the side, /// ready to be published by `commitSettingsChanges`. struct PreparedSettingsChanges diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index 2d78df3ec7d8f..5f8ff8578902b 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -1,5 +1,6 @@ #include #include +#include #if USE_PARQUET @@ -132,11 +133,11 @@ void UnityCatalog::getTableMetadata( throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from unity catalog"); } -Poco::JSON::Object::Ptr UnityCatalog::requestReadCredentials(const String & table_id) const +Poco::JSON::Object::Ptr UnityCatalog::requestCredentials(const String & table_id, const String & operation) const { Poco::JSON::Object request_body; request_body.set("table_id", table_id); - request_body.set("operation", "READ"); + request_body.set("operation", operation); auto callback = [&request_body] (std::ostream & os) { request_body.stringify(os); }; auto [json, _] = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, callback); @@ -172,7 +173,7 @@ void UnityCatalog::getCredentials(const String & table_id, TableMetadata & metad if (storage_type != StorageType::S3 && storage_type != StorageType::Azure) return; - auto response = requestReadCredentials(table_id); + auto response = requestCredentials(table_id, "READ"); std::shared_ptr creds; switch (storage_type) @@ -489,12 +490,28 @@ ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCa "Cannot build a Unity credentials refresh callback for `{}`: StorageID has no UUID", table_id.getNameForLogs()); - const String unity_table_id = toString(table_id.uuid); + return getCredentialsCallbackForOperation(DB::toString(table_id.uuid), "READ"); +} + +/// `StorageID::uuid` of a `DataLakeCatalog` table is the `table_id` returned by Unity. +ICatalog::CredentialsRefreshCallback UnityCatalog::getWriteCredentialsConfigurationCallback(const DB::StorageID & table_id) +{ + if (table_id.uuid == DB::UUIDHelpers::Nil) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "Cannot build a Unity credentials refresh callback for `{}`: the table has no UUID", + table_id.getNameForLogs()); + + return getCredentialsCallbackForOperation(DB::toString(table_id.uuid), "READ_WRITE"); +} - return [this, unity_table_id] () -> std::shared_ptr { - LOG_DEBUG(log, "Update credentials in the catalog"); +ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsCallbackForOperation(const String & unity_table_id, const String & operation) +{ + return [this, unity_table_id, operation] () -> std::shared_ptr + { + LOG_DEBUG(log, "Update {} credentials in the catalog", operation); - return parseS3Credentials(requestReadCredentials(unity_table_id)); + return parseS3Credentials(requestCredentials(unity_table_id, operation)); }; } diff --git a/src/Databases/DataLake/UnityCatalog.h b/src/Databases/DataLake/UnityCatalog.h index 840822b0e0da7..783d0fb2b0181 100644 --- a/src/Databases/DataLake/UnityCatalog.h +++ b/src/Databases/DataLake/UnityCatalog.h @@ -66,7 +66,7 @@ class UnityCatalog final : public ICatalog, private DB::WithContext CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; void getCredentials(const String & table_id, TableMetadata & metadata) const; - Poco::JSON::Object::Ptr requestReadCredentials(const String & table_id) const; + Poco::JSON::Object::Ptr requestCredentials(const String & table_id, const String & operation) const; std::shared_ptr parseS3Credentials(const Poco::JSON::Object::Ptr & response) const; std::shared_ptr parseAzureCredentials(const Poco::JSON::Object::Ptr & response) const; @@ -77,6 +77,10 @@ class UnityCatalog final : public ICatalog, private DB::WithContext TableMetadata & result) const; ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & table_id) override; + + ICatalog::CredentialsRefreshCallback getWriteCredentialsConfigurationCallback(const DB::StorageID & table_id) override; + + ICatalog::CredentialsRefreshCallback getCredentialsCallbackForOperation(const String & unity_table_id, const String & operation); }; } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 8c0a5c2858fb7..d176c3b94f0f2 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -451,6 +451,8 @@ class IObjectStorage /// `ReadBufferFromS3` / `getObjectMetadata` error handlers that normally invoke this. virtual bool tryRefreshCredentialsViaCallback() { return false; } + virtual bool hasCredentialsRefreshCallback() const { return false; } + #if USE_AZURE_BLOB_STORAGE || USE_AWS_S3 /// Assign tag on objects virtual void tagObjects(const StoredObjects &, const std::string &, const std::string &) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index ab26f5e998d01..5339d5b3908ed 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -40,7 +40,7 @@ class S3ObjectStorage : public IObjectStorage ObjectStorageKeyGeneratorPtr key_generator_, const String & disk_name_, bool for_disk_s3_ = true, - const S3CredentialsRefreshCallback & credentials_refresh_callback_ = [] -> std::unique_ptr{ return nullptr; }, + const S3CredentialsRefreshCallback & credentials_refresh_callback_ = {}, bool client_restricts_server_credentials_ = true) : uri(uri_) , disk_name(disk_name_) @@ -155,6 +155,7 @@ class S3ObjectStorage : public IObjectStorage std::shared_ptr tryGetS3StorageClient() override; bool tryRefreshCredentialsViaCallback() override; + bool hasCredentialsRefreshCallback() const override { return static_cast(credentials_refresh_callback); } S3::URI getURI() const { return uri; } S3Settings getS3Settings() const { return *s3_settings.get(); } diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp index 47bbc604ad4b5..1bc42de2adecd 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp @@ -38,6 +38,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; + extern const int DATALAKE_DATABASE_ERROR; } namespace FailPoints @@ -615,12 +616,12 @@ ReadFromFormatInfo DeltaLakeMetadataDeltaKernel::prepareReadingFromFormat( SinkToStoragePtr DeltaLakeMetadataDeltaKernel::write( SharedHeader sample_block, - const StorageID & /* table_id */, + const StorageID & table_id, ObjectStoragePtr object_storage_, StorageObjectStorageConfigurationPtr configuration, const std::optional & format_settings, ContextPtr context, - std::shared_ptr /* catalog */) + std::shared_ptr catalog) { if (!context->getSettingsRef()[Setting::allow_experimental_delta_lake_writes]) { @@ -643,14 +644,30 @@ SinkToStoragePtr DeltaLakeMetadataDeltaKernel::write( "Writing to DeltaLake tables with column mapping enabled is not supported"); } - auto delta_transaction = std::make_shared(kernel_helper); + auto write_object_storage = object_storage_; + auto write_kernel_helper = kernel_helper; + if (catalog && object_storage_->hasCredentialsRefreshCallback()) + { + if (auto write_credentials_callback = catalog->getWriteCredentialsConfigurationCallback(table_id)) + { + write_object_storage = configuration->createObjectStorage(context, /* is_readonly */ false, write_credentials_callback); + if (!write_object_storage->tryRefreshCredentialsViaCallback()) + throw Exception( + ErrorCodes::DATALAKE_DATABASE_ERROR, + "The catalog did not vend storage credentials for writing into table {}", + table_id.getNameForLogs()); + write_kernel_helper = DB::getKernelHelper(configuration, write_object_storage); + } + } + + auto delta_transaction = std::make_shared(write_kernel_helper); delta_transaction->create(partition_columns, snapshot->getTableSchema()); if (partition_columns.empty()) { return std::make_shared( delta_transaction, - object_storage_, + write_object_storage, context, sample_block, format_settings, @@ -661,7 +678,7 @@ SinkToStoragePtr DeltaLakeMetadataDeltaKernel::write( return std::make_shared( delta_transaction, partition_columns, - object_storage_, + write_object_storage, context, sample_block, format_settings, diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 735827eb82222..806825224e3cb 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -179,15 +179,16 @@ ObjectStoragePtr StorageS3Configuration::createObjectStorage(ContextPtr context, url, *s3_settings, context, /* for_disk_s3 */ false, /*opt_disk_name*/ {}, /*refresh_credentials_callback*/ std::nullopt, is_loading_from_existing_metadata, force_anonymous_load_fallback); - auto client_refresher = [refresh_credentials_callback, this, context_ = Context::createCopy(context)] () -> std::unique_ptr + S3ObjectStorage::S3CredentialsRefreshCallback client_refresher; + if (refresh_credentials_callback) { - if (!refresh_credentials_callback) - return nullptr; - auto new_client = getClient( - url, *s3_settings, context_, /* for_disk_s3 */ false, /*opt_disk_name*/ {}, refresh_credentials_callback, - is_loading_from_existing_metadata, force_anonymous_load_fallback); - return new_client; - }; + client_refresher = [refresh_credentials_callback, this, context_ = Context::createCopy(context)] () -> std::unique_ptr + { + return getClient( + url, *s3_settings, context_, /* for_disk_s3 */ false, /*opt_disk_name*/ {}, refresh_credentials_callback, + is_loading_from_existing_metadata, force_anonymous_load_fallback); + }; + } return std::make_shared( std::move(client), std::make_unique(*s3_settings), From 419d61702a54154ee9fd33cccb09d8c63911f74f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 11:26:40 +0000 Subject: [PATCH 197/296] Backport #124088 to 26.8: Speed up 04510_distributed_final_many_partitions: 65 partitions instead of 150 --- ...0_distributed_final_many_partitions.reference | 4 ++-- .../04510_distributed_final_many_partitions.sh | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference b/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference index 1545bdb2fa433..7caab58d91b66 100644 --- a/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference +++ b/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference @@ -1,3 +1,3 @@ split groups all partitions into the target tasks 1 -distributed plan 15000 112567500 -plain plan 15000 112567500 +distributed plan 6500 21154250 +plain plan 6500 21154250 diff --git a/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh b/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh index 5c4cfb379b5e4..1e4009602a451 100755 --- a/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh +++ b/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh @@ -18,28 +18,28 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # not expose the split, so the check reads the planner's trace message. $CLICKHOUSE_CLIENT --multiquery -q " -SET max_partitions_per_insert_block = 150; --- Statistics play no role here, and writing a statistics blob for each of the 300 tiny parts --- makes the setup exceed the test time limit on sanitizer builds with object storage. +-- Statistics play no role here, and writing a statistics blob for each tiny part slows the setup +-- down on sanitizer builds with object storage. SET materialize_statistics_on_insert = 0; DROP TABLE IF EXISTS t_final_many_partitions; CREATE TABLE t_final_many_partitions (k UInt64, p UInt16, v UInt64, ver UInt64) ENGINE = ReplacingMergeTree(ver) PARTITION BY p ORDER BY k SETTINGS index_granularity = 64, auto_statistics_types = ''; SYSTEM STOP MERGES t_final_many_partitions; --- 150 partitions, far more than the 4 tasks x 16 layers per task the planner aims for. Every key +-- 65 partitions, one more than the 4 tasks x 16 layers per task the planner aims for. Every key -- appears in every partition and must survive per-partition dedup, and its newer version must win. -INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number, 1 FROM numbers(15000); -INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number + 5, 2 FROM numbers(15000); +INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number, 1 FROM numbers(6500); +INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number + 5, 2 FROM numbers(6500); " -SETTINGS="enable_parallel_replicas = 0, automatic_parallel_replicas_mode = 0, max_rows_to_group_by = 0, +# The table is small enough to be broadcast; force the bucketed read regardless of size. +SETTINGS="enable_parallel_replicas = 0, automatic_parallel_replicas_mode = 0, max_rows_to_group_by = 0, distributed_plan_max_rows_to_broadcast = 0, distributed_plan_default_reader_bucket_count = 4, do_not_merge_across_partitions_select_final = 1" echo -n "split groups all partitions into the target tasks " if $CLICKHOUSE_CLIENT --send_logs_level=trace -q " SELECT count(), sum(v) FROM t_final_many_partitions FINAL FORMAT Null SETTINGS make_distributed_plan = 1, $SETTINGS" 2>&1 \ - | grep -q "Distributed FINAL read bucketed: 150 layers in 38 lanes per task make 4 tasks" + | grep -q "Distributed FINAL read bucketed: 65 layers in 17 lanes per task make 4 tasks" then echo 1; else echo 0; fi # Both plans must return identical results. From ef0bd94e4c276d50aec5c613c04f3b17e9373bd9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 14:56:06 +0000 Subject: [PATCH 198/296] Backport #123829 to 26.8: Do not read a JOIN input that cannot contribute under a WHERE no row passes --- src/Interpreters/ActionsDAG.cpp | 24 +++++--- src/Interpreters/ActionsDAG.h | 4 +- .../Optimizations/filterPushDown.cpp | 10 +++- ...always_false_where_reads_nothing.reference | 7 +++ ...under_always_false_where_reads_nothing.sql | 58 +++++++++++++++++++ 5 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/05320_join_under_always_false_where_reads_nothing.reference create mode 100644 tests/queries/0_stateless/05320_join_under_always_false_where_reads_nothing.sql diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index ea637f7688a36..48484dd0f4ea7 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -3430,7 +3430,8 @@ ActionsDAG::ActionsForJOINFilterPushDown ActionsDAG::splitActionsForJOINFilterPu const Block & right_stream_header, const Names & equivalent_columns_to_push_down, const std::unordered_map & equivalent_left_stream_column_to_right_stream_column, - const std::unordered_map & equivalent_right_stream_column_to_left_stream_column) + const std::unordered_map & equivalent_right_stream_column_to_left_stream_column, + bool filter_is_always_false) { Node * predicate = const_cast(tryFindInOutputs(filter_name)); if (!predicate) @@ -3501,14 +3502,19 @@ ActionsDAG::ActionsForJOINFilterPushDown ActionsDAG::splitActionsForJOINFilterPu const bool left_stream_push_down_enabled = !left_stream_allowed_nodes.empty(); const bool right_stream_push_down_enabled = !right_stream_allowed_nodes.empty(); - if (!left_stream_push_down_enabled) - keep_conjuncts_depending_on_allowed_input(left_stream_push_down_conjunctions, left_stream_allowed_nodes); - if (!right_stream_push_down_enabled) - keep_conjuncts_depending_on_allowed_input(right_stream_push_down_conjunctions, right_stream_allowed_nodes); - /// A both-streams conjunct is pushed to BOTH sides, so a no-input conjunct here is unsafe if - /// EITHER side is disabled. - if (!left_stream_push_down_enabled || !right_stream_push_down_enabled) - keep_conjuncts_depending_on_allowed_input(both_streams_push_down_conjunctions, both_streams_allowed_nodes); + /// If no row passes the filter, the join output is already empty once a side that may be filtered + /// receives it, so a disabled side can receive the no-input conjuncts too and is not read in vain. + if (!filter_is_always_false) + { + if (!left_stream_push_down_enabled) + keep_conjuncts_depending_on_allowed_input(left_stream_push_down_conjunctions, left_stream_allowed_nodes); + if (!right_stream_push_down_enabled) + keep_conjuncts_depending_on_allowed_input(right_stream_push_down_conjunctions, right_stream_allowed_nodes); + /// A both-streams conjunct is pushed to BOTH sides, so a no-input conjunct here is unsafe if + /// EITHER side is disabled. + if (!left_stream_push_down_enabled || !right_stream_push_down_enabled) + keep_conjuncts_depending_on_allowed_input(both_streams_push_down_conjunctions, both_streams_allowed_nodes); + } NodeRawConstPtrs left_stream_allowed_conjunctions = std::move(left_stream_push_down_conjunctions.allowed); NodeRawConstPtrs right_stream_allowed_conjunctions = std::move(right_stream_push_down_conjunctions.allowed); diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 4182c9f950eb5..f9f60ca8aa4d7 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -507,6 +507,7 @@ class ActionsDAG * to left and right streams. * @param equivalent_left_stream_column_to_right_stream_column - equivalent left stream column name to right stream column map. * @param equivalent_right_stream_column_to_left_stream_column - equivalent right stream column name to left stream column map. + * @param filter_is_always_false - no row passes the filter, and a side whose emptiness empties the join output receives it. */ ActionsForJOINFilterPushDown splitActionsForJOINFilterPushDown( const std::string & filter_name, @@ -517,7 +518,8 @@ class ActionsDAG const Block & right_stream_header, const Names & equivalent_columns_to_push_down, const std::unordered_map & equivalent_left_stream_column_to_right_stream_column, - const std::unordered_map & equivalent_right_stream_column_to_left_stream_column); + const std::unordered_map & equivalent_right_stream_column_to_left_stream_column, + bool filter_is_always_false); /** Build filter dag from multiple filter dags. * diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 79c3786a7b0e1..0f2bdd9bf73d2 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -819,6 +820,12 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: if (is_filter_column_const_before) original_filter_const_column = filter->getOutputHeader()->getByName(filter->getFilterColumnName()).column; + /// With no input given only the conjuncts that read no column are evaluated, so FALSE means no row passes. + const bool filter_is_always_false + = (left_stream_filter_push_down_input_columns_available || right_stream_filter_push_down_input_columns_available) + && !isSensitiveToEvaluationCount(filter->getExpression()) + && filterResultForNotMatchedRows(filter->getExpression(), filter->getFilterColumnName(), Block{}) == FilterResult::FALSE; + auto join_filter_push_down_actions = filter->getExpression().splitActionsForJOINFilterPushDown( filter->getFilterColumnName(), filter->removesFilterColumn(), @@ -828,7 +835,8 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: *right_stream_input_header, equivalent_columns_to_push_down, equivalent_left_stream_column_to_right_stream_column, - equivalent_right_stream_column_to_left_stream_column); + equivalent_right_stream_column_to_left_stream_column, + filter_is_always_false); if (is_filter_column_const_before && !join_filter_push_down_actions.is_filter_const_after_all_push_downs) { diff --git a/tests/queries/0_stateless/05320_join_under_always_false_where_reads_nothing.reference b/tests/queries/0_stateless/05320_join_under_always_false_where_reads_nothing.reference new file mode 100644 index 0000000000000..31b7bd3346427 --- /dev/null +++ b/tests/queries/0_stateless/05320_join_under_always_false_where_reads_nothing.reference @@ -0,0 +1,7 @@ +LEFT JOIN 69 +INNER JOIN 69 +LEFT JOIN, aggregated right side 69 +05320_inner 1 +05320_left 1 +05320_left_aggregated 1 +indexHint on the non-preserved side 150 diff --git a/tests/queries/0_stateless/05320_join_under_always_false_where_reads_nothing.sql b/tests/queries/0_stateless/05320_join_under_always_false_where_reads_nothing.sql new file mode 100644 index 0000000000000..29f66682cd1db --- /dev/null +++ b/tests/queries/0_stateless/05320_join_under_always_false_where_reads_nothing.sql @@ -0,0 +1,58 @@ +-- A WHERE that no row passes is pushed to every JOIN input, including one that push-down is +-- disabled for, so the second UNION ALL branch does not read `t_big` (issue #123702). + +SET enable_parallel_replicas = 0; +SET query_plan_optimize_join_order_randomize = 0; + +DROP TABLE IF EXISTS t_live; +DROP TABLE IF EXISTS t_small; +DROP TABLE IF EXISTS t_big; +DROP TABLE IF EXISTS t_hint_left; +DROP TABLE IF EXISTS t_hint_right; + +CREATE TABLE t_live (d Date, c String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t_small (d Date, c String) ENGINE = MergeTree ORDER BY d; +CREATE TABLE t_big (c String, v UInt64) ENGINE = MergeTree ORDER BY c; + +INSERT INTO t_live SELECT toDate('2026-01-01') + number, toString(number % 10) FROM numbers(100); +INSERT INTO t_small SELECT toDate('2026-01-01') + number, toString(number % 10) FROM numbers(100); +INSERT INTO t_big SELECT toString(number % 10), number FROM numbers(100000); + +-- The outer WHERE is false for the second UNION ALL branch, whose constant `k` it reads. +SELECT 'LEFT JOIN', count() +FROM (SELECT d, 'b' AS k FROM t_live UNION ALL SELECT s.d AS d, 'x' AS k FROM t_small AS s LEFT JOIN t_big AS b ON s.c = b.c) +WHERE k = 'b' AND d >= '2026-02-01' +SETTINGS query_plan_join_swap_table = 'false', log_comment = '05320_left'; + +SELECT 'INNER JOIN', count() +FROM (SELECT d, 'b' AS k FROM t_live UNION ALL SELECT s.d AS d, 'x' AS k FROM t_small AS s INNER JOIN t_big AS b ON s.c = b.c) +WHERE k = 'b' AND d >= '2026-02-01' +SETTINGS query_plan_join_swap_table = 'false', log_comment = '05320_inner'; + +SELECT 'LEFT JOIN, aggregated right side', count() +FROM (SELECT d, 'b' AS k FROM t_live UNION ALL SELECT s.d AS d, 'x' AS k FROM t_small AS s LEFT JOIN (SELECT c, max(v) AS v FROM t_big GROUP BY c) AS b ON s.c = b.c) +WHERE k = 'b' AND d >= '2026-02-01' +SETTINGS query_plan_join_swap_table = 'false', log_comment = '05320_left_aggregated'; + +SYSTEM FLUSH LOGS query_log; +SELECT log_comment, read_rows < 10000 +FROM system.query_log +WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment IN ('05320_left', '05320_inner', '05320_left_aggregated') +ORDER BY log_comment; + +-- `indexHint` reads no column and is true: it must stay off the non-preserved side, where it would prune rows. +CREATE TABLE t_hint_left (id UInt32) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 64; +CREATE TABLE t_hint_right (id UInt32, v Int64) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 64; +INSERT INTO t_hint_left SELECT number FROM numbers(1000); +INSERT INTO t_hint_right SELECT number, number + 1000 FROM numbers(1000); + +SELECT 'indexHint on the non-preserved side', count() +FROM t_hint_left AS l LEFT JOIN t_hint_right AS r ON l.id = r.id +WHERE indexHint(r.id < 150) AND (r.id < 150) +SETTINGS use_join_disjunctions_push_down = 0, query_plan_join_swap_table = 'false'; + +DROP TABLE t_live; +DROP TABLE t_small; +DROP TABLE t_big; +DROP TABLE t_hint_left; +DROP TABLE t_hint_right; From ae0e3b7772248569f619c69fb84cd64ae91f776e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 15:29:21 +0000 Subject: [PATCH 199/296] Backport #120371 to 26.8: Hide secret function arguments in EXPLAIN when the value comes from another plan step --- .../FunctionSecretArgumentsFinderTreeNode.cpp | 3 +- src/Functions/HMAC.cpp | 13 ++- src/Interpreters/InterpreterExplainQuery.cpp | 4 +- src/Parsers/FunctionSecretArgumentsFinder.h | 7 ++ src/Planner/PlannerActionsVisitor.cpp | 17 +-- src/Processors/QueryPlan/QueryPlan.cpp | 3 +- src/Processors/QueryPlan/QueryPlan.h | 3 + src/Processors/QueryPlan/QueryPlanFormat.cpp | 107 ++++++++++++------ src/Processors/QueryPlan/QueryPlanFormat.h | 13 +-- ...xplain_hide_secrets_across_steps.reference | 9 ++ ...5213_explain_hide_secrets_across_steps.sql | 72 ++++++++++++ ...n_hide_secrets_definer_view_join.reference | 76 +++++++++++++ ..._explain_hide_secrets_definer_view_join.sh | 97 ++++++++++++++++ ...lain_show_secrets_with_privilege.reference | 18 +++ ...219_explain_show_secrets_with_privilege.sh | 34 ++++++ 15 files changed, 415 insertions(+), 61 deletions(-) create mode 100644 tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.reference create mode 100644 tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.sql create mode 100644 tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.reference create mode 100755 tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.sh create mode 100644 tests/queries/0_stateless/05219_explain_show_secrets_with_privilege.reference create mode 100755 tests/queries/0_stateless/05219_explain_show_secrets_with_privilege.sh diff --git a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp index 8c61d13923ca5..345fbe80a8659 100644 --- a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp +++ b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp @@ -61,8 +61,7 @@ void forEachSecretArgumentNode( continue; } - const bool in_span = secret_arguments.start <= n && n < secret_arguments.start + secret_arguments.count; - if (in_span || secret_arguments.masked_arguments.contains(n) || secret_arguments.replaced_arguments.contains(n)) + if (secret_arguments.isSecretArgument(n)) on_secret(n, secretValueSlot(arguments[n])); } } diff --git a/src/Functions/HMAC.cpp b/src/Functions/HMAC.cpp index e520289573f12..ee170892f8bbf 100644 --- a/src/Functions/HMAC.cpp +++ b/src/Functions/HMAC.cpp @@ -30,9 +30,12 @@ namespace ErrorCodes namespace { -const EVP_MD * getHashAlgorithm(const std::string_view & mode) +using EVP_MD_ptr = std::unique_ptr; + +/// EVP_MD_fetch returns a reference-counted handle that must be released; the digest itself stays cached by OpenSSL. +EVP_MD_ptr getHashAlgorithm(const std::string_view & mode) { - return EVP_MD_fetch(nullptr, std::string{mode}.c_str(), nullptr); + return EVP_MD_ptr(EVP_MD_fetch(nullptr, std::string{mode}.c_str(), nullptr), EVP_MD_free); } class FunctionHMAC final : public IFunction @@ -137,7 +140,7 @@ class FunctionHMAC final : public IFunction ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const std::string_view mode = arguments[0].column->getDataAt(0); - const EVP_MD * evp_md = getHashAlgorithm(mode); + const EVP_MD_ptr evp_md = getHashAlgorithm(mode); if (evp_md == nullptr) throw Exception( @@ -153,7 +156,7 @@ class FunctionHMAC final : public IFunction ColumnString::Chars & result_data = result_column->getChars(); ColumnString::Offsets & result_offsets = result_column->getOffsets(); - const size_t digest_length = EVP_MD_size(evp_md); + const size_t digest_length = EVP_MD_size(evp_md.get()); // Pre-allocate result data const size_t total_size = input_rows_count * digest_length; @@ -168,7 +171,7 @@ class FunctionHMAC final : public IFunction unsigned int actual_digest_length = 0; const unsigned char * hmac_result = HMAC( - evp_md, + evp_md.get(), key_value.data(), static_cast(key_value.size()), reinterpret_cast(message_value.data()), diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index e2459d3f53149..8c50a6235b6bc 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -1067,6 +1067,7 @@ InterpreterExplainQuery::AnalyzedInnerQuery & InterpreterExplainQuery::getAnalyz auto result = std::make_unique(); result->query_plan_options = checkAndGetSettings(ast.getSettings()).query_plan_options; + result->query_plan_options.show_secrets = canDisplaySecrets(getContext()); /// This is the only place that turns join statistics on, and it must happen before any interpreter /// is built. Every join of the query reads the mode from the context, so joins in nested plans get it as well. @@ -1245,6 +1246,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() } auto settings = checkAndGetSettings(ast_settings, pretty_version); + settings.query_plan_options.show_secrets = canDisplaySecrets(query_context); QueryPlan plan; @@ -1486,7 +1488,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() /// Build the per-plan pretty-names registry now: buildQueryPipeline below moves the ActionsDAGs /// out of the plan steps, so the names must be snapshotted before the pipeline consumes the plan. /// EXPLAIN ANALYZE rejects distributed plans above, so this covers the whole plan tree. - PrettyNamesPerPlan precomputed_pretty_names = QueryPlanFormat::buildPrettyNamesPerPlan(plan); + PrettyNamesPerPlan precomputed_pretty_names = QueryPlanFormat::buildPrettyNamesPerPlan(plan, analyzed.query_plan_options.show_secrets); plan.setConcurrencyControl(context->getSettingsRef()[Setting::use_concurrency_control]); diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 79894454c833b..2f742efa084a8 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -81,6 +81,13 @@ class FunctionSecretArgumentsFinder { return count != 0 || !nested_maps.empty() || !replaced_arguments.empty() || !masked_arguments.empty(); } + + /// Whether the argument at raw index `n` is hidden: by the span, individually or by a partial + /// replacement. The nested maps are not positional: their entries are matched by name. + bool isSecretArgument(size_t n) const + { + return (n >= start && n - start < count) || masked_arguments.contains(n) || replaced_arguments.contains(n); + } }; explicit FunctionSecretArgumentsFinder(std::unique_ptr && function_) : function(std::move(function_)) {} diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index bbb12fb77553a..f38dcd91240f7 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -1224,24 +1224,17 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi /// gates on the setting). void markFoldedSecretConstants(const FunctionNode & function_node, const ActionsDAG::NodeRawConstPtrs & children) { - auto secret_arguments = FunctionSecretArgumentsFinderTreeNode(function_node).getResult(); + const auto secret_arguments = FunctionSecretArgumentsFinderTreeNode(function_node).getResult(); if (!secret_arguments.hasSecrets()) return; - auto mark = [&](size_t index) + for (size_t i = 0; i < children.size(); ++i) { /// Any node carrying a constant column is a folded secret value, whether it is a plain COLUMN /// node or a FUNCTION node folded to a constant (e.g. `concat(k1, k2)`); flag either. - if (index < children.size() && children[index]->column && !children[index]->is_masked_secret) - const_cast(children[index])->is_masked_secret = true; - }; - - for (size_t i = secret_arguments.start; i < secret_arguments.start + secret_arguments.count; ++i) - mark(i); - for (const auto & [index, _] : secret_arguments.masked_arguments) - mark(index); - for (const auto & [index, _] : secret_arguments.replaced_arguments) - mark(index); + if (secret_arguments.isSecretArgument(i) && children[i]->column && !children[i]->is_masked_secret) + const_cast(children[i])->is_masked_secret = true; + } } PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitFunction(const QueryTreeNodePtr & node) diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index cb3a85558dca1..106a9db4f21ba 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -644,7 +644,7 @@ void QueryPlan::explainPlan( PrettyNamesPerPlan local_pretty_names; if (options.pretty && !precomputed_pretty_names) { - local_pretty_names = QueryPlanFormat::buildPrettyNamesPerPlan(*this); + local_pretty_names = QueryPlanFormat::buildPrettyNamesPerPlan(*this, options.show_secrets); precomputed_pretty_names = &local_pretty_names; } @@ -663,6 +663,7 @@ void QueryPlan::explainPlan( .write_header = options.header, .compact = options.compact, .pretty = options.pretty, + .show_secrets = options.show_secrets, .pretty_names = plan_pretty_names ? plan_pretty_names->pretty_names : empty_pretty_names.pretty_names, .runtime_filter_names = plan_pretty_names ? plan_pretty_names->runtime_filter_names : empty_pretty_names.runtime_filter_names }; diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index e465bb92bed4b..ed67ba1a6526b 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -81,6 +81,9 @@ struct ExplainPlanOptions bool compact = false; /// Print query plan with pretty formatting bool pretty = false; + /// Print the values of secret function arguments (keys, passwords) in pretty expressions instead of + /// `[HIDDEN]`. Not a user-facing EXPLAIN option: set from `canDisplaySecrets` by the interpreter. + bool show_secrets = false; /// Show estimates bool estimates = false; /// For EXPLAIN ANALYZE: print the per-processor elapsed time distribution (min/median/max/sum). diff --git a/src/Processors/QueryPlan/QueryPlanFormat.cpp b/src/Processors/QueryPlan/QueryPlanFormat.cpp index e59246e92fe11..f50b80372eced 100644 --- a/src/Processors/QueryPlan/QueryPlanFormat.cpp +++ b/src/Processors/QueryPlan/QueryPlanFormat.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -146,12 +147,34 @@ namespace QueryPlanFormat out << '\n'; } + /// Whether secret function arguments (keys, passwords) render as `[HIDDEN]` or as written. + enum class SecretRendering + { + ShowAll, + HideSecrets, + }; + + static String formatNodePretty( + const ActionsDAG::Node * node, + const PrettyColumnNameMap & pretty_names, + const PrettyRuntimeFilterNameMap & runtime_filter_names, + PrettySetNameMap & subquery_set_names, + SecretRendering secrets, + int parent_precedence = 0); + + String formatNodePretty(const ActionsDAG::Node * node, const ExplainFormatSettings & settings, PrettySetNameMap & subquery_set_names) + { + const auto secrets = settings.show_secrets ? SecretRendering::ShowAll : SecretRendering::HideSecrets; + return formatNodePretty(node, settings.pretty_names, settings.runtime_filter_names, subquery_set_names, secrets); + } + static PrettyColumnName formatFilterPretty( const ActionsDAG & dag, const String & column_name, const std::unordered_map & pretty_names, const std::unordered_map & runtime_filter_names, - std::unordered_map & subquery_set_names) + std::unordered_map & subquery_set_names, + SecretRendering secrets) { const auto * root = dag.tryFindInOutputs(column_name); if (!root) @@ -166,9 +189,9 @@ namespace QueryPlanFormat if (atom->type == ActionsDAG::ActionType::FUNCTION && atom->function_base && atom->function_base->getName() == "__applyFilter") - rf_parts.push_back(formatNodePretty(atom, pretty_names, runtime_filter_names, subquery_set_names, 4)); + rf_parts.push_back(formatNodePretty(atom, pretty_names, runtime_filter_names, subquery_set_names, secrets, 4)); else - user_parts.push_back(formatNodePretty(atom, pretty_names, runtime_filter_names, subquery_set_names, 4)); + user_parts.push_back(formatNodePretty(atom, pretty_names, runtime_filter_names, subquery_set_names, secrets, 4)); } String expression; @@ -313,11 +336,12 @@ namespace QueryPlanFormat } } - String formatNodePretty( + static String formatNodePretty( const ActionsDAG::Node * node, const std::unordered_map & pretty_names, const std::unordered_map & runtime_filter_names, std::unordered_map & subquery_set_names, + SecretRendering secrets, int parent_precedence) { using ActionType = ActionsDAG::ActionType; @@ -340,10 +364,10 @@ namespace QueryPlanFormat case ActionType::COLUMN: return formatConstant(node); case ActionType::ALIAS: - return formatNodePretty(node->children.front(), pretty_names, runtime_filter_names, subquery_set_names, parent_precedence); + return formatNodePretty(node->children.front(), pretty_names, runtime_filter_names, subquery_set_names, secrets, parent_precedence); case ActionType::ARRAY_JOIN: - return "arrayJoin(" + formatNodePretty(node->children.front(), pretty_names, runtime_filter_names, subquery_set_names) + ")"; + return "arrayJoin(" + formatNodePretty(node->children.front(), pretty_names, runtime_filter_names, subquery_set_names, secrets, 0) + ")"; case ActionType::FUNCTION: { @@ -373,7 +397,7 @@ namespace QueryPlanFormat if ((func_name == "_CAST" || func_name == "CAST") && node->children.size() == 2) { - auto inner = formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names); + auto inner = formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, 0); Field type_field; node->children[1]->column->get(0, type_field); return "CAST(" + inner + " AS " + type_field.safeGet() + ")"; @@ -383,7 +407,7 @@ namespace QueryPlanFormat if (func_name == "not" && node->children.size() == 1) { - String result = "NOT " + formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence); + String result = "NOT " + formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence); if (op_info->precedence < parent_precedence) result = "(" + std::move(result) + ")"; return result; @@ -391,17 +415,17 @@ namespace QueryPlanFormat if (func_name == "negate" && node->children.size() == 1) { - String result = "-" + formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence); + String result = "-" + formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence); if (op_info->precedence < parent_precedence) result = "(" + std::move(result) + ")"; return result; } if (func_name == "isNull" && node->children.size() == 1) - return formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence) + " IS NULL"; + return formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence) + " IS NULL"; if (func_name == "isNotNull" && node->children.size() == 1) - return formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence) + " IS NOT NULL"; + return formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence) + " IS NOT NULL"; if ((func_name == "and" || func_name == "or") && node->children.size() >= 2) { @@ -409,7 +433,7 @@ namespace QueryPlanFormat std::vector parts; parts.reserve(node->children.size()); for (const auto * child : node->children) - parts.push_back(formatNodePretty(child, pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence)); + parts.push_back(formatNodePretty(child, pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence)); String result = fmt::format("{}", fmt::join(parts, separator)); if (op_info->precedence < parent_precedence) @@ -419,22 +443,22 @@ namespace QueryPlanFormat if (func_name == "arrayElement" && node->children.size() == 2) { - auto arr = formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence); - auto idx = formatNodePretty(node->children[1], pretty_names, runtime_filter_names, subquery_set_names); + auto arr = formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence); + auto idx = formatNodePretty(node->children[1], pretty_names, runtime_filter_names, subquery_set_names, secrets, 0); return arr + "[" + idx + "]"; } if (func_name == "tupleElement" && node->children.size() == 2) { - auto tup = formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence); - auto elem = formatNodePretty(node->children[1], pretty_names, runtime_filter_names, subquery_set_names); + auto tup = formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence); + auto elem = formatNodePretty(node->children[1], pretty_names, runtime_filter_names, subquery_set_names, secrets, 0); return tup + "." + elem; } if (op_info && (op_info->symbol == "IN" || op_info->symbol == "NOT IN") && node->children.size() == 2) { - auto lhs = formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence); + auto lhs = formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence); auto rhs = formatSetPretty(node->children[1], subquery_set_names); String result = fmt::format("{} {} {}", lhs, op_info->symbol, rhs); if (op_info->precedence < parent_precedence) @@ -445,18 +469,28 @@ namespace QueryPlanFormat if (op_info && !op_info->symbol.empty() && node->children.size() == 2) { String result = fmt::format("{} {} {}", - formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence), + formatNodePretty(node->children[0], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence), op_info->symbol, - formatNodePretty(node->children[1], pretty_names, runtime_filter_names, subquery_set_names, op_info->precedence)); + formatNodePretty(node->children[1], pretty_names, runtime_filter_names, subquery_set_names, secrets, op_info->precedence)); if (op_info->precedence < parent_precedence) result = "(" + std::move(result) + ")"; return result; } + /// A secret argument is hidden whole, as `SHOW CREATE` does. Rendering its structure would + /// have to catch every place a value can surface (constants, `IN` sets, ...), so fail closed. + std::optional secret_arguments; + if (secrets == SecretRendering::HideSecrets) + secret_arguments = FunctionSecretArgumentsFinderActionsDAG(*node).getResult(); std::vector args; args.reserve(node->children.size()); - for (const auto * child : node->children) - args.push_back(formatNodePretty(child, pretty_names, runtime_filter_names, subquery_set_names)); + for (size_t i = 0; i < node->children.size(); ++i) + { + if (secret_arguments && secret_arguments->isSecretArgument(i)) + args.push_back("[HIDDEN]"); + else + args.push_back(formatNodePretty(node->children[i], pretty_names, runtime_filter_names, subquery_set_names, secrets, 0)); + } return func_name + "(" + fmt::format("{}", fmt::join(args, ", ")) + ")"; } @@ -609,10 +643,11 @@ namespace QueryPlanFormat PrettyColumnNameMap & pretty_names, PrettyRuntimeFilterNameMap & runtime_filter_names, PrettySetNameMap & subquery_set_names, - PerPlanColumnMaps & per_plan_columns) + PerPlanColumnMaps & per_plan_columns, + SecretRendering secrets) { for (auto it = node->children.rbegin(); it != node->children.rend(); ++it) - buildPrettyNamesForNode(*it, pretty_names, runtime_filter_names, subquery_set_names, per_plan_columns); + buildPrettyNamesForNode(*it, pretty_names, runtime_filter_names, subquery_set_names, per_plan_columns, secrets); for (auto * child_plan : node->step->getChildPlans()) { @@ -624,7 +659,7 @@ namespace QueryPlanFormat /// `materialize(materialize(...))`. Runtime-filter and subquery-set names are global ids, /// so they stay shared across the whole tree. auto & child_columns = per_plan_columns[child_plan]; - buildPrettyNamesForNode(child_plan->getRootNode(), child_columns, runtime_filter_names, subquery_set_names, per_plan_columns); + buildPrettyNamesForNode(child_plan->getRootNode(), child_columns, runtime_filter_names, subquery_set_names, per_plan_columns, secrets); } } @@ -636,14 +671,14 @@ namespace QueryPlanFormat const auto & dag = static_cast(step.get())->getExpression(); for (const auto * output : dag.getOutputs()) if (output->type != ActionsDAG::ActionType::INPUT) - pretty_names[output->result_name] = PrettyColumnName(formatNodePretty(output, pretty_names, runtime_filter_names, subquery_set_names)); + pretty_names[output->result_name] = PrettyColumnName(formatNodePretty(output, pretty_names, runtime_filter_names, subquery_set_names, secrets)); } else if (step_name == "Filter") { const auto & dag = static_cast(step.get())->getExpression(); for (const auto * output : dag.getOutputs()) if (output->type != ActionsDAG::ActionType::INPUT) - pretty_names[output->result_name] = PrettyColumnName(formatNodePretty(output, pretty_names, runtime_filter_names, subquery_set_names)); + pretty_names[output->result_name] = PrettyColumnName(formatNodePretty(output, pretty_names, runtime_filter_names, subquery_set_names, secrets)); } else if (step_name == "Aggregating") { @@ -664,7 +699,7 @@ namespace QueryPlanFormat { for (const auto * output : dag->getOutputs()) if (output->type != ActionsDAG::ActionType::INPUT) - pretty_names[output->result_name] = PrettyColumnName(formatNodePretty(output, pretty_names, runtime_filter_names, subquery_set_names)); + pretty_names[output->result_name] = PrettyColumnName(formatNodePretty(output, pretty_names, runtime_filter_names, subquery_set_names, secrets)); } } else if (step_name == "Window") @@ -698,7 +733,8 @@ namespace QueryPlanFormat prewhere->prewhere_column_name, pretty_names, runtime_filter_names, - subquery_set_names); + subquery_set_names, + secrets); } if (auto row_level = source->getRowLevelFilter()) { @@ -707,7 +743,8 @@ namespace QueryPlanFormat row_level->column_name, pretty_names, runtime_filter_names, - subquery_set_names); + subquery_set_names, + secrets); } if (step_name == "ReadFromMergeTree") @@ -720,7 +757,8 @@ namespace QueryPlanFormat deferred_row_level_filter->column_name, pretty_names, runtime_filter_names, - subquery_set_names); + subquery_set_names, + secrets); } if (auto deferred_prewhere = read_from_merge_tree_step->getDeferredPrewhereInfo()) { @@ -729,14 +767,17 @@ namespace QueryPlanFormat deferred_prewhere->prewhere_column_name, pretty_names, runtime_filter_names, - subquery_set_names); + subquery_set_names, + secrets); } } } } - PrettyNamesPerPlan buildPrettyNamesPerPlan(const QueryPlan & plan) + PrettyNamesPerPlan buildPrettyNamesPerPlan(const QueryPlan & plan, bool show_secrets) { + const auto secrets = show_secrets ? SecretRendering::ShowAll : SecretRendering::HideSecrets; + /// Runtime-filter and subquery-set names are global ids; keep them shared across the whole tree /// so their numbering stays consistent regardless of plan boundaries. Only column names are scoped. PrettyRuntimeFilterNameMap runtime_filter_names; @@ -747,7 +788,7 @@ namespace QueryPlanFormat auto & top_columns = per_plan_columns[&plan]; auto * root = plan.getRootNode(); if (root) - buildPrettyNamesForNode(root, top_columns, runtime_filter_names, subquery_set_names, per_plan_columns); + buildPrettyNamesForNode(root, top_columns, runtime_filter_names, subquery_set_names, per_plan_columns, secrets); PrettyNamesPerPlan result; for (auto & [plan_ptr, columns] : per_plan_columns) diff --git a/src/Processors/QueryPlan/QueryPlanFormat.h b/src/Processors/QueryPlan/QueryPlanFormat.h index fbf96856fc3a2..b8d8e625841fc 100644 --- a/src/Processors/QueryPlan/QueryPlanFormat.h +++ b/src/Processors/QueryPlan/QueryPlanFormat.h @@ -66,6 +66,8 @@ struct ExplainFormatSettings bool compact = false; bool pretty = false; bool compact_repeated_processor_chains = false; + /// Print secret function arguments (keys, passwords) as written instead of `[HIDDEN]`. + bool show_secrets = false; const PrettyColumnNameMap & pretty_names; const PrettyRuntimeFilterNameMap & runtime_filter_names; }; @@ -76,16 +78,13 @@ namespace QueryPlanFormat void formatOutputColumns(const PrettyColumnNameMap & pretty_names, WriteBuffer & out, const IQueryPlanStep & step, const String & prefix); void formatJoinOutputColumns(WriteBuffer & out, const IQueryPlanStep & step, const String & prefix); - String formatNodePretty( - const ActionsDAG::Node * node, - const PrettyColumnNameMap & pretty_names, - const PrettyRuntimeFilterNameMap & runtime_filter_names, - PrettySetNameMap & subquery_set_names, - int parent_precedence = 0); + /// Renders a DAG node the way the outputs of an `Expression` step are rendered, for steps that describe + /// an expression computed inside the operator (a join condition) rather than a named plan column. + String formatNodePretty(const ActionsDAG::Node * node, const ExplainFormatSettings & settings, PrettySetNameMap & subquery_set_names); String formatColumnPretty(const String & column_name, const std::unordered_map & pretty_names); std::string_view getColumnAnnotation(const String & column_name, const ExplainFormatSettings & settings); - PrettyNamesPerPlan buildPrettyNamesPerPlan(const QueryPlan & plan); + PrettyNamesPerPlan buildPrettyNamesPerPlan(const QueryPlan & plan, bool show_secrets); } } diff --git a/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.reference b/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.reference new file mode 100644 index 0000000000000..0574372757b1d --- /dev/null +++ b/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.reference @@ -0,0 +1,9 @@ +0 1 +0 +0 1 +0 +0 1 +1 +0 +0 1 +0 1 diff --git a/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.sql b/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.sql new file mode 100644 index 0000000000000..17bc2285af964 --- /dev/null +++ b/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.sql @@ -0,0 +1,72 @@ +-- Tags: no-fasttest +-- no-fasttest: encrypt/decrypt/HMAC require the OpenSSL-based functions, absent in the fast test build. + +-- A secret argument can be bound to a constant that is computed in another plan step: a derived +-- table, or the opposite side of a JOIN. Such an argument is an INPUT while the query is planned, and +-- plan optimizations later replace it with (or inline) the constant. The mask must travel with the +-- argument so the pretty ActionsDAG dump never prints the value. viewExplain lets us assert that no +-- fragment leaks without dumping the config-dependent plan into the reference. + +SET enable_analyzer = 1; +SET format_display_secrets_in_show_and_select = 0; + +-- Key from a derived table, consumed by WHERE: the filter is merged with the projection of the +-- subquery and the masked copy of the constant is deduplicated with an unmasked twin. +SELECT countIf(explain LIKE '%SEKRIT_SUBQKEY%') AS derived_table_leaks, countIf(explain LIKE '%[HIDDEN]%') > 0 AS derived_table_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT number FROM (SELECT number, 'SEKRIT_SUBQKEY' AS k FROM numbers(1)) AS s + WHERE empty(HMAC('sha256', toString(number), s.k)))); + +-- Same, with the key decrypted in the subquery: the plaintext exists only as a folded constant. +SELECT countIf(explain LIKE '%SEKRIT_DECRYPTED%') AS derived_table_decrypt_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT number FROM ( + SELECT number, decrypt('aes-128-ecb', encrypt('aes-128-ecb', 'SEKRIT_DECRYPTED', '0123456789abcdef'), '0123456789abcdef') AS k + FROM numbers(1)) AS s + WHERE empty(HMAC('sha256', toString(number), s.k)))); + +-- Key from the opposite JOIN side: the ON condition is pushed down to the other side and the +-- constant is inlined in place of the unbound input. +SELECT countIf(explain LIKE '%SEKRIT_JOINKEY%') AS join_leaks, countIf(explain LIKE '%[HIDDEN]%') > 0 AS join_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT n.number FROM numbers(1) AS n + INNER JOIN (SELECT 'SEKRIT_JOINKEY' AS k) AS s ON HMAC('sha256', toString(n.number), s.k) = '')); + +SELECT countIf(explain LIKE '%SEKRIT_JOINKEY%') AS join_logical_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1, keep_logical_steps = 1', ( + SELECT n.number FROM numbers(1) AS n + INNER JOIN (SELECT 'SEKRIT_JOINKEY' AS k) AS s ON HMAC('sha256', toString(n.number), s.k) = '')); + +-- A non-constant key from the opposite JOIN side stays an input; the pretty dump must not expand it +-- into the expression the other side computes it from. The slot is hidden whole, as SHOW CREATE does. +SELECT countIf(explain LIKE '%SEKRIT_MATKEY%') AS join_column_leaks, countIf(explain LIKE '%HMAC(''sha256'', toString(number), [HIDDEN])%') AS join_column_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT n.number FROM numbers(1) AS n + INNER JOIN (SELECT materialize('SEKRIT_MATKEY') AS k) AS s ON HMAC('sha256', toString(n.number), s.k) = '')); + +-- Every argument in a secret position is hidden whole, a column expression included: the secret +-- span of encrypt covers the plaintext as well as the key. +SELECT countIf(explain LIKE '%encrypt(''aes-128-ecb'', [HIDDEN], [HIDDEN])%') AS secret_span_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT encrypt('aes-128-ecb', toString(number), 'SEKRIT_LITERALKEY') FROM numbers(1))); + +-- A literal set inside a secret slot: the IN operator formats its set without going through the +-- per-node rendering, so the slot must be hidden before descending into the argument. +SELECT countIf(explain LIKE '%SEKRIT_IN%') AS in_set_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT number FROM numbers(1) + WHERE empty(HMAC('sha256', toString(number), if(toString(number) IN ('SEKRIT_IN1', 'SEKRIT_IN2'), 'a', 'b'))))); + +-- The exact shape from the report: the HMAC key is a decrypt() of a hex ciphertext with an IV, +-- computed on the opposite JOIN side. +SELECT countIf(explain LIKE '%JOIN_SIDE_SECRET_PLAINTEXT%') AS report_join_leaks, countIf(explain LIKE '%[HIDDEN]%') > 0 AS report_join_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT n.number FROM numbers(1) AS n + INNER JOIN ( + SELECT decrypt('aes-128-cbc', unhex(hex(encrypt('aes-128-cbc', 'JOIN_SIDE_SECRET_PLAINTEXT', '0123456789abcdef', 'abcdef9876543210'))), '0123456789abcdef', 'abcdef9876543210') AS k + ) AS s ON HMAC('sha256', toString(n.number), s.k) = '')); + +-- The report's positive control: a key bound through a WITH alias in the same scope. +SELECT countIf(explain LIKE '%WITH_ALIAS_SECRET%') AS with_alias_leaks, countIf(explain LIKE '%[HIDDEN]%') > 0 AS with_alias_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + WITH 'WITH_ALIAS_SECRET' AS k SELECT number FROM numbers(1) WHERE empty(HMAC('sha256', toString(number), k)))); diff --git a/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.reference b/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.reference new file mode 100644 index 0000000000000..c3da9be8f327d --- /dev/null +++ b/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.reference @@ -0,0 +1,76 @@ +-- the attacker has no privilege to display secrets +0 +-- protected_v: SHOW CREATE hides the secret arguments +2 +0 +-- protected_v: SELECT works and returns nothing +0 +-- protected_v: EXPLAIN PLAN actions = 1: lines with the secret +0 +-- protected_v: EXPLAIN PLAN actions = 1, pretty = 1: lines with the secret +0 +-- protected_v: EXPLAIN PLAN header = 1: lines with the secret +0 +-- protected_v: EXPLAIN PLAN actions = 1, pretty = 0, compact = 0: lines with the secret +0 +-- protected_v: EXPLAIN PLAN json = 1, actions = 1: lines with the secret +0 +-- protected_v: EXPLAIN QUERY TREE and EXPLAIN SYNTAX: lines with the secret +0 +0 +-- derived_v: SHOW CREATE hides the secret arguments +2 +0 +-- derived_v: SELECT works and returns nothing +0 +-- derived_v: EXPLAIN PLAN actions = 1: lines with the secret +0 +-- derived_v: EXPLAIN PLAN actions = 1, pretty = 1: lines with the secret +0 +-- derived_v: EXPLAIN PLAN header = 1: lines with the secret +0 +-- derived_v: EXPLAIN PLAN actions = 1, pretty = 0, compact = 0: lines with the secret +0 +-- derived_v: EXPLAIN PLAN json = 1, actions = 1: lines with the secret +0 +-- derived_v: EXPLAIN QUERY TREE and EXPLAIN SYNTAX: lines with the secret +0 +0 +-- pos_literal_v: SHOW CREATE hides the secret arguments +1 +0 +-- pos_literal_v: SELECT works and returns nothing +0 +-- pos_literal_v: EXPLAIN PLAN actions = 1: lines with the secret +0 +-- pos_literal_v: EXPLAIN PLAN actions = 1, pretty = 1: lines with the secret +0 +-- pos_literal_v: EXPLAIN PLAN header = 1: lines with the secret +0 +-- pos_literal_v: EXPLAIN PLAN actions = 1, pretty = 0, compact = 0: lines with the secret +0 +-- pos_literal_v: EXPLAIN PLAN json = 1, actions = 1: lines with the secret +0 +-- pos_literal_v: EXPLAIN QUERY TREE and EXPLAIN SYNTAX: lines with the secret +0 +0 +-- pos_alias_v: SHOW CREATE hides the secret arguments +1 +1 +-- pos_alias_v: SELECT works and returns nothing +0 +-- pos_alias_v: EXPLAIN PLAN actions = 1: lines with the secret +0 +-- pos_alias_v: EXPLAIN PLAN actions = 1, pretty = 1: lines with the secret +0 +-- pos_alias_v: EXPLAIN PLAN header = 1: lines with the secret +0 +-- pos_alias_v: EXPLAIN PLAN actions = 1, pretty = 0, compact = 0: lines with the secret +0 +-- pos_alias_v: EXPLAIN PLAN json = 1, actions = 1: lines with the secret +0 +-- pos_alias_v: EXPLAIN QUERY TREE and EXPLAIN SYNTAX: lines with the secret +0 +0 +-- without the grant the plan is not reachable at all +ACCESS_DENIED diff --git a/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.sh b/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.sh new file mode 100755 index 0000000000000..2fbecdf4fd1f2 --- /dev/null +++ b/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: the encryption functions are not available in the fast test build +# Tag no-replicated-database: SQL SECURITY DEFINER views and users are set up per-test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Reproduces the reported scenario: a SQL SECURITY DEFINER view derives its HMAC key with decrypt() +# on the opposite side of a JOIN (or in a derived table). A user holding only SELECT on the view, +# without the displaySecretsInShowAndSelect privilege, must not recover the decrypted key through +# any EXPLAIN flavour, while SHOW CREATE keeps hiding the decrypt() arguments and SELECT stays usable. +# The plans are not dumped into the reference (they depend on the configuration); only the absence +# of the plaintext is checked. + +db=${CLICKHOUSE_DATABASE} +owner="owner_${db}_$RANDOM" +attacker="attacker_${db}_$RANDOM" +nogrant="nogrant_${db}_$RANDOM" +key='0123456789abcdef' +iv='abcdef9876543210' +plaintext='JOIN_SIDE_SECRET_PLAINTEXT' + +ciphertext=$(${CLICKHOUSE_CLIENT} --query "SELECT hex(encrypt('aes-128-cbc', '$plaintext', '$key', '$iv'))") + +${CLICKHOUSE_CLIENT} <&1 | grep -o "ACCESS_DENIED" | head -1 + +for view in protected_v derived_v pos_literal_v pos_alias_v; do + ${CLICKHOUSE_CLIENT} --query "DROP VIEW $db.$view" +done +${CLICKHOUSE_CLIENT} --query "DROP USER $owner, $attacker, $nogrant" diff --git a/tests/queries/0_stateless/05219_explain_show_secrets_with_privilege.reference b/tests/queries/0_stateless/05219_explain_show_secrets_with_privilege.reference new file mode 100644 index 0000000000000..6f38766a208a9 --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_show_secrets_with_privilege.reference @@ -0,0 +1,18 @@ +-- format_display_secrets_in_show_and_select = 0 +-- same-side literal +Filter column: empty(HMAC(\'sha256\', toString(number), [HIDDEN])) +-- key from a derived table +Filter column: empty(HMAC(\'sha256\', toString(number), [HIDDEN])) +-- key from the opposite JOIN side +Filter column: empty(HMAC(\'sha256\', toString(number), [HIDDEN])) +-- legacy format +Filter column: empty(HMAC(\'sha256\'_String, toString(__table1.number), [HIDDEN id: 1]_String)) (removed) +-- format_display_secrets_in_show_and_select = 1 +-- same-side literal +Filter column: empty(HMAC(\'sha256\', toString(number), \'LITERAL_SECRET\')) +-- key from a derived table +Filter column: empty(HMAC(\'sha256\', toString(number), \'DERIVED_SECRET\')) +-- key from the opposite JOIN side +Filter column: empty(HMAC(\'sha256\', toString(number), \'JOIN_SECRET\')) +-- legacy format +Filter column: empty(HMAC(\'sha256\'_String, toString(__table1.number), \'LITERAL_SECRET\'_String)) (removed) diff --git a/tests/queries/0_stateless/05219_explain_show_secrets_with_privilege.sh b/tests/queries/0_stateless/05219_explain_show_secrets_with_privilege.sh new file mode 100755 index 0000000000000..fc33418d25300 --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_show_secrets_with_privilege.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the encryption functions are not available in the fast test build + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The pretty plan hides secret function arguments unless the user may display secrets: the server +# setting `display_secrets_in_show_and_select`, the `displaySecretsInShowAndSelect` privilege and the +# session setting `format_display_secrets_in_show_and_select` must all be on. The stateless server has +# the server setting off, so clickhouse-local is used to enable it; the default user there holds every +# privilege. Only the filter lines are printed: the rest of the plan depends on the configuration. + +explain_filters() { + ${CLICKHOUSE_LOCAL} --enable_analyzer=1 --format_display_secrets_in_show_and_select="$1" \ + --query "$2" -- --display_secrets_in_show_and_select=1 | grep -o "Filter column: .*" +} + +for show in 0 1; do + echo "-- format_display_secrets_in_show_and_select = $show" + + echo "-- same-side literal" + explain_filters "$show" "EXPLAIN PLAN actions = 1 SELECT number FROM numbers(1) WHERE empty(HMAC('sha256', toString(number), 'LITERAL_SECRET'))" + + echo "-- key from a derived table" + explain_filters "$show" "EXPLAIN PLAN actions = 1 SELECT number FROM (SELECT number, 'DERIVED_SECRET' AS k FROM numbers(1)) AS s WHERE empty(HMAC('sha256', toString(number), s.k))" + + echo "-- key from the opposite JOIN side" + explain_filters "$show" "EXPLAIN PLAN actions = 1 SELECT n.number FROM numbers(1) AS n INNER JOIN (SELECT 'JOIN_SECRET' AS k) AS s ON HMAC('sha256', toString(n.number), s.k) = ''" + + echo "-- legacy format" + explain_filters "$show" "EXPLAIN PLAN actions = 1, pretty = 0 SELECT number FROM numbers(1) WHERE empty(HMAC('sha256', toString(number), 'LITERAL_SECRET'))" +done From d360571b31b12aa6e6e5d167e2f9a3a9d4f543f3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 15:33:15 +0000 Subject: [PATCH 200/296] Backport #122697 to 26.8: Fix restoring a lightweight snapshot from Azure when the source disk has a key prefix --- src/Backups/BackupIO_AzureBlobStorage.cpp | 78 +++++++++++++++++++ src/Backups/BackupIO_AzureBlobStorage.h | 16 ++++ .../registerBackupEngineAzureBlobStorage.cpp | 10 ++- 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 30446234dc677..5b6112e9f80f7 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -13,8 +13,11 @@ #include #include +#include #include #include +#include +#include #include @@ -26,6 +29,7 @@ namespace DB namespace ErrorCodes { + extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; } @@ -61,7 +65,81 @@ namespace {"sdk_retry_max_backoff_ms", std::to_string(settings.sdk_retry_max_backoff_ms)}, }; } + +#if CLICKHOUSE_CLOUD + /// The recorded endpoint of a snapshot's source disk as a service URL: scheme, host, port and path, without + /// query parameters and trailing slashes. A source disk that authenticates with a SAS records it in the + /// query, and one configured with a connection string records the connection string itself; the snapshot + /// is read with the backup's credential in any case, so only the service URL is kept. + String snapshotServiceURL(const String & endpoint) + { + Azure::Core::Url parsed; + try + { + parsed = endpoint.starts_with("http") ? Azure::Core::Url(endpoint) + : Azure::Storage::_internal::ParseConnectionString(endpoint).BlobServiceUrl; + } + catch (const std::exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse the endpoint of a lightweight snapshot: {}", e.what()); + } + if (parsed.GetScheme().empty() || parsed.GetHost().empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The endpoint of a lightweight snapshot is not a service URL"); + + String url = parsed.GetScheme() + "://" + parsed.GetHost(); + if (parsed.GetPort() != 0) + url += ":" + std::to_string(parsed.GetPort()); + if (!parsed.GetPath().empty()) + url += "/" + parsed.GetPath(); + while (url.ends_with('/')) + url.pop_back(); + return url; + } + + /// The connection string with its blob endpoint replaced by `service_url`. `CreateFromConnectionString()` + /// takes the endpoint from the connection string alone, so this is how a connection string is pointed at + /// the recorded endpoint of a snapshot; the account key or SAS it carries then authorises the reads there, + /// or Azure refuses them if it belongs to another account. + String withBlobEndpoint(const String & connection_string, const String & service_url) + { + std::vector parts; + boost::split(parts, connection_string, boost::is_any_of(";")); + String result; + for (const auto & part : parts) + if (!part.empty() && !part.starts_with("BlobEndpoint=")) + result += part + ";"; + return result + "BlobEndpoint=" + service_url; + } +#endif +} + +#if CLICKHOUSE_CLOUD +AzureBlobStorage::ConnectionParams makeSnapshotSourceConnectionParams( + const AzureBlobStorage::ConnectionParams & backup_connection_params, const String & endpoint, const String & blob_namespace) +{ + auto connection_params = backup_connection_params; + + /// The objects are read from the recorded endpoint with the backup's credential, as the S3 reader does. + const String service_url = snapshotServiceURL(endpoint); + if (const auto * connection_string = std::get_if(&connection_params.auth_method)) + { + const String repointed = withBlobEndpoint(connection_string->toUnderType(), service_url); + connection_params.auth_method = AzureBlobStorage::ConnectionString{repointed}; + connection_params.endpoint.storage_account_url = repointed; + } + else + connection_params.endpoint.storage_account_url = service_url; + + const auto slash_pos = blob_namespace.find('/'); + connection_params.endpoint.container_name = blob_namespace.substr(0, slash_pos); + connection_params.endpoint.prefix = (slash_pos == String::npos) ? "" : blob_namespace.substr(slash_pos + 1); + + /// The snapshot was taken from this container, so it exists; the existence check is also a + /// container-level request. + connection_params.endpoint.container_already_exists = true; + return connection_params; } +#endif BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( const AzureBlobStorage::ConnectionParams & connection_params_, diff --git a/src/Backups/BackupIO_AzureBlobStorage.h b/src/Backups/BackupIO_AzureBlobStorage.h index 3f262f83316b5..4df6207442f17 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.h +++ b/src/Backups/BackupIO_AzureBlobStorage.h @@ -10,6 +10,22 @@ namespace DB { +#if CLICKHOUSE_CLOUD +/// Connection params for reading the objects a lightweight snapshot references, built from the endpoint +/// and object namespace its manifest records for the source disk. For Azure that namespace is `` +/// or `/` when the disk endpoint has a path below the container +/// (`AzureObjectStorage::getObjectsNamespace`), while the recorded object keys are relative to the prefix, so +/// it is split back into a container and a blob prefix: used as a container name, `/` +/// gives every container-level request extra path segments, which Azure rejects with `400 InvalidUri`. +/// The objects are read from the recorded endpoint with the backup's credential, as the S3 reader does. A +/// connection string is pointed there by replacing its `BlobEndpoint`, since `CreateFromConnectionString()` +/// takes the endpoint from the connection string alone; its key or SAS then authorises the reads, or Azure +/// refuses them if it belongs to another account. Query parameters of the recorded endpoint (a SAS of the +/// source disk) are dropped, and an endpoint recorded as a connection string is reduced to its service URL. +AzureBlobStorage::ConnectionParams makeSnapshotSourceConnectionParams( + const AzureBlobStorage::ConnectionParams & backup_connection_params, const String & endpoint, const String & blob_namespace); +#endif + /// Represents a backup stored to Azure class BackupReaderAzureBlobStorage : public BackupReaderDefault { diff --git a/src/Backups/registerBackupEngineAzureBlobStorage.cpp b/src/Backups/registerBackupEngineAzureBlobStorage.cpp index d4e63706d270a..d3d4afa57d654 100644 --- a/src/Backups/registerBackupEngineAzureBlobStorage.cpp +++ b/src/Backups/registerBackupEngineAzureBlobStorage.cpp @@ -301,12 +301,11 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory) params.write_settings, params.context); - auto snapshot_reader_creator = [&](const String & endpoint, const String & container_name) +#if CLICKHOUSE_CLOUD + auto snapshot_reader_creator = [&](const String & endpoint, const String & blob_namespace) { - connection_params.endpoint.storage_account_url = endpoint; - connection_params.endpoint.container_name = container_name; return std::make_shared( - connection_params, + makeSnapshotSourceConnectionParams(connection_params, endpoint, blob_namespace), "", params.allow_azure_native_copy, params.read_settings, @@ -315,6 +314,9 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory) }; return std::make_unique(params, archive_params, reader, snapshot_reader_creator); +#else + return std::make_unique(params, archive_params, reader); +#endif } auto writer = std::make_shared( From f07e25e18397644d1ea5286a3d06bf397a245f6b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 16:54:12 +0000 Subject: [PATCH 201/296] Backport #122672 to 26.8: Respect validate_enum_literals_in_operators in index analysis and notEquals --- src/DataTypes/DataTypeEnum.cpp | 19 +++++ src/DataTypes/DataTypeEnum.h | 3 + src/Functions/FunctionsComparison.h | 2 +- src/Storages/MergeTree/KeyCondition.cpp | 24 ++++++ src/Storages/MergeTree/KeyCondition.h | 2 + .../MergeTree/MergeTreeIndexBloomFilter.cpp | 27 ++++++- .../MergeTree/MergeTreeIndexBloomFilter.h | 1 + ...um_unknown_literal_key_condition.reference | 30 ++++++++ ...243_enum_unknown_literal_key_condition.sql | 75 +++++++++++++++++++ 9 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.reference create mode 100644 tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.sql diff --git a/src/DataTypes/DataTypeEnum.cpp b/src/DataTypes/DataTypeEnum.cpp index 837b3c8ceb430..16376725fff3e 100644 --- a/src/DataTypes/DataTypeEnum.cpp +++ b/src/DataTypes/DataTypeEnum.cpp @@ -217,6 +217,25 @@ SerializationPtr DataTypeEnum::doGetSerialization(const SerializationInfoS template class DataTypeEnum; template class DataTypeEnum; +bool isUnknownEnumElement(const IDataType & type, const Field & value) +{ + if (value.getType() != Field::Types::String) + return false; + + const auto & name = value.safeGet(); + if (const auto * enum8 = typeid_cast(&type)) + { + Int8 res = 0; + return !enum8->tryGetValue(res, name); + } + if (const auto * enum16 = typeid_cast(&type)) + { + Int16 res = 0; + return !enum16->tryGetValue(res, name); + } + return false; +} + static void checkASTStructure(const ASTPtr & child) { const auto * func = child->as(); diff --git a/src/DataTypes/DataTypeEnum.h b/src/DataTypes/DataTypeEnum.h index 8e5222499559c..a9385142f215f 100644 --- a/src/DataTypes/DataTypeEnum.h +++ b/src/DataTypes/DataTypeEnum.h @@ -91,6 +91,9 @@ class DataTypeEnum final : public IDataTypeEnum, public EnumValues size_t getRelativeFlagsSize() const { return relative_flags.size(); } }; +/// True when `type` is an `Enum` and `value` is a `String` that names none of its elements. +bool isUnknownEnumElement(const IDataType & type, const Field & value); + template DataTypePtr mergeEnumTypes(const DataTypeEnum & base, const DataTypeEnum & add); diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 4b67844f86259..5b7a8946e4e7f 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -1194,7 +1194,7 @@ class FunctionComparison final : public IFunction auto is_string_not_in_enum = [this, &string_value](const EnumValues * enum_values) -> bool { - if constexpr (!IsOperation::equals && IsOperation::not_equals) + if constexpr (!IsOperation::equals && !IsOperation::not_equals) return false; if (params.validate_enum_literals_in_operators) return false; diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 5069cdb88714d..92fc44f782939 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -64,6 +65,7 @@ namespace Setting extern const SettingsBool analyze_index_with_space_filling_curves; extern const SettingsDateTimeOverflowBehavior date_time_overflow_behavior; extern const SettingsTimezone session_timezone; + extern const SettingsBool validate_enum_literals_in_operators; } namespace ErrorCodes @@ -1473,6 +1475,7 @@ KeyCondition::KeyCondition( , single_point(single_point_) , date_time_overflow_behavior_ignore( context->getSettingsRef()[Setting::date_time_overflow_behavior] == FormatSettings::DateTimeOverflowBehavior::Ignore) + , validate_enum_literals_in_operators(context->getSettingsRef()[Setting::validate_enum_literals_in_operators]) { size_t key_index = 0; for (const auto & name : key_column_names_) @@ -4085,6 +4088,27 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, const Bu return false; } + /// With validation off, `equals`/`notEquals` fold an unknown enum literal to a + /// constant instead of throwing, so the index must do the same instead of converting. + /// Nullable keys are declined, as for NaN above: `NULL 'x'` is NULL, not a constant. + if (!validate_enum_literals_in_operators && isUnknownEnumElement(*key_expr_type_not_null, const_value)) + { + if (key_expr_type_is_nullable) + return false; + + if (func_name == "equals") + { + out.function = RPNElement::ALWAYS_FALSE; + return true; + } + if (func_name == "notEquals") + { + out.function = RPNElement::ALWAYS_TRUE; + return true; + } + return false; + } + const_value = convertFieldToType(const_value, *key_expr_type_not_null); if (const_value.isNull()) return false; diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 9f0a1264cbe50..a7996da866f76 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -689,6 +689,8 @@ class KeyCondition /// Holds the result of (setting.date_time_overflow_behavior == DateTimeOverflowBehavior::Ignore) /// Used to check toDateTime monotonicity. bool date_time_overflow_behavior_ignore; + /// Consulted only while extracting atoms, so the private copying constructor leaves it unset. + bool validate_enum_literals_in_operators = true; /// Holds whether the key columns are sorted in reverse (ORDER BY ... DESC) or not. KeyOrder key_order; diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index 87c2b6dabf7e8..356091fd75fc3 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -15,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +34,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool validate_enum_literals_in_operators; +} + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -310,7 +318,10 @@ std::optional tryResolveMapInfoFromNode(const RPNBuilderTreeNode & MergeTreeIndexConditionBloomFilter::MergeTreeIndexConditionBloomFilter( const ActionsDAG::Node * predicate, ContextPtr context_, const Block & header_, size_t hash_functions_) - : WithContext(context_), header(header_), hash_functions(hash_functions_) + : WithContext(context_) + , header(header_) + , hash_functions(hash_functions_) + , validate_enum_literals_in_operators(context_->getSettingsRef()[Setting::validate_enum_literals_in_operators]) { if (!predicate) { @@ -995,6 +1006,12 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( if (array_type && bloomFilterHashDomainMatches(value_type, array_type->getNestedType())) { const DataTypePtr actual_type = BloomFilter::getPrimitiveType(array_type->getNestedType()); + if (!validate_enum_literals_in_operators && isUnknownEnumElement(*actual_type, value_field)) + { + out.function = RPNElement::ALWAYS_FALSE; + return true; + } + auto converted_field = convertFieldToType(value_field, *actual_type, value_type.get()); if (converted_field.isNull()) return false; @@ -1103,6 +1120,14 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( return false; } + /// With validation off, `equals` against a name the enum lacks is constant false for every row, + /// so the conversion below must not throw; `KeyCondition` folds the same way. + if (function_name == "equals" && !validate_enum_literals_in_operators && isUnknownEnumElement(*actual_type, value_field)) + { + out.function = RPNElement::ALWAYS_FALSE; + return true; + } + auto converted_field = convertFieldToType(value_field, *actual_type, value_type.get()); if (converted_field.isNull()) return false; diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h index e0b19f513f71a..c978cd7c66869 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h @@ -91,6 +91,7 @@ class MergeTreeIndexConditionBloomFilter final : public IMergeTreeIndexCondition private: const Block & header; const size_t hash_functions; + const bool validate_enum_literals_in_operators; std::vector rpn; bool mayBeTrueOnGranule(const MergeTreeIndexGranuleBloomFilter * granule, const UpdatePartialDisjunctionResultFn & update_partial_result_disjuntion_fn) const; diff --git a/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.reference b/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.reference new file mode 100644 index 0000000000000..1b83138d790f1 --- /dev/null +++ b/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.reference @@ -0,0 +1,30 @@ +key column +1 +0 +0 +2 +2 +0 +1 +1 +1 +2 +non-key column +0 +2 +bloom filter index +0 +2 +2 +0 +1 +0 +0 +1 +0 +nullable key +0 +1 +partition key +0 +2 diff --git a/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.sql b/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.sql new file mode 100644 index 0000000000000..c0ef817ac1d7a --- /dev/null +++ b/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.sql @@ -0,0 +1,75 @@ +DROP TABLE IF EXISTS t_enum_key; +DROP TABLE IF EXISTS t_enum_nokey; + +CREATE TABLE t_enum_key (e Enum8('a' = 1, 'b' = 2), v UInt8) ENGINE = MergeTree ORDER BY e; +INSERT INTO t_enum_key VALUES ('a', 1), ('b', 2); + +CREATE TABLE t_enum_nokey (e Enum16('a' = 1, 'b' = 2), v UInt8) ENGINE = MergeTree ORDER BY v; +INSERT INTO t_enum_nokey VALUES ('a', 1), ('b', 2); + +SET validate_enum_literals_in_operators = 0; + +SELECT 'key column'; +SELECT count() FROM t_enum_key WHERE e = 'a'; +SELECT count() FROM t_enum_key WHERE e = 'c'; +SELECT count() FROM t_enum_key WHERE 'c' = e; +SELECT count() FROM t_enum_key WHERE e != 'c'; +SELECT count() FROM t_enum_key WHERE 'c' != e; +SELECT count() FROM t_enum_key WHERE e IN ('c'); +SELECT count() FROM t_enum_key WHERE e IN ('a', 'c'); +SELECT count() FROM t_enum_key WHERE e = 'c' OR v = 2; +SELECT count() FROM t_enum_key WHERE e != 'c' AND v = 2; +SELECT count() FROM t_enum_key WHERE NOT (e = 'c'); +SELECT count() FROM t_enum_key WHERE e < 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } +SELECT count() FROM t_enum_key WHERE e >= 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } + +SELECT 'non-key column'; +SELECT count() FROM t_enum_nokey WHERE e = 'c'; +SELECT count() FROM t_enum_nokey WHERE e != 'c'; +SELECT count() FROM t_enum_nokey WHERE e < 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } +SELECT count() FROM t_enum_nokey WHERE e >= 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } + +SELECT 'bloom filter index'; +DROP TABLE IF EXISTS t_enum_bf; +CREATE TABLE t_enum_bf (e Enum8('a' = 1, 'b' = 2), arr Array(Enum8('a' = 1, 'b' = 2)), v UInt8, + INDEX bf_e e TYPE bloom_filter GRANULARITY 1, INDEX bf_arr arr TYPE bloom_filter GRANULARITY 1) +ENGINE = MergeTree ORDER BY v; +INSERT INTO t_enum_bf VALUES ('a', ['a'], 1), ('b', ['b'], 2); +SELECT count() FROM t_enum_bf WHERE e = 'c'; +SELECT count() FROM t_enum_bf WHERE e != 'c'; +SELECT count() FROM t_enum_bf WHERE NOT (e = 'c'); +SELECT count() FROM t_enum_bf PREWHERE e = 'c'; +SELECT count() FROM t_enum_bf WHERE e = 'c' OR v = 2; +SELECT count() FROM t_enum_bf ARRAY JOIN arr AS x WHERE x = 'c'; +SELECT count() FROM t_enum_bf WHERE e IN ('c'); +SELECT count() FROM t_enum_bf WHERE e IN ('a', 'c'); +SELECT count() FROM t_enum_bf WHERE arrayJoin(arr) IN ('c'); + +SELECT 'nullable key'; +DROP TABLE IF EXISTS t_enum_null; +CREATE TABLE t_enum_null (e Nullable(Enum8('a' = 1, 'b' = 2)), v UInt8) ENGINE = MergeTree ORDER BY e SETTINGS allow_nullable_key = 1; +INSERT INTO t_enum_null VALUES ('a', 1), (NULL, 2); +SELECT count() FROM t_enum_null WHERE e = 'c'; +SELECT count() FROM t_enum_null WHERE e != 'c'; + +SELECT 'partition key'; +DROP TABLE IF EXISTS t_enum_part; +CREATE TABLE t_enum_part (e Enum8('a' = 1, 'b' = 2), v UInt8) ENGINE = MergeTree PARTITION BY e ORDER BY v; +INSERT INTO t_enum_part VALUES ('a', 1), ('b', 2); +SELECT count() FROM t_enum_part WHERE e = 'c'; +SELECT count() FROM t_enum_part WHERE e != 'c'; + +SET validate_enum_literals_in_operators = 1; + +SELECT count() FROM t_enum_key WHERE e = 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } +SELECT count() FROM t_enum_key WHERE e != 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } +SELECT count() FROM t_enum_key WHERE e IN ('c'); -- { serverError UNKNOWN_ELEMENT_OF_ENUM } +SELECT count() FROM t_enum_nokey WHERE e = 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } +SELECT count() FROM t_enum_nokey WHERE e != 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } +SELECT count() FROM t_enum_bf WHERE e = 'c'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } + +DROP TABLE t_enum_key; +DROP TABLE t_enum_nokey; +DROP TABLE t_enum_bf; +DROP TABLE t_enum_null; +DROP TABLE t_enum_part; From 4d84adac178656bbc4e3a289db9ffd27cb3ebaa7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 17:10:15 +0000 Subject: [PATCH 202/296] Update autogenerated version to 26.8.19.9 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 453a7b1ee28de..c03b781aac3ee 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54531) +SET(VERSION_REVISION 54532) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 19) -SET(VERSION_GITHASH 9f84259255b23824e0b5e684f6e2fe90734f46bb) -SET(VERSION_DESCRIBE v26.8.19.1-lts) -SET(VERSION_STRING 26.8.19.1) +SET(VERSION_PATCH 20) +SET(VERSION_GITHASH 2ea9c3c9e37a707c67c363095a760d24a2019c74) +SET(VERSION_DESCRIBE v26.8.20.1-lts) +SET(VERSION_STRING 26.8.20.1) # end of autochange From 34f42da56078f89c83e2e9a2b1c153a5b628d05b Mon Sep 17 00:00:00 2001 From: Vladimir Cherkasov Date: Tue, 6 Oct 2026 19:19:52 +0200 Subject: [PATCH 203/296] add isSensitiveToEvaluationCount --- .../QueryPlan/Optimizations/Utils.cpp | 19 +++++++++++++++++++ .../QueryPlan/Optimizations/Utils.h | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/src/Processors/QueryPlan/Optimizations/Utils.cpp b/src/Processors/QueryPlan/Optimizations/Utils.cpp index cc87989799db7..3f1f7ec94d4ea 100644 --- a/src/Processors/QueryPlan/Optimizations/Utils.cpp +++ b/src/Processors/QueryPlan/Optimizations/Utils.cpp @@ -113,6 +113,25 @@ bool dagContainsNonDeterministicFunction(const ActionsDAG & dag) return false; } +bool isSensitiveToEvaluationCount(const ActionsDAG & dag) +{ + auto is_insensitive = [](const IFunctionBase & function) + { + return function.isDeterministicInScopeOfQuery() && !function.isStateful() && !function.hasObservableSideEffects(); + }; + + /// A lambda without captures is constant-folded into a `COLUMN` node holding a `ColumnFunction`, which + /// hides the functions of its body from a plain scan over the function nodes; `allNodeFunctions` + /// descends into it. + for (const auto & node : dag.getNodes()) + { + if (!allNodeFunctions(node, is_insensitive)) + return true; + } + + return false; +} + FilterResult filterResultForNotMatchedRows( const ActionsDAG & filter_dag, const String & filter_column_name, diff --git a/src/Processors/QueryPlan/Optimizations/Utils.h b/src/Processors/QueryPlan/Optimizations/Utils.h index d3d17199b54d0..297fcce202903 100644 --- a/src/Processors/QueryPlan/Optimizations/Utils.h +++ b/src/Processors/QueryPlan/Optimizations/Utils.h @@ -76,6 +76,11 @@ enum class FilterResult [[nodiscard]] bool dagContainsNonDeterministicFunction(const ActionsDAG & dag); +/// True when the expression holds a function that is non-deterministic within the query (`rand`), stateful +/// (`rowNumberInAllBlocks`, `aiEmbed`), or has observable side effects (`sleep`), so an observer can see how +/// many times and on which rows it was evaluated. +[[nodiscard]] bool isSensitiveToEvaluationCount(const ActionsDAG & dag); + [[nodiscard]] FilterResult filterResultForNotMatchedRows( const ActionsDAG & filter_dag, const String & filter_column_name, From c557e772c51f68d941df2d7f3a10a0a91038d004 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:34:02 +0000 Subject: [PATCH 204/296] Write isSensitiveToEvaluationCount in 26.8 terms for the backport of #123829 The previous commit copies master's body, which uses `allNodeFunctions` and `IFunctionBase::hasObservableSideEffects`; 26.8 has neither, so `Utils.cpp` does not compile. Check the same properties with what 26.8 has: a function that is non-deterministic within the query or stateful, `sleep` and `sleepEachRow` by name (the only functions master reports as having observable side effects), and any lambda, since on 26.8 neither this check nor `dagContainsNonDeterministicFunction` looks into a lambda body. Related: https://github.com/ClickHouse/ClickHouse/pull/123829 Co-Authored-By: Claude Opus 5.5 --- .../QueryPlan/Optimizations/Utils.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/Utils.cpp b/src/Processors/QueryPlan/Optimizations/Utils.cpp index 3f1f7ec94d4ea..e65f389683cd7 100644 --- a/src/Processors/QueryPlan/Optimizations/Utils.cpp +++ b/src/Processors/QueryPlan/Optimizations/Utils.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -115,17 +116,15 @@ bool dagContainsNonDeterministicFunction(const ActionsDAG & dag) bool isSensitiveToEvaluationCount(const ActionsDAG & dag) { - auto is_insensitive = [](const IFunctionBase & function) - { - return function.isDeterministicInScopeOfQuery() && !function.isStateful() && !function.hasObservableSideEffects(); - }; - - /// A lambda without captures is constant-folded into a `COLUMN` node holding a `ColumnFunction`, which - /// hides the functions of its body from a plain scan over the function nodes; `allNodeFunctions` - /// descends into it. + /// `sleep` and `sleepEachRow` are the functions with observable side effects. A lambda hides the + /// functions of its body, so it counts as sensitive. for (const auto & node : dag.getNodes()) { - if (!allNodeFunctions(node, is_insensitive)) + if (node.result_type && WhichDataType(node.result_type).isFunction()) + return true; + if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base + && (!node.function_base->isDeterministicInScopeOfQuery() || node.function_base->isStateful() + || node.function_base->getName() == "sleep" || node.function_base->getName() == "sleepEachRow")) return true; } From fd4a33f7797ea30d2e1c1d703af382a41f22efce Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 6 Oct 2026 19:53:50 +0000 Subject: [PATCH 205/296] Adapt the backported tests to 26.8, which lacks #110665 The tests of #119850 also cover behaviour that came from https://github.com/ClickHouse/ClickHouse/pull/110665, which was not backported to 26.8 (its cherry-pick was closed). In 26.8 the `custom_local_disks_base_directory` check of a disk defined in SQL runs after the disk is registered and its directory created, does not cover object storage disks over the local filesystem (they are reported as remote), and exempts a disk named `backup`. Keep the part that this backport fixes: `ALTER TABLE ... MODIFY SETTING disk = disk(...)` with a local path outside the base directory is now rejected, and the settings constraints still compare the resolved disk. Drop the checks of registration and directory creation, the object storage cases, and the integration test for a `locations` child with a local backend, which all depend on #110665. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=124151&sha=1348e3964bcfa0dfa2569720e5461b11401f7b5b&name_0=BackportPR PR: https://github.com/ClickHouse/ClickHouse/pull/124151 Co-Authored-By: Claude Opus 5.5 --- .../configs/dynamic_disk_include.xml | 2 -- .../configs/dynamic_disk_include_source.xml | 10 ------ .../test_s3_credentials_hardening/test.py | 26 --------------- ...217_custom_disk_local_path_alter.reference | 16 ---------- .../05217_custom_disk_local_path_alter.sh | 32 ++++++------------- 5 files changed, 9 insertions(+), 77 deletions(-) diff --git a/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include.xml b/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include.xml index 163f9a71cab63..34ffe8e09b853 100644 --- a/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include.xml +++ b/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include.xml @@ -2,6 +2,4 @@ /etc/clickhouse-server/config.d/dynamic_disk_include_source.xml - - /var/lib/clickhouse/disks/ diff --git a/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include_source.xml b/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include_source.xml index f8f975b6777ab..f5a02ed7b5c69 100644 --- a/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include_source.xml +++ b/tests/integration/test_s3_credentials_hardening/configs/dynamic_disk_include_source.xml @@ -33,14 +33,4 @@ - - -
- local - /etc/clickhouse-server/ - 1 - 1 -
- - diff --git a/tests/integration/test_s3_credentials_hardening/test.py b/tests/integration/test_s3_credentials_hardening/test.py index 6e4095215412a..eca2cb680ea5c 100644 --- a/tests/integration/test_s3_credentials_hardening/test.py +++ b/tests/integration/test_s3_credentials_hardening/test.py @@ -591,32 +591,6 @@ def test_dynamic_disk_include_locations_child_is_restricted(): assert "ACCESS_DENIED" in error or "must provide its credentials explicitly" in error, error -def test_dynamic_disk_include_locations_child_local_path_is_fenced(): - # The same shape, for the local filesystem rather than S3: an `include` can inject a - # `locations.` child whose backend is `local` and whose `path` is outside - # `custom_local_disks_base_directory`. A disk root that names no local path of its own must not let - # that child through -- `RegisterDiskObjectStorage` builds one object storage per child, each with - # its own `path`. - # A `type = object_storage` disk with an `include` and no literal `object_storage_type` may resolve to S3, - # so the credentials restriction rejects it before the disk is created; lift it to reach the path check. - node_with_includes.query("DROP TABLE IF EXISTS t_loc_local SYNC") - error = node_with_includes.query_and_get_error( - "CREATE TABLE t_loc_local (x UInt8) ENGINE = MergeTree ORDER BY tuple() " - "SETTINGS disk = disk(type = object_storage, include = 'evil_locations_local_path')", - settings={ - "dynamic_disk_allow_include": 1, - "s3_allow_server_credentials_in_user_queries": 1, - }, - ) - assert "must be inside" in error, error - assert ( - node_with_includes.query( - "SELECT count() FROM system.disks WHERE path = '/etc/clickhouse-server/'" - ).strip() - == "0" - ) - - def test_persistent_table_does_not_reuse_credentialed_client_across_sessions(): # A persistent S3 table whose credentials are server-managed (use_environment_credentials) is non-static, so # its client is rebuilt per query. An opt-in session reading it builds a credentialed client in the shared diff --git a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference index 005a61cd1bdaf..1f63a15bb0843 100644 --- a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference +++ b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference @@ -1,23 +1,7 @@ local 1 -0 -no directory -local object storage -1 -0 -no directory -metadata path -1 -0 -no directory no settings clause 1 -0 -no directory -named backup -1 -0 -no directory constraints 1 unrelated change allowed diff --git a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh index 372dc17955715..a0cae4db32a75 100755 --- a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh +++ b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh @@ -10,18 +10,17 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Every location on the local filesystem that a disk defined in SQL names has to be inside # `custom_local_disks_base_directory`. `CREATE TABLE` checks that; `ALTER TABLE ... MODIFY SETTING disk` # has to check it the same way. +# In this release the check runs after the disk is created, and does not cover object storage disks over +# the local filesystem, so only the rejection of the statement is checked here. OUTSIDE="${USER_FILES_PATH}/05217_outside_${CLICKHOUSE_DATABASE}" INSIDE="${CLICKHOUSE_DISKS_FILES}/05217_${CLICKHOUSE_DATABASE}" -# Prints 1 if the statement was rejected by the fence, then whether the disk is registered and whether -# the rejected location was created. +# Prints 1 if the statement was rejected by the fence. function check_rejected() { - local disk_name=$1 location=$2 query=$3 + local query=$1 $CLICKHOUSE_CLIENT -q "$query" 2>&1 | grep -m1 -c -F "must be inside" - $CLICKHOUSE_CLIENT -q "SELECT count() FROM system.disks WHERE name = '${disk_name}'" - test -e "$location" && echo "the rejected disk created ${location}" || echo "no directory" } $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_05217" @@ -32,34 +31,18 @@ $CLICKHOUSE_CLIENT -q " # `ALTER TABLE ... MODIFY SETTING disk = disk(...)` used to resolve the definition as if it came from # stored metadata, which registered the disk unchecked, before the check of the statement ran. echo "local" -check_rejected "05217_local_${CLICKHOUSE_DATABASE}" "${OUTSIDE}_local/" " +check_rejected " ALTER TABLE t_05217 MODIFY SETTING disk = disk(name = '05217_local_${CLICKHOUSE_DATABASE}', type = local, path = '${OUTSIDE}_local/')" -echo "local object storage" -check_rejected "05217_object_storage_${CLICKHOUSE_DATABASE}" "${OUTSIDE}_object_storage/" " - ALTER TABLE t_05217 MODIFY SETTING disk = disk(name = '05217_object_storage_${CLICKHOUSE_DATABASE}', - type = object_storage, object_storage_type = local, path = '${OUTSIDE}_object_storage/')" - -echo "metadata path" -check_rejected "05217_metadata_${CLICKHOUSE_DATABASE}" "${OUTSIDE}_metadata/" " - ALTER TABLE t_05217 MODIFY SETTING disk = disk(name = '05217_metadata_${CLICKHOUSE_DATABASE}', - type = object_storage, object_storage_type = local, path = '${INSIDE}_metadata/', metadata_path = '${OUTSIDE}_metadata/')" - $CLICKHOUSE_CLIENT -q "DROP TABLE t_05217" # A table created without a SETTINGS clause goes through the same check. echo "no settings clause" $CLICKHOUSE_CLIENT -q "CREATE TABLE t_05217_plain (x UInt64) ENGINE = MergeTree ORDER BY x" -check_rejected "05217_plain_${CLICKHOUSE_DATABASE}" "${OUTSIDE}_plain/" " +check_rejected " ALTER TABLE t_05217_plain MODIFY SETTING disk = disk(name = '05217_plain_${CLICKHOUSE_DATABASE}', type = local, path = '${OUTSIDE}_plain/')" $CLICKHOUSE_CLIENT -q "DROP TABLE t_05217_plain" -# The disk name is chosen by the query, so it cannot be what exempts a disk from the fence. -echo "named backup" -check_rejected "backup" "${OUTSIDE}_backup/" " - CREATE TABLE t_05217_backup (x UInt64) ENGINE = MergeTree ORDER BY x - SETTINGS disk = disk(name = 'backup', type = local, path = '${OUTSIDE}_backup/')" - # The settings constraints compare the `disk` the table has after the `ALTER`, not one left unresolved: # a constant `merge_tree_disk` rejects a change of the disk and nothing else. echo "constraints" @@ -83,3 +66,6 @@ $CLICKHOUSE_CLIENT -q " DROP TABLE t_05217_disk; DROP SETTINGS PROFILE ${profile}; DROP USER ${user};" + +# In this release a rejected disk still creates its directory, so remove what the statements above left. +rm -rf "${OUTSIDE}"_* From 9f0f4c4cbac70c0ccfdf5824d5e7355d73c69418 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 20:10:16 +0000 Subject: [PATCH 206/296] Backport #123620 to 26.8: Do not copy patch partitions of lightweight updates to another table --- src/Storages/MergeTree/MergeTreeData.cpp | 16 ++++++ src/Storages/StorageReplicatedMergeTree.cpp | 2 + ...patch_partition_to_another_table.reference | 15 ++++++ ...u_copy_patch_partition_to_another_table.sh | 51 +++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 tests/queries/0_stateless/05317_lwu_copy_patch_partition_to_another_table.reference create mode 100755 tests/queries/0_stateless/05317_lwu_copy_patch_partition_to_another_table.sh diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index eb0a0814121ab..00574bea81b27 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -8453,6 +8453,20 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String } } +/// Patch parts carry data versions allocated from the block numbers of their own table, so they cannot be copied to another table. +static void assertNotPatchPartition(const MergeTreeData & data, const ASTPtr & partition, ContextPtr query_context, std::string_view command) +{ + if (partition->as().all) + return; + + const auto partition_id = data.getPartitionIDFromQuery(partition, query_context); + if (isPatchPartitionId(partition_id)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot execute {} for partition {} of patch parts, they cannot be copied to another table. " + "Apply them with `ALTER TABLE ... APPLY PATCHES IN PARTITION ID '{}'` and use that partition instead", + command, partition_id, getOriginalPartitionIdOfPatch(partition_id)); +} + void MergeTreeData::movePartitionToTable(const PartitionCommand & command, ContextPtr query_context) { String dest_database = query_context->resolveDatabase(command.to_database); @@ -8468,6 +8482,7 @@ void MergeTreeData::movePartitionToTable(const PartitionCommand & command, Conte "Cannot move partition from table {} to table {} with storage {}", getStorageID().getNameForLogs(), dest_storage->getStorageID().getNameForLogs(), dest_storage->getName()); + assertNotPatchPartition(*this, command.partition, query_context, "MOVE PARTITION TO TABLE"); dest_storage_merge_tree->waitForOutdatedPartsToBeLoaded(); movePartitionToTable(dest_storage, command.partition, query_context); } @@ -8593,6 +8608,7 @@ Pipe MergeTreeData::alterPartition( case PartitionCommand::REPLACE_PARTITION: { + assertNotPatchPartition(*this, command.partition, query_context, command.replace ? "REPLACE PARTITION" : "ATTACH PARTITION FROM"); if (command.replace) checkPartitionCanBeDropped(command.partition, query_context); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 1af170cec4750..33a9ce1173d22 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -9184,7 +9184,9 @@ void StorageReplicatedMergeTree::replacePartitionFrom( if (replace) throw DB::Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support DROP/DETACH/ATTACH PARTITION ALL currently"); + /// Patch parts cannot be copied to another table. Partitions with unapplied patches are rejected by `replacePartitionFromImpl`. partitions = src_data.getAllPartitionIds(); + std::erase_if(partitions, isPatchPartitionId); } else { diff --git a/tests/queries/0_stateless/05317_lwu_copy_patch_partition_to_another_table.reference b/tests/queries/0_stateless/05317_lwu_copy_patch_partition_to_another_table.reference new file mode 100644 index 0000000000000..6e426bdacfc70 --- /dev/null +++ b/tests/queries/0_stateless/05317_lwu_copy_patch_partition_to_another_table.reference @@ -0,0 +1,15 @@ +patch parts in t_dst 0 +0 0 +1 10 +2 2 +3 3 +4 10 +5 0 +6 0 +7 0 +8 0 +9 0 +BAD_ARGUMENTS +BAD_ARGUMENTS +BAD_ARGUMENTS +v in t_src 7 diff --git a/tests/queries/0_stateless/05317_lwu_copy_patch_partition_to_another_table.sh b/tests/queries/0_stateless/05317_lwu_copy_patch_partition_to_another_table.sh new file mode 100755 index 0000000000000..0ff2aa9c4650a --- /dev/null +++ b/tests/queries/0_stateless/05317_lwu_copy_patch_partition_to_another_table.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Tags: zookeeper + +# Patch parts carry data versions allocated from the block numbers of their own table. A copy in another table +# made updates there fail with "Found patch part ... that intersects mutation with version ...". + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `remove_unused_patch_parts = 0` keeps the applied patches in `t_src` and any copies of them in `t_dst`. +$CLICKHOUSE_CLIENT --query " + CREATE TABLE t_src (id UInt64, v UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_src', '1') ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, remove_unused_patch_parts = 0; + + CREATE TABLE t_dst AS t_src + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_dst', '1') ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, remove_unused_patch_parts = 0; + + INSERT INTO t_src SELECT number, 0 FROM numbers(10); + UPDATE t_src SET v = 1 WHERE id = 1; + UPDATE t_src SET v = 2 WHERE id = 2; + UPDATE t_src SET v = 3 WHERE id = 3; +" + +patch_partition_id=$($CLICKHOUSE_CLIENT --query " + SELECT DISTINCT partition_id FROM system.parts + WHERE database = currentDatabase() AND table = 't_src' AND active AND startsWith(partition_id, 'patch-')") + +# One patch part with data versions [1, 3], applied to the base part. `ATTACH PARTITION ALL` is what `CLONE AS` runs. +$CLICKHOUSE_CLIENT --query " + OPTIMIZE TABLE t_src PARTITION ID '$patch_partition_id' FINAL; + ALTER TABLE t_src APPLY PATCHES IN PARTITION ID 'all' SETTINGS mutations_sync = 2; + ALTER TABLE t_dst ATTACH PARTITION ALL FROM t_src; + + SELECT 'patch parts in t_dst', count() FROM system.parts + WHERE database = currentDatabase() AND table = 't_dst' AND active AND startsWith(partition_id, 'patch-'); + + UPDATE t_dst SET v = 10 WHERE id IN (1, 4); + SELECT id, v FROM t_dst ORDER BY id; +" + +# An unapplied patch, which a move of its partition would take away from `t_src`. +$CLICKHOUSE_CLIENT --query "UPDATE t_src SET v = 7 WHERE id = 7" + +$CLICKHOUSE_CLIENT --query "ALTER TABLE t_dst ATTACH PARTITION ID '$patch_partition_id' FROM t_src" 2>&1 | grep -o -m1 'BAD_ARGUMENTS' +$CLICKHOUSE_CLIENT --query "ALTER TABLE t_dst REPLACE PARTITION ID '$patch_partition_id' FROM t_src" 2>&1 | grep -o -m1 'BAD_ARGUMENTS' +$CLICKHOUSE_CLIENT --query "ALTER TABLE t_src MOVE PARTITION ID '$patch_partition_id' TO TABLE t_dst" 2>&1 | grep -o -m1 'BAD_ARGUMENTS' + +$CLICKHOUSE_CLIENT --query "SELECT 'v in t_src', v FROM t_src WHERE id = 7" From 612ce588cea73c53892188c12578b66c714175a2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 6 Oct 2026 21:09:55 +0000 Subject: [PATCH 207/296] Move `iceberg_tolerate_conflicting_manifest_schemas` to the `26.8` settings history block The backport of #119743 kept the master label `26.10`, so any `compatibility` below 26.10 reverted the setting to `false` on a 26.8 server. Co-Authored-By: Claude Fable 5.1 --- src/Core/SettingsChangesHistory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index cd304b3354214..2ea0b98998303 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -44,10 +44,10 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() addSettingsChanges(settings_changes_history, "26.10", { {"azure_validate_etag_on_read", false, true, "New setting to detect concurrent in-place overwrites of Azure blobs during a read by pinning every `GET` to the listed `ETag` with `If-Match` and validating the `ETag` of the response, like `s3_validate_etag_on_read` does for S3. `compatibility` with versions before 26.10 restores the previous behavior (no validation)."}, - {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.10 restores the previous strict behavior."}, }); addSettingsChanges(settings_changes_history, "26.8", { + {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.8 restores the previous strict behavior."}, {"analyzer_compatibility_allow_cte_redefinition", false, false, "New compatibility setting. When enabled, the analyzer accepts a CTE name defined more than once in a single `WITH` clause and lets a later definition shadow the earlier ones, as the query analysis before v24.3 did."}, {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, {"allow_executable_tables", true, true, "New setting to disable reading through the `executable` table function and from `Executable` and `ExecutablePool` tables."}, From 09aaa63ddf7a3e82884a4db77185970a52e0035c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 6 Oct 2026 21:23:20 +0000 Subject: [PATCH 208/296] Add a test for `iceberg_tolerate_conflicting_manifest_schemas` under `compatibility` Co-Authored-By: Claude Fable 5.1 --- ...e_conflicting_manifest_schemas_compatibility.reference | 2 ++ ...olerate_conflicting_manifest_schemas_compatibility.sql | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/05323_iceberg_tolerate_conflicting_manifest_schemas_compatibility.reference create mode 100644 tests/queries/0_stateless/05323_iceberg_tolerate_conflicting_manifest_schemas_compatibility.sql diff --git a/tests/queries/0_stateless/05323_iceberg_tolerate_conflicting_manifest_schemas_compatibility.reference b/tests/queries/0_stateless/05323_iceberg_tolerate_conflicting_manifest_schemas_compatibility.reference new file mode 100644 index 0000000000000..b261da18d51a3 --- /dev/null +++ b/tests/queries/0_stateless/05323_iceberg_tolerate_conflicting_manifest_schemas_compatibility.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/tests/queries/0_stateless/05323_iceberg_tolerate_conflicting_manifest_schemas_compatibility.sql b/tests/queries/0_stateless/05323_iceberg_tolerate_conflicting_manifest_schemas_compatibility.sql new file mode 100644 index 0000000000000..3edc7a6a8b2b3 --- /dev/null +++ b/tests/queries/0_stateless/05323_iceberg_tolerate_conflicting_manifest_schemas_compatibility.sql @@ -0,0 +1,8 @@ +-- The backport of the setting must be recorded in the `26.8` block of the settings history, not in +-- the master block. Otherwise a `compatibility` of this release line turns the fix off. + +SET compatibility = '26.8'; +SELECT value FROM system.settings WHERE name = 'iceberg_tolerate_conflicting_manifest_schemas'; + +SET compatibility = '26.7'; +SELECT value FROM system.settings WHERE name = 'iceberg_tolerate_conflicting_manifest_schemas'; From 2ddec3de943559a15ba5958fa0c004a433047db2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 00:03:49 +0000 Subject: [PATCH 209/296] Backport #124262 to 26.8: Fix analyzer resolution of `database.table` when a table alias equals the database name --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 9 ++++--- ...5_analyzer_aliases_compound_expression.sql | 4 +-- ...table_alias_equals_database_name.reference | 7 ++++++ ...lyzer_table_alias_equals_database_name.sql | 25 +++++++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05331_analyzer_table_alias_equals_database_name.reference create mode 100644 tests/queries/0_stateless/05331_analyzer_table_alias_equals_database_name.sql diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 71b19fafddb43..443e177066048 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -1194,6 +1194,10 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierFromAliases(const Ide { const auto & identifier_bind_part = identifier_lookup.identifier.front(); + /// Table aliases are single-part names, so a compound table identifier can only be `database.table`. + if (identifier_lookup.isTableExpressionLookup() && identifier_lookup.identifier.isCompound()) + return {}; + auto * it = scope.aliases.find(identifier_lookup, ScopeAliases::FindOption::FIRST_NAME); if (it == nullptr) return {}; @@ -1294,13 +1298,12 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierFromAliases(const Ide } return {}; } - if (identifier_lookup.isFunctionLookup() || identifier_lookup.isTableExpressionLookup()) + if (identifier_lookup.isFunctionLookup()) { throw Exception( ErrorCodes::UNKNOWN_IDENTIFIER, - "Compound identifier '{}' cannot be resolved as {}. In scope {}", + "Compound identifier '{}' cannot be resolved as function. In scope {}", identifier_lookup.identifier.getFullName(), - identifier_lookup.isFunctionLookup() ? "function" : "table expression", scope.scope_node->formatASTForErrorMessage()); } } diff --git a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql index 6c42607cc565a..79d53d6a2e77a 100644 --- a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql +++ b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql @@ -16,6 +16,6 @@ SELECT '--'; SELECT * FROM t1 AS t2, (SELECT 1) AS t1; -SELECT * FROM (SELECT 1) AS t1, t1.nested AS t2; -- { serverError UNKNOWN_IDENTIFIER } +SELECT * FROM (SELECT 1) AS t1, t1.nested AS t2; -- { serverError UNKNOWN_DATABASE } -SELECT * FROM t1.nested AS t2, (SELECT 1) AS t1; -- { serverError UNKNOWN_IDENTIFIER } +SELECT * FROM t1.nested AS t2, (SELECT 1) AS t1; -- { serverError UNKNOWN_DATABASE } diff --git a/tests/queries/0_stateless/05331_analyzer_table_alias_equals_database_name.reference b/tests/queries/0_stateless/05331_analyzer_table_alias_equals_database_name.reference new file mode 100644 index 0000000000000..6f2df6bdd78ee --- /dev/null +++ b/tests/queries/0_stateless/05331_analyzer_table_alias_equals_database_name.reference @@ -0,0 +1,7 @@ +1 +1 +0 1 +1 +1 +1 +p1 1.5 diff --git a/tests/queries/0_stateless/05331_analyzer_table_alias_equals_database_name.sql b/tests/queries/0_stateless/05331_analyzer_table_alias_equals_database_name.sql new file mode 100644 index 0000000000000..49a703e276119 --- /dev/null +++ b/tests/queries/0_stateless/05331_analyzer_table_alias_equals_database_name.sql @@ -0,0 +1,25 @@ +-- Table alias equal to the database name of another table expression in the same FROM. +-- https://github.com/ClickHouse/ClickHouse/issues/122239 + +SELECT count() FROM system.one AS system, system.one AS o; +SELECT count() FROM system.one AS o, system.one AS system; +SELECT o.dummy, count() FROM system.one AS system INNER JOIN system.one AS o ON system.dummy = o.dummy GROUP BY o.dummy; +SELECT count() FROM (SELECT 1 AS x) AS system, system.one; +SELECT count() FROM numbers(1) AS system, system.one; +SELECT count() FROM system.one AS system WHERE dummy IN (SELECT dummy FROM system.one); + +DROP TABLE IF EXISTS intraday_sensitivities; +DROP TABLE IF EXISTS intraday_positions; +CREATE TABLE intraday_sensitivities (account_id UInt64, run_at DateTime, calculation_value Float64) ENGINE = Memory; +CREATE TABLE intraday_positions (account_id UInt64, run_at DateTime, portfolio String) ENGINE = Memory; +INSERT INTO intraday_sensitivities VALUES (1, '2026-01-01 00:00:00', 1.5); +INSERT INTO intraday_positions VALUES (1, '2026-01-01 00:00:00', 'p1'); + +SELECT positions.portfolio, sum({CLICKHOUSE_DATABASE:Identifier}.calculation_value) +FROM {CLICKHOUSE_DATABASE:Identifier}.intraday_sensitivities AS {CLICKHOUSE_DATABASE:Identifier} +INNER JOIN {CLICKHOUSE_DATABASE:Identifier}.intraday_positions AS positions + ON positions.account_id = {CLICKHOUSE_DATABASE:Identifier}.account_id AND positions.run_at = {CLICKHOUSE_DATABASE:Identifier}.run_at +GROUP BY positions.portfolio; + +DROP TABLE intraday_sensitivities; +DROP TABLE intraday_positions; From c4a04c3f16d46eb86152bc38e8f2358c9c5193c9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 10:45:15 +0000 Subject: [PATCH 210/296] Backport #121687 to 26.8: Iceberg: ignore unparsable sort-order transform instead of aborting the server --- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 17 ++++- ...erg_sort_order_unknown_transform.reference | 10 +++ ...37_iceberg_sort_order_unknown_transform.sh | 65 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05237_iceberg_sort_order_unknown_transform.reference create mode 100755 tests/queries/0_stateless/05237_iceberg_sort_order_unknown_transform.sh diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 57f604b78c419..b6a29bd096fe1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -1520,7 +1520,22 @@ KeyDescription getSortingKeyDescriptionFromMetadata(Poco::JSON::Object::Ptr meta auto column_name = source_id_to_column_name[source_id]; int direction = field->getValue(f_direction) == "asc" ? 1 : -1; auto iceberg_transform_name = field->getValue(f_transform); - auto clickhouse_transform_name = parseTransformAndArgument(iceberg_transform_name); + std::optional clickhouse_transform_name; + try + { + clickhouse_transform_name = parseTransformAndArgument(iceberg_transform_name); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::BAD_ARGUMENTS) + throw; + } + if (!clickhouse_transform_name.has_value()) + { + /// An unknown or malformed transform is not a reason to reject the table: an Iceberg + /// sort order is only an optimization hint, so drop it and read/write the table as unsorted. + return KeyDescription{}; + } /// Quote the column name so identifiers with special characters (e.g. `@timestamp`) /// produce a parseable ORDER BY clause. auto quoted_column_name = backQuoteIfNeed(column_name); diff --git a/tests/queries/0_stateless/05237_iceberg_sort_order_unknown_transform.reference b/tests/queries/0_stateless/05237_iceberg_sort_order_unknown_transform.reference new file mode 100644 index 0000000000000..36473939de0f7 --- /dev/null +++ b/tests/queries/0_stateless/05237_iceberg_sort_order_unknown_transform.reference @@ -0,0 +1,10 @@ +10 a +20 b +1 +10 a +20 b +1 +10 a +20 b +30 c +1 diff --git a/tests/queries/0_stateless/05237_iceberg_sort_order_unknown_transform.sh b/tests/queries/0_stateless/05237_iceberg_sort_order_unknown_transform.sh new file mode 100755 index 0000000000000..401909df0652f --- /dev/null +++ b/tests/queries/0_stateless/05237_iceberg_sort_order_unknown_transform.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" +rm -rf "${TABLE_PATH}" + +# Regression test for #121514: an Iceberg sort-order field naming a transform that +# ClickHouse cannot parse used to dereference a disengaged std::optional in +# getSortingKeyDescriptionFromMetadata and abort the whole server process. +# The sort order must be dropped instead, so that the table stays readable. +# The table is created with use_iceberg_metadata_files_cache = 0: IcebergMetadata +# captures the metadata files cache into its persistent table components when +# the table is constructed, and IcebergStorageSink reuses that pointer on every +# INSERT regardless of the query settings. Without this, the INSERT below could +# be served the old cached metadata.json and never see the rewritten transform. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + CREATE TABLE ${TABLE} (x Int64, y String) + ENGINE = IcebergLocal('${TABLE_PATH}') + ORDER BY icebergTruncate(3, x) + SETTINGS iceberg_format_version = 2; +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --use_iceberg_metadata_files_cache=0 --query "INSERT INTO ${TABLE} VALUES (10, 'a'), (20, 'b')" + +# Rewrite the transform in the latest metadata file to a value that +# parseTransformAndArgument cannot handle ('bucket' with a non-numeric +# argument returns std::nullopt). use_iceberg_metadata_files_cache is disabled +# on the reads so the rewritten metadata.json is really re-parsed instead of +# being served from the cache (the cache key does not change on an in-place +# rewrite of the same file). +LATEST=$(ls "${TABLE_PATH}metadata"/v*.metadata.json | sort -V | tail -1) +sed -i 's/"truncate\[3\]"/"bucket[xyz]"/' "${LATEST}" + +# Reading the table must still work without the sorted-read optimization, +# and must not kill the server. +${CLICKHOUSE_CLIENT} --query "SELECT x, y FROM icebergLocal('${TABLE_PATH}', 'Parquet') ORDER BY x SETTINGS use_iceberg_metadata_files_cache = 0" + +# The server must still be alive. +${CLICKHOUSE_CLIENT} --query "SELECT 1" + +# A malformed spelling that takes the throwing path in parseTransformAndArgument +# ('bucket]' has no '[' and raises BAD_ARGUMENTS) must also be dropped instead +# of rejecting the table. +sed -i 's/"bucket\[xyz\]"/"bucket]"/' "${LATEST}" +${CLICKHOUSE_CLIENT} --query "SELECT x, y FROM icebergLocal('${TABLE_PATH}', 'Parquet') ORDER BY x SETTINGS use_iceberg_metadata_files_cache = 0" +${CLICKHOUSE_CLIENT} --query "SELECT 1" + +# The write path must tolerate the dropped sort order too: IcebergStorageSink +# parses the sort order of the latest metadata through +# getSortingKeyDescriptionFromMetadata on every INSERT, so inserting a row +# into the table over the corrupted metadata used to fail the same way. +# The table is not recreated: CREATE over a path that already has Iceberg +# metadata throws TABLE_ALREADY_EXISTS. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --use_iceberg_metadata_files_cache=0 --query "INSERT INTO ${TABLE} VALUES (30, 'c')" +${CLICKHOUSE_CLIENT} --query "SELECT x, y FROM icebergLocal('${TABLE_PATH}', 'Parquet') ORDER BY x SETTINGS use_iceberg_metadata_files_cache = 0" +${CLICKHOUSE_CLIENT} --query "SELECT 1" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" +rm -rf "${TABLE_PATH}" From 229f00536e155730fe076478021534e4849d727a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 11:37:44 +0000 Subject: [PATCH 211/296] Backport #118978 to 26.8: Query log secret masking for QueryRunner engine and startup scripts --- programs/server/Server.cpp | 4 ++-- src/Interpreters/executeQuery.cpp | 21 +++++++++++++++++++++ src/Interpreters/executeQuery.h | 4 ++++ src/Storages/StorageQueryRunner.cpp | 6 ++++-- tests/integration/test_query_runner/test.py | 16 ++++++++++++++++ 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8877da89122b2..9884fc325c09d 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1136,7 +1136,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const auto condition_read_buffer = ReadBufferFromString(condition); auto condition_write_buffer = WriteBufferFromOwnString(); - LOG_DEBUG(log, "Checking startup query condition `{}`", condition); + LOG_DEBUG(log, "Checking startup query condition `{}`", formatQueryForLogging(condition, startup_context->getSettingsRef())); startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); startup_context->setCurrentQueryId(""); @@ -1170,7 +1170,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const auto read_buffer = ReadBufferFromString(query); auto write_buffer = WriteBufferFromOwnString(); - LOG_DEBUG(log, "Executing query `{}`", query); + LOG_DEBUG(log, "Executing query `{}`", formatQueryForLogging(query, startup_context->getSettingsRef())); startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); startup_context->setCurrentQueryId(""); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 8615932970d47..b27dbf94cbbc7 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -523,6 +523,27 @@ static String httpRequestURLForLogging(const ContextPtr & context) return url.substr(0, url.find_first_of("?#")); } +String formatQueryForLogging(const String & query, const Settings & settings) +{ + const char * pos = query.data(); + const char * end = pos + query.size(); + ParserQuery parser(end, settings[Setting::allow_settings_after_format_in_insert], settings[Setting::implicit_select]); + String parse_error; + const ASTPtr ast = tryParseQuery( + parser, + pos, + end, + parse_error, + /*hilite*/ false, + "", + /*allow_multi_statements*/ false, + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks], + /*skip_insignificant*/ true); + return ast ? ast->formatForLogging(settings[Setting::log_queries_cut_to_length]) : ""; +} + QueryLogElement logQueryStart( const std::chrono::time_point & query_start_time, const ContextMutablePtr & context, diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 8870bbb6c2a52..62214bcf9af81 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -23,6 +23,7 @@ class WriteBuffer; class IOutputFormat; struct QueryStatusInfo; struct QueryPlanAndSets; +struct Settings; struct QueryResultDetails { @@ -126,6 +127,9 @@ void finishExecutedQuery(BlockIO & io, const QueryFinishCallback & query_finish_ /// also used for generated queries that bypass `executeQuery`, such as in the `eval` table function. void validateAnalyzerSettings(ASTPtr ast, bool context_value); +/// Parses the query only to produce text with secrets hidden; returns an empty string if the query does not parse. +String formatQueryForLogging(const String & query, const Settings & settings); + /// Prepares a QueryLogElement and, if enabled, logs it to system.query_log QueryLogElement logQueryStart( const std::chrono::time_point & query_start_time, diff --git a/src/Storages/StorageQueryRunner.cpp b/src/Storages/StorageQueryRunner.cpp index e10d6f5d681db..a6c2191919f59 100644 --- a/src/Storages/StorageQueryRunner.cpp +++ b/src/Storages/StorageQueryRunner.cpp @@ -466,7 +466,7 @@ class QueryRunnerDispatcher : WithContext } else { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "The `QueryRunner` engine does not support this query: {}", job.query); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "The `QueryRunner` engine does not support this query"); } } io.onFinish(); @@ -598,6 +598,8 @@ class QueryRunnerDispatcher : WithContext const auto event_time = std::chrono::system_clock::now(); + const String query_for_logging = formatQueryForLogging(job.query, settings); + query_log->add([&](QueryLogElement & element) { element.type = type; @@ -606,7 +608,7 @@ class QueryRunnerDispatcher : WithContext element.query_start_time = timeInSeconds(query_start_time); element.query_start_time_microseconds = timeInMicroseconds(query_start_time); element.query_duration_ms = duration_ms; - element.query = job.query; + element.query = query_for_logging; element.current_database = job.database; element.log_comment = settings[Setting::log_comment]; element.client_info = job_context->getClientInfo(); diff --git a/tests/integration/test_query_runner/test.py b/tests/integration/test_query_runner/test.py index 9b2a055705ffb..49036e56c360b 100644 --- a/tests/integration/test_query_runner/test.py +++ b/tests/integration/test_query_runner/test.py @@ -134,6 +134,22 @@ def test_cluster_query_log(): ) +def test_cluster_query_log_hides_secrets(): + node_query_runner.query(runner_ddl("query String, settings Map(String, String)", "synchronous", "cluster")) + node_query_runner.query( + "INSERT INTO runner VALUES (" + "'SELECT * FROM url(''http://cluster_user:cluster_secret@node_cluster:8123/?query=SELECT+1'', ''LineAsString'', ''s String'')', " + "{'log_comment': 'qr_cluster_log_secret'})" + ) + node_query_runner.query("SYSTEM FLUSH LOGS query_log") + assert_eq_with_retry( + node_query_runner, + "SELECT type, query LIKE '%[HIDDEN]%', query LIKE '%cluster_secret%' FROM system.query_log " + "WHERE log_comment = 'qr_cluster_log_secret' AND is_internal ORDER BY type", + "QueryStart\t1\t0\nQueryFinish\t1\t0", + ) + + def test_cluster_insert_requires_remote(): node_query_runner.query(runner_ddl("query String", "synchronous", "cluster")) node_query_runner.query("CREATE USER no_remote_user") From 3205b5b8ba66b316924c58069de75467b68097fa Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 14:31:50 +0000 Subject: [PATCH 212/296] Backport #121675 to 26.8: Add a setting to control pushing filters below `LIMIT BY` --- src/Core/Settings.cpp | 14 +++ src/Core/SettingsChangesHistory.cpp | 1 + .../QueryPlan/Optimizations/Optimizations.h | 1 + .../QueryPlanOptimizationSettings.cpp | 3 + .../QueryPlanOptimizationSettings.h | 1 + .../Optimizations/filterPushDown.cpp | 3 + .../QueryPlan/Optimizations/optimizeTree.cpp | 9 +- .../04366_filter_push_down_limit_by.reference | 8 ++ .../04366_filter_push_down_limit_by.sql | 97 +++++++++++++++++++ 9 files changed, 136 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index fa401792157dc..0b689cff5a958 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6716,6 +6716,20 @@ This is an expert-level setting which should only be used for debugging by devel Possible values: +- 0 - Disable +- 1 - Enable +)", 0) \ + DECLARE(Bool, query_plan_filter_push_down_below_limit_by, true, R"( +Toggles pushing filters on `LIMIT BY` key columns below the `LIMIT BY` step. +Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1. +It is read independently of [query_plan_filter_push_down](#query_plan_filter_push_down): the push-down pass also runs, with that setting off, once a `JOIN` runtime filter has been added. + +:::note +This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed. +::: + +Possible values: + - 0 - Disable - 1 - Enable )", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 7ae4b8ec11d4f..0ffb6f7296826 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -165,6 +165,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"iceberg_compaction_commit_batch_size", 100, 100, "New setting"}, {"output_format_arrow_record_batch_size", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target row count. The default `0` preserves one record batch per block."}, {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, + {"query_plan_filter_push_down_below_limit_by", false, true, "New setting to control pushing a filter on the `LIMIT BY` key columns below the `LIMIT BY` step. Set it to false to keep the filter above the `LIMIT BY`."}, }); addSettingsChanges(settings_changes_history, "26.7", { diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index c512274db8fe4..e54b0544287e9 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -85,6 +85,7 @@ struct Optimization /// optimization when the plan is going to be distributed or serialized. bool make_distributed_plan = false; bool serialize_query_plan = false; + bool filter_push_down_below_limit_by = true; }; using Function = size_t (*)(QueryPlan::Node *, QueryPlan::Nodes &, const ExtraSettings &); diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index a1450a3dbc2ea..ee68091b256df 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -60,6 +60,7 @@ namespace Setting extern const SettingsBool query_plan_enable_optimizations; extern const SettingsBool query_plan_execute_functions_after_sorting; extern const SettingsBool query_plan_filter_push_down; + extern const SettingsBool query_plan_filter_push_down_below_limit_by; extern const SettingsBool query_plan_join_shard_by_pk_ranges; extern const SettingsBool query_plan_lift_up_array_join; extern const SettingsBool query_plan_lift_up_union; @@ -178,6 +179,8 @@ QueryPlanOptimizationSettings::QueryPlanOptimizationSettings( merge_filters = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_merge_filters]; push_limit_by_into_sort = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_push_limit_by_into_sort]; filter_push_down = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_filter_push_down]; + /// The filter push-down pass also runs with `filter_push_down` off, once a JOIN runtime filter has been added. + filter_push_down_below_limit_by = from[Setting::query_plan_filter_push_down_below_limit_by]; push_down_volume_reducing_functions = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_push_down_volume_reducing_functions]; convert_outer_join_to_inner_join = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_convert_outer_join_to_inner_join]; diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index e22022e41e4ab..7923e0cced7db 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -55,6 +55,7 @@ struct QueryPlanOptimizationSettings bool merge_expressions; bool merge_filters; bool filter_push_down; + bool filter_push_down_below_limit_by; bool push_down_volume_reducing_functions; bool convert_outer_join_to_inner_join; bool short_circuit_constant_false_join; diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 79c3786a7b0e1..57a9433f4a426 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -1170,6 +1170,9 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (const auto * limit_by = typeid_cast(child.get())) { + if (!settings.filter_push_down_below_limit_by) + return 0; + /// A predicate on the LIMIT BY key columns removes whole groups, so the surviving /// per-group rows (and therefore the result) are identical whether it runs above or /// below the LIMIT BY. But it is only safe to push when every non-empty input group diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index e48e683728e7d..9556da9e95500 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -102,6 +102,7 @@ void optimizeTreeFirstPass(const QueryPlanOptimizationSettings & optimization_se optimization_settings.push_down_volume_reducing_functions, optimization_settings.make_distributed_plan, optimization_settings.serialize_query_plan, + optimization_settings.filter_push_down_below_limit_by, }; while (!stack.empty()) @@ -232,6 +233,7 @@ void optimizeTreeSecondPass( optimization_settings.push_down_volume_reducing_functions, optimization_settings.make_distributed_plan, optimization_settings.serialize_query_plan, + optimization_settings.filter_push_down_below_limit_by, }; Stack stack; @@ -307,6 +309,11 @@ void optimizeTreeSecondPass( /// to move newly added runtime filter as deep in the tree as possible if (join_runtime_filters_were_added) { + /// The re-run passes default `ExtraSettings`, except for the `LIMIT BY` gate: this pass also pushes + /// filters below `LIMIT BY`, so it must follow `query_plan_filter_push_down_below_limit_by` too. + Optimization::ExtraSettings push_down_settings{}; + push_down_settings.filter_push_down_below_limit_by = optimization_settings.filter_push_down_below_limit_by; + traverseQueryPlan(stack, root, [&](auto & frame_node) { @@ -316,7 +323,7 @@ void optimizeTreeSecondPass( size_t changed_nodes = 0; changed_nodes += tryMergeExpressions(&frame_node, nodes, {}); changed_nodes += tryMergeFilters(&frame_node, nodes, {}); - changed_nodes += tryPushDownFilter(&frame_node, nodes, {}); + changed_nodes += tryPushDownFilter(&frame_node, nodes, push_down_settings); if (!changed_nodes) break; diff --git a/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference b/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference index 24970f73243fa..30402fed27e44 100644 --- a/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference +++ b/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference @@ -1,9 +1,17 @@ 1 0 +1 +0 +1 +0 0 0 1 0 1 5 1 5 +1 5 a +false +true +true diff --git a/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql b/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql index 587399f309b54..c26c4b44f01eb 100644 --- a/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql +++ b/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql @@ -34,6 +34,89 @@ FROM ( ) WHERE key = '5' ); +-- Same query with query_plan_filter_push_down_below_limit_by = 0: the conjunct stays above the +-- LIMIT BY, so the primary key gets no condition -> the plan shape used before 26.7. +SELECT countIf(match(explain, 'Condition: \(key in ')) > 0 AS pushed +FROM ( + EXPLAIN indexes = 1 + SELECT * FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0 +); + +-- `filter_above_limitby` = 1 iff a Filter step exists AND its plan line precedes LimitBy's. +-- Absence of a primary key condition alone cannot tell "filter above LimitBy" from "filter +-- pushed below it but index analysis produced nothing", so the OFF path needs this placement +-- oracle too. Same shape as 04512_filter_push_down_limit_by_nondeterministic. +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT * FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0, enable_parallel_replicas = 0 + ) + ) +); + +-- The push-down pass also runs with `query_plan_filter_push_down = 0` once a JOIN runtime +-- filter has been added, so the setting has to be honoured on that path as well: at its default +-- the key conjunct still reaches below LimitBy (no Filter step above it -> 0), at 0 the conjunct +-- stays above LimitBy (-> 1). The two arms differ only in the new setting. +-- `query_plan_optimize_join_order_randomize = 0` (its default) is pinned in both arms because a +-- non-zero seed feeds the join order optimizer random cardinalities, which can reverse the two +-- join inputs; the reversed plan prints the right-hand branch first and `fl < ll` then compares +-- lines from different branches. +DROP TABLE IF EXISTS t_04366_rhs; +CREATE TABLE t_04366_rhs (key String) ENGINE = MergeTree ORDER BY key AS SELECT '5'; + +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT l.key FROM ( + SELECT key, ts FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) AS l INNER JOIN t_04366_rhs AS r ON l.key = r.key + WHERE l.key = '5' + SETTINGS query_plan_filter_push_down = 0, query_plan_filter_push_down_below_limit_by = 1, + enable_join_runtime_filters = 1, enable_parallel_replicas = 0, + query_plan_optimize_join_order_randomize = 0 + ) + ) +); + +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT l.key FROM ( + SELECT key, ts FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) AS l INNER JOIN t_04366_rhs AS r ON l.key = r.key + WHERE l.key = '5' + SETTINGS query_plan_filter_push_down = 0, query_plan_filter_push_down_below_limit_by = 0, + enable_join_runtime_filters = 1, enable_parallel_replicas = 0, + query_plan_optimize_join_order_randomize = 0 + ) + ) +); + +DROP TABLE t_04366_rhs; + -- LIMIT n OFFSET m BY (m > 0): NOT pushed. A group of size <= m is fully dropped, so a -- pushed throwing key predicate could be evaluated on rows the query never reaches. SELECT countIf(match(explain, 'Condition: \(key in ')) > 0 AS pushed @@ -94,6 +177,11 @@ SELECT count(), sum(val) FROM ( SELECT key, ts, val FROM t_04366 WHERE key = '5' ORDER BY key, ts LIMIT 1 BY key ); +-- Same result with the push-down declined, so the OFF path is not covered by EXPLAIN alone. +SELECT count(), sum(val) FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key +) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0; + DROP TABLE t_04366; -- Exception-semantics regression: a singleton group '0' dropped by OFFSET 1 must NOT be @@ -129,3 +217,12 @@ SELECT key FROM ( SELECT key, x FROM t_04366_nonkey ORDER BY key, ord LIMIT 1 BY key ) WHERE intDiv(1, x) > 0; DROP TABLE t_04366_nonkey; + +-- `compatibility` with a version before the one the setting was added in turns it off, so the +-- filter stays above the `LIMIT BY`; from that version on, and unset, it is on. +SET compatibility = '26.7'; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); +SET compatibility = '26.8'; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); +SET compatibility = ''; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); From c2ddf54911ce1bdea66742a30942a18d7f12cfd7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 16:11:45 +0000 Subject: [PATCH 213/296] Backport #124303 to 26.8: Fix wrong results when skip indexes evaluate a predicate with OR --- src/Storages/MergeTree/KeyCondition.cpp | 10 +-- ...p_index_disjunction_rpn_position.reference | 8 ++ ...31_skip_index_disjunction_rpn_position.sql | 76 +++++++++++++++++++ 3 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05331_skip_index_disjunction_rpn_position.reference create mode 100644 tests/queries/0_stateless/05331_skip_index_disjunction_rpn_position.sql diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 5069cdb88714d..957d62000a3a7 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -5747,9 +5747,12 @@ BoolMask KeyCondition::checkInHyperrectangle( return SpaceFillingCurveType::Unknown; }; - size_t element_idx = 0; - for (const auto & element : rpn) + /// The reported position is the element's index in `rpn`: the disjunction bitset is read positionally + /// against the template RPN. A position that is never reported keeps the bitset's all-true default. + for (size_t element_idx = 0; element_idx < rpn.size(); ++element_idx) { + const auto & element = rpn[element_idx]; + if (element.argument_num_of_space_filling_curve.has_value()) { /// If a condition on argument of a space filling curve wasn't collapsed into FUNCTION_ARGS_IN_HYPERRECTANGLE, @@ -6155,10 +6158,7 @@ BoolMask KeyCondition::checkInHyperrectangle( throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function type in KeyCondition::RPNElement"); if (update_partial_disjunction_result_fn) - { update_partial_disjunction_result_fn(element_idx, rpn_stack.back().can_be_true, (element.function == RPNElement::FUNCTION_UNKNOWN)); - ++element_idx; - } } if (rpn_stack.size() != 1) diff --git a/tests/queries/0_stateless/05331_skip_index_disjunction_rpn_position.reference b/tests/queries/0_stateless/05331_skip_index_disjunction_rpn_position.reference new file mode 100644 index 0000000000000..c39ecb52eeb18 --- /dev/null +++ b/tests/queries/0_stateless/05331_skip_index_disjunction_rpn_position.reference @@ -0,0 +1,8 @@ +1 +1 +1 +0 +667 +667 +667 +100 diff --git a/tests/queries/0_stateless/05331_skip_index_disjunction_rpn_position.sql b/tests/queries/0_stateless/05331_skip_index_disjunction_rpn_position.sql new file mode 100644 index 0000000000000..40fcc2eac181e --- /dev/null +++ b/tests/queries/0_stateless/05331_skip_index_disjunction_rpn_position.sql @@ -0,0 +1,76 @@ +DROP TABLE IF EXISTS t_rpn_pos; + +CREATE TABLE t_rpn_pos +( + id UInt32, + a UInt8, + x UInt8, + y UInt8, + INDEX iax (a, x) TYPE minmax GRANULARITY 2, + INDEX iy y TYPE minmax GRANULARITY 2 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; + +INSERT INTO t_rpn_pos VALUES (0, 0, 1, 1), (1, 200, 1, 1), (2, 0, 1, 7), (3, 200, 1, 1); + +-- `intDiv(a, toInt8(-1))` reinterprets `a` through a signed cast, so it is not monotonic over an `a` +-- range that crosses 128, and the minmax index on (a, x) cannot decide that atom for either index +-- granule here. Row 2 matches only through the `y = 7` arm, which the index on y does decide. +SELECT count() FROM t_rpn_pos WHERE intDiv(a, toInt8(-1)) < 1 AND (x = 5 OR y = 7) +SETTINGS use_skip_indexes = 0; + +SELECT count() FROM t_rpn_pos WHERE intDiv(a, toInt8(-1)) < 1 AND (x = 5 OR y = 7) +SETTINGS use_skip_indexes = 1, use_skip_indexes_for_disjunctions = 1; + +-- The counts above are only meaningful if the merged-disjunction path actually ran: with it off no +-- granule is pruned and row-level filtering returns the same 1. `` appears in +-- EXPLAIN only while that path is on, so these two probes are its positive and negative control. +SELECT count() > 0 FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t_rpn_pos WHERE intDiv(a, toInt8(-1)) < 1 AND (x = 5 OR y = 7) + SETTINGS use_skip_indexes = 1, use_skip_indexes_for_disjunctions = 1, + use_skip_indexes_on_data_read = 0, use_query_condition_cache = 0, + parallel_replicas_local_plan = 1, explain_query_plan_default = 'legacy' +) WHERE explain ILIKE '%%'; + +SELECT count() > 0 FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t_rpn_pos WHERE intDiv(a, toInt8(-1)) < 1 AND (x = 5 OR y = 7) + SETTINGS use_skip_indexes = 1, use_skip_indexes_for_disjunctions = 0, + use_skip_indexes_on_data_read = 0, use_query_condition_cache = 0, + parallel_replicas_local_plan = 1, explain_query_plan_default = 'legacy' +) WHERE explain ILIKE '%%'; + +DROP TABLE t_rpn_pos; + +-- An atom the index cannot evaluate (a function of the column), followed by a constant disjunct that +-- the next index reports as false. +DROP TABLE IF EXISTS t_skip_or0; +CREATE TABLE t_skip_or0 +( + id UInt32, + e Enum8('a' = 1, 'b' = 2, 'c' = 3), + INDEX ie e TYPE minmax GRANULARITY 1, + INDEX ii id TYPE minmax GRANULARITY 1, + INDEX ise e TYPE set(10) GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_skip_or0 SELECT number, ['a', 'b', 'c'][number % 3 + 1] FROM numbers(1000); + +SELECT count() FROM t_skip_or0 WHERE toString(e) < 'c' AND (0 OR id != 5) +SETTINGS use_skip_indexes = 1, use_skip_indexes_for_disjunctions = 1; +SELECT count() FROM t_skip_or0 WHERE e < 'c' AND (toString(id) != '5' OR 0) +SETTINGS use_skip_indexes = 1, use_skip_indexes_for_disjunctions = 1; +-- The same through a set index on e. +SELECT count() FROM t_skip_or0 WHERE toString(e) < 'c' AND (0 OR id != 5) +SETTINGS use_skip_indexes = 1, use_skip_indexes_for_disjunctions = 1, ignore_data_skipping_indices = 'ie'; +DROP TABLE t_skip_or0; + +-- Implicit minmax indexes; the comparison of a UInt128 column with a Nullable(Int256) constant. +DROP TABLE IF EXISTS t_rpn_implicit; +CREATE TABLE t_rpn_implicit (v UInt128, w UInt8) ENGINE = MergeTree ORDER BY tuple() +SETTINGS index_granularity = 1, add_minmax_index_for_numeric_columns = 1; +INSERT INTO t_rpn_implicit SELECT number, number % 3 + 1 FROM numbers(100); +SELECT count() FROM t_rpn_implicit WHERE w > 0 AND (toNullable(toInt256(-1)) < v OR 0) +SETTINGS use_skip_indexes = 1, use_skip_indexes_for_disjunctions = 1; +DROP TABLE t_rpn_implicit; From 16a8b1fb5588cb5d88d0370cf4ddb472a1cb802a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 17:51:31 +0000 Subject: [PATCH 214/296] Update autogenerated version to 26.8.20.9 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index c03b781aac3ee..215c265dcc12a 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54532) +SET(VERSION_REVISION 54533) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 20) -SET(VERSION_GITHASH 2ea9c3c9e37a707c67c363095a760d24a2019c74) -SET(VERSION_DESCRIBE v26.8.20.1-lts) -SET(VERSION_STRING 26.8.20.1) +SET(VERSION_PATCH 21) +SET(VERSION_GITHASH 68170c6e7043e16504bb80174ee032a196a1aa63) +SET(VERSION_DESCRIBE v26.8.21.1-lts) +SET(VERSION_STRING 26.8.21.1) # end of autochange From f19e1d47626929c1b1214c62952c8b32b7b61621 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 19:58:04 +0000 Subject: [PATCH 215/296] Backport #119630 to 26.8: Do not expand a WITH element into a subquery that does not see it when updating --- src/Interpreters/ApplyWithSubqueryVisitor.cpp | 49 +++++++++- src/Interpreters/ApplyWithSubqueryVisitor.h | 13 ++- src/Interpreters/InterpreterUpdateQuery.cpp | 7 +- ...update_cte_out_of_scope_database.reference | 9 ++ ...05189_update_cte_out_of_scope_database.sql | 98 +++++++++++++++++++ 5 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference create mode 100644 tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp index c3c922348a8d5..fa32a281190fb 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.cpp +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,16 +7,46 @@ #include #include #include +#include #include #include #include #include +#include #include +#include namespace DB { +namespace Setting +{ + extern const SettingsBool enable_global_with_statement; +} + +namespace +{ + +/// A name is looked up in an enclosing scope only while `enable_global_with_statement` holds in the +/// subquery's own context, so a CTE name that a subquery does not see is a table name there. +/// The clause is clamped rather than rejected, so a subquery cannot widen the reader's constraints, +/// and it is applied to a copy, so the AST keeps the clause as written. +ContextPtr getSubqueryContext(const ASTSelectQuery & select, const ContextPtr & context) +{ + auto settings_ast = select.settings(); + if (!settings_ast) + return context; + + auto changes = settings_ast->as().changes; + auto subquery_context = Context::createCopy(context); + subquery_context->clampToSettingsConstraints(changes, SettingSource::QUERY); + subquery_context->applySettingsChanges(changes); + return subquery_context; +} + +} + void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) { checkStackSize(); @@ -35,18 +66,30 @@ void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) void ApplyWithSubqueryVisitor::visit(ASTSelectQuery & ast, const Data & data) { + /// The CTEs this select declares itself are registered below either way: only the inherited + /// ones are out of scope here. + std::optional scope_data; + if (data.context) + { + scope_data = data; + scope_data->context = getSubqueryContext(ast, data.context); + if (!scope_data->context->getSettingsRef()[Setting::enable_global_with_statement]) + scope_data->subqueries.clear(); + } + const Data & scope = scope_data ? *scope_data : data; + std::optional new_data; if (auto with = ast.with()) { for (auto & child : with->children) { - visit(child, new_data ? *new_data : data); + visit(child, new_data ? *new_data : scope); auto * ast_with_elem = child->as(); auto child_alias = child->tryGetAlias(); if (ast_with_elem || !child_alias.empty()) { if (!new_data) - new_data = data; + new_data = scope; if (ast_with_elem) new_data->subqueries[ast_with_elem->name] = ast_with_elem->subquery; else @@ -58,7 +101,7 @@ void ApplyWithSubqueryVisitor::visit(ASTSelectQuery & ast, const Data & data) for (auto & child : ast.children) { if (child != ast.with()) - visit(child, new_data ? *new_data : data); + visit(child, new_data ? *new_data : scope); } } diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.h b/src/Interpreters/ApplyWithSubqueryVisitor.h index 72fc07c71483d..36ce500dff77e 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.h +++ b/src/Interpreters/ApplyWithSubqueryVisitor.h @@ -3,6 +3,7 @@ #include #include +#include #include @@ -20,9 +21,19 @@ class ApplyWithSubqueryVisitor { std::map subqueries; std::map literals; + /// When set, each subquery's own settings are applied while descending, so that an inherited + /// `subqueries` element is not substituted into a subquery whose settings hide it. Inherited + /// `literals` are substituted either way. + ContextPtr context; }; - static void visit(ASTPtr & ast) { visit(ast, {}); } + static void visit(ASTPtr & ast) { visit(ast, Data{}); } + static void visit(ASTPtr & ast, ContextPtr context) + { + Data data; + data.context = std::move(context); + visit(ast, data); + } static void visit(ASTSelectQuery & select) { visit(select, {}); } static void visit(ASTSelectWithUnionQuery & select) { visit(select, {}); } diff --git a/src/Interpreters/InterpreterUpdateQuery.cpp b/src/Interpreters/InterpreterUpdateQuery.cpp index e03f38901d07c..f6393967ee712 100644 --- a/src/Interpreters/InterpreterUpdateQuery.cpp +++ b/src/Interpreters/InterpreterUpdateQuery.cpp @@ -174,16 +174,17 @@ BlockIO InterpreterUpdateQuery::execute() } /// Expand CTEs before filling the default database, otherwise a CTE alias is qualified as if it - /// were a table. + /// were a table. The context makes CTE expansion respect `enable_global_with_statement`: a CTE + /// name a subquery does not see is a table name there, and has to be qualified. if (update_query.predicate) { ASTPtr predicate = update_query.predicate->ptr(); - ApplyWithSubqueryVisitor::visit(predicate); + ApplyWithSubqueryVisitor::visit(predicate, getContext()); } if (update_query.assignments) { ASTPtr assignments = update_query.assignments->ptr(); - ApplyWithSubqueryVisitor::visit(assignments); + ApplyWithSubqueryVisitor::visit(assignments, getContext()); } /// Add default database to table identifiers that we can encounter in the update expression. diff --git a/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference new file mode 100644 index 0000000000000..88d32f1ade59d --- /dev/null +++ b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference @@ -0,0 +1,9 @@ +2 +2 +2 +2 +2 +7002 +7 +2 +2 diff --git a/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql new file mode 100644 index 0000000000000..c7408aab6b1f3 --- /dev/null +++ b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql @@ -0,0 +1,98 @@ +-- A common table expression is expanded before the database is filled into a mutation command, so a +-- table identifier hidden by an expression alias of the same name is left unqualified. With +-- `enable_global_with_statement` disabled the alias is not visible in the subquery, so the identifier +-- names a table there and has to be resolved in the database of the updated table, not of the session. +-- The source table exists in both databases with a different row, so an expression resolved in the +-- database of the session silently reads the wrong row instead of failing. +-- The reference is one `SELECT` deeper than the alias: only a lookup in an enclosing scope is +-- disabled, so a reference in the select that declares the alias reads it either way. +-- The old analyzer resolves a common table expression in a subquery of a mutation as a table, so the +-- analyzer is requested explicitly, as in `04693_merge_table_function_in_mutation`. + +CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; + +CREATE TABLE src (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO src VALUES (99); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.src (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.src VALUES (2); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.u (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.u VALUES (1, 0), (2, 0), (3, 0), (4, 0), (5, 0), (6, 0), (8, 0), (99, 0); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.t VALUES (1, 0), (2, 0), (3, 0), (99, 0); + +-- Every row starts at 0 and every expected value is non-zero, so a statement that does not update the +-- row at all also fails. The row `id = 99` exists so that a predicate resolved in the database of the +-- session updates the wrong row rather than no row. + +-- An assignment reads the table of the updated database (2), not the table of the session (99). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 1 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 1; + +-- The `ALTER TABLE ... UPDATE` spelling of the same expression answers the same. +ALTER TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t + UPDATE v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 1 SETTINGS mutations_sync = 2, enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.t WHERE id = 1; + +-- The predicate is expanded by a separate call, so it is asserted separately: it marks the row the +-- updated database names (2), not the one the session database names (99). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = 11 + WHERE id IN (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + SETTINGS enable_analyzer = 1; +SELECT id FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE v = 11; + +ALTER TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t + UPDATE v = 11 + WHERE id IN (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + SETTINGS mutations_sync = 2, enable_analyzer = 1; +SELECT id FROM {CLICKHOUSE_DATABASE_1:Identifier}.t WHERE v = 11; + +-- The effective value of the setting is read, so a profile that carries it answers the same as the +-- setting written out. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS compatibility = '20.3')) + WHERE id = 3 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 3; + +-- A clause is read per arm of a union: the arm carrying it names a table there, resolved in the +-- updated database (2), while the arm without it keeps reading the alias (7 * 1000). The two arms +-- are weighted differently so that reading the clause for the wrong arm, for both arms or for +-- neither answers 2007, 2002 and 7007 rather than the expected value. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT sum(m) FROM ( + SELECT max(id) * 1000 AS m FROM src + UNION ALL + SELECT max(id) AS m FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 8 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 8; + +-- With the setting at its default the alias IS visible in the subquery, so the same reference reads +-- the common table expression (7) and must not be qualified as a table. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src)) + WHERE id = 4 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 4; + +-- A reference in the select that declares the alias keeps reading the alias, and the table its body +-- names is resolved in the updated database (2). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH c AS (SELECT max(id) AS m FROM src) SELECT m FROM c) + WHERE id = 5 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 5; + +-- An identifier that no alias hides was always resolved in the updated database (2). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (SELECT max(id) FROM src) + WHERE id = 6 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 6; + +DROP TABLE src; +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; From 40dcc2ab743b852354c07e872c4a07f07693491f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 21:10:35 +0000 Subject: [PATCH 216/296] Backport #123566 to 26.8: Fix parallel replicas over a Distributed table over a Distributed table --- .../ClusterProxy/SelectStreamFactory.cpp | 39 +++- .../ClusterProxy/executeQuery.cpp | 34 +--- src/Planner/Planner.cpp | 5 +- .../__init__.py | 0 .../configs/hub.xml | 34 ++++ .../configs/single.xml | 12 ++ .../configs/spoke.xml | 12 ++ .../test.py | 171 ++++++++++++++++++ 8 files changed, 274 insertions(+), 33 deletions(-) create mode 100644 tests/integration/test_parallel_replicas_dist_over_dist/__init__.py create mode 100644 tests/integration/test_parallel_replicas_dist_over_dist/configs/hub.xml create mode 100644 tests/integration/test_parallel_replicas_dist_over_dist/configs/single.xml create mode 100644 tests/integration/test_parallel_replicas_dist_over_dist/configs/spoke.xml create mode 100644 tests/integration/test_parallel_replicas_dist_over_dist/test.py diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index a2e6bbca35cd3..9e73aa12f598a 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +43,7 @@ namespace Setting extern const SettingsBool serialize_query_plan; extern const SettingsBool skip_unavailable_shards; extern const SettingsUInt64 distributed_group_by_no_merge; + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; } namespace ErrorCodes @@ -158,10 +161,32 @@ void SelectStreamFactory::createForShardImpl( AdditionalShardFilterGenerator shard_filter_generator, const UnavailableShardTrackerPtr & unavailable_shard_tracker) const { - auto emplace_local_stream = [&]() + ContextPtr context_without_parallel_replicas; + + /// `local_storage` is the table the local plan will read, null when it is not resolved. + auto emplace_local_stream = [&](const StoragePtr & local_storage) { + /// A local plan does not go through `ReadFromRemote`, so nothing sets `cluster_for_parallel_replicas` + /// to this hop's cluster and the read would be scoped by another one. Keep parallel replicas only + /// for a nested `Distributed`, which sets up its own cluster. A `View` over a `Distributed` is left + /// out: it reads without parallel replicas, as it did before. + auto local_context = context; + if (context->canUseTaskBasedParallelReplicas() + && !castStorage(local_storage, DeferredTable::Load)) + { + if (!context_without_parallel_replicas) + { + auto mutable_context = Context::createCopy(context); + Settings settings_without_parallel_replicas = mutable_context->getSettingsCopy(); + settings_without_parallel_replicas[Setting::allow_experimental_parallel_reading_from_replicas] = 0; + mutable_context->setSettings(settings_without_parallel_replicas); + context_without_parallel_replicas = std::move(mutable_context); + } + local_context = context_without_parallel_replicas; + } + local_plans.emplace_back(createLocalPlan( - query_ast, *header, context, processed_stage, shard_info.shard_num, shard_count)); + query_ast, *header, local_context, processed_stage, shard_info.shard_num, shard_count)); }; // If lazy is true, a lazy pipe will be created. It will try to use the local replica and, if not possible, will use DelayedSource for reading from remote replica. @@ -259,7 +284,7 @@ void SelectStreamFactory::createForShardImpl( unavailable_shard_tracker->onShardSkipped(); } else - emplace_local_stream(); /// Let it fail the usual way. + emplace_local_stream(main_table_storage); /// Let it fail the usual way. return; } @@ -269,7 +294,7 @@ void SelectStreamFactory::createForShardImpl( if (!replicated_storage) { /// Table is not replicated, use local server. - emplace_local_stream(); + emplace_local_stream(main_table_storage); return; } @@ -277,7 +302,7 @@ void SelectStreamFactory::createForShardImpl( if (!max_allowed_delay) { - emplace_local_stream(); + emplace_local_stream(main_table_storage); return; } @@ -285,7 +310,7 @@ void SelectStreamFactory::createForShardImpl( if (local_delay < max_allowed_delay) { - emplace_local_stream(); + emplace_local_stream(main_table_storage); return; } @@ -312,7 +337,7 @@ void SelectStreamFactory::createForShardImpl( if (!shard_info.hasRemoteConnections()) { /// There are no remote replicas but we are allowed to fall back to stale local replica. - emplace_local_stream(); + emplace_local_stream(main_table_storage); return; } diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 7fa2d6fadecb9..627c98d8a067b 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -429,34 +429,18 @@ static ContextMutablePtr updateSettingsAndClientInfoForCluster(const Cluster & c new_settings[Setting::additional_table_filters].value.push_back(std::move(tuple)); } - /// disable parallel replicas if cluster contains only shards with 1 replica - if (context->canUseTaskBasedParallelReplicas()) + if (context->canUseTaskBasedParallelReplicas() && is_remote_function) { - bool disable_parallel_replicas = false; - if (is_remote_function) - { - if (cluster.getName().empty()) // disable parallel replicas with remote() table functions w/o configured cluster - disable_parallel_replicas = true; - else - new_settings[Setting::cluster_for_parallel_replicas] = cluster.getName(); - } - - if (!disable_parallel_replicas) - { - disable_parallel_replicas = true; - for (const auto & shard : cluster.getShardsInfo()) - { - if (shard.getAllNodeCount() > 1) - { - disable_parallel_replicas = false; - break; - } - } - } - - if (disable_parallel_replicas) + /// `remote()` without a configured cluster has no cluster to scope parallel replicas to. + if (cluster.getName().empty()) new_settings[Setting::allow_experimental_parallel_reading_from_replicas] = 0; + else + new_settings[Setting::cluster_for_parallel_replicas] = cluster.getName(); } + /// Parallel replicas are not disabled here for a cluster whose every shard has one replica: + /// `new_settings` is what the shard receives, and its own table may be a `Distributed` table over a + /// cluster that can use them. Whether this hop uses them is decided per shard below, and a shard + /// that cannot declines on its own, see `canUseParallelReplicasOnInitiator`. if (settings[Setting::max_execution_time_leaf].totalMicroseconds() > 0) { diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 5fba025023647..ff3719d8ce6a1 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -2670,9 +2670,12 @@ void Planner::buildPlanForQueryNode() /// applied later as a plan transformation (QueryPlanOptimizations::applyParallelReplicas). So skip the /// old parallel-replicas planning path here (it would emit ReadFromLocalReplica / /// ReadFromRemoteParallelReplicas, e.g. inside a view/union inner query) and use the normal plan. + /// Same for a plan that is shipped to a shard: those steps are local to the server that planned + /// them and have no serialized form, and the shard decides on parallel replicas on its own. if (planner_context->getMutableQueryContext()->canUseTaskBasedParallelReplicas() && planner_context->getGlobalPlannerContext()->parallel_replicas_node == &query_node - && !settings[Setting::parallel_replicas_plan_based]) + && !settings[Setting::parallel_replicas_plan_based] + && !select_query_options.build_logical_plan) { join_tree_query_plan = buildQueryPlanForParallelReplicas(query_node, planner_context, select_query_info.storage_limits); } diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/__init__.py b/tests/integration/test_parallel_replicas_dist_over_dist/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/configs/hub.xml b/tests/integration/test_parallel_replicas_dist_over_dist/configs/hub.xml new file mode 100644 index 0000000000000..ed214c14ffe84 --- /dev/null +++ b/tests/integration/test_parallel_replicas_dist_over_dist/configs/hub.xml @@ -0,0 +1,34 @@ + + + + + + true + n19000 + n29000 + n39000 + + + + + + false + n19000 + + + false + n49000 + + + false + n69000 + + + + diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/configs/single.xml b/tests/integration/test_parallel_replicas_dist_over_dist/configs/single.xml new file mode 100644 index 0000000000000..598ee98360974 --- /dev/null +++ b/tests/integration/test_parallel_replicas_dist_over_dist/configs/single.xml @@ -0,0 +1,12 @@ + + + + + + true + n69000 + + + + diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/configs/spoke.xml b/tests/integration/test_parallel_replicas_dist_over_dist/configs/spoke.xml new file mode 100644 index 0000000000000..340c7488a6930 --- /dev/null +++ b/tests/integration/test_parallel_replicas_dist_over_dist/configs/spoke.xml @@ -0,0 +1,12 @@ + + + + + + true + n49000 + n59000 + + + + diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/test.py b/tests/integration/test_parallel_replicas_dist_over_dist/test.py new file mode 100644 index 0000000000000..15a744a350a05 --- /dev/null +++ b/tests/integration/test_parallel_replicas_dist_over_dist/test.py @@ -0,0 +1,171 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# Three instances in a hub-and-spoke arrangement. Each carries its own `remote_servers`, where the +# cluster named `default` - the one a Cloud instance always has - is that instance's own replicas: +# three for the hub, two for the spoke and one for the single-node instance, which therefore cannot +# use parallel replicas at all. Only the hub knows `outer_cluster`, the entry point that fans a query +# out to all three. +# One `ClickHouseCluster` is used for all of them: separate `ClickHouseCluster` objects get separate docker +# compose projects and therefore separate networks, so their nodes could not reach each other. +hub_nodes = [ + cluster.add_instance(f"n{i}", main_configs=["configs/hub.xml"], with_zookeeper=True) + for i in (1, 2, 3) +] +spoke_nodes = [ + cluster.add_instance( + f"n{i}", main_configs=["configs/spoke.xml"], with_zookeeper=True + ) + for i in (4, 5) +] +single_nodes = [ + cluster.add_instance( + "n6", main_configs=["configs/single.xml"], with_zookeeper=True + ) +] +INSTANCES = [("hub", hub_nodes), ("spoke", spoke_nodes), ("single", single_nodes)] +nodes = hub_nodes + spoke_nodes + single_nodes + +TABLE = "logs" +INNER_DIST = "logs_dist" +OUTER_DIST = "logs_dist_dist" + +SERVICE = "id-api-auth-service" +WINDOW_START = "2026-09-28 15:40:00" +WINDOW_END = "2026-09-28 15:45:00" + +QUERY = f""" + SELECT count(), avg(length(Body)) + FROM {OUTER_DIST} + WHERE (ServiceName = '{SERVICE}') + AND (Timestamp >= '{WINDOW_START}') AND (Timestamp < '{WINDOW_END}') + """ + +# The hub holds 1000 matching rows with a body of 10 bytes, the spoke 3000 with a body of 20 and the +# single-node instance 1000 with a body of 10, so an instance read twice or not at all moves both the +# count and the average: (1000 * 10 + 3000 * 20 + 1000 * 10) / 5000. +EXPECTED = "5000\t16\n" + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + create_tables() + yield cluster + finally: + cluster.shutdown() + + +def create_tables(): + for instance, instance_nodes in INSTANCES: + for i, node in enumerate(instance_nodes): + node.query(f""" + CREATE TABLE {TABLE} + ( + Timestamp DateTime, + ServiceName LowCardinality(String), + Body String + ) + Engine=ReplicatedMergeTree('/test_pr_dist_over_dist/{instance}/{TABLE}', 'r{i}') + ORDER BY (ServiceName, Timestamp) + -- Small granules so there are mark segments to spread over the replicas of a shard. + SETTINGS index_granularity = 8 + """) + + # Every server reads its own instance through the same cluster name. + node.query(f""" + CREATE TABLE {INNER_DIST} AS {TABLE} + Engine=Distributed(default, currentDatabase(), {TABLE}, rand()) + """) + + hub_nodes[0].query(f""" + CREATE TABLE {OUTER_DIST} AS {INNER_DIST} + Engine=Distributed(outer_cluster, currentDatabase(), {INNER_DIST}, rand()) + """) + + insert_data(hub_nodes[0], matching_rows=1000, body_length=10) + insert_data(spoke_nodes[0], matching_rows=3000, body_length=20) + insert_data(single_nodes[0], matching_rows=1000, body_length=10) + + for node in nodes: + node.query(f"SYSTEM SYNC REPLICA {TABLE}") + + +def insert_data(node, matching_rows, body_length): + node.query(f""" + INSERT INTO {TABLE} + SELECT toDateTime('{WINDOW_START}') + (number % 300), '{SERVICE}', repeat('a', {body_length}) + FROM numbers({matching_rows}) + """) + # Rows the WHERE has to exclude. Their bodies are far longer than any matching row, so letting one + # through shows up in the average and not only in the count. + node.query(f""" + INSERT INTO {TABLE} VALUES + ('{WINDOW_START}', 'some-other-service', '{'x' * 100}'), + (toDateTime('{WINDOW_START}') - 1, '{SERVICE}', '{'x' * 100}'), + ('{WINDOW_END}', '{SERVICE}', '{'x' * 100}') + """) + + +def parallel_replicas_coordinators(query_id): + """Names of the nodes that hosted a parallel-replicas reading coordinator """ + + coordinators = set() + for node in nodes: + # SYSTEM FLUSH LOGS is not cluster-aware, it has to be issued on each node separately. + node.query("SYSTEM FLUSH LOGS query_log") + if int( + node.query( + f""" + SELECT countIf(ProfileEvents['ParallelReplicasNumRequests'] > 0) + FROM system.query_log + WHERE initial_query_id = '{query_id}' AND type = 'QueryFinish' + SETTINGS enable_parallel_replicas = 0 + """ + ) + ): + coordinators.add(node.name) + return coordinators + + +@pytest.mark.parametrize("prefer_localhost_replica", [0, 1]) +def test_parallel_replicas_over_distributed_over_distributed( + start_cluster, prefer_localhost_replica +): + # Without parallel replicas + assert ( + hub_nodes[0].query(QUERY, settings={"enable_parallel_replicas": 0}) == EXPECTED + ) + + query_id = str(uuid.uuid4()) + assert ( + hub_nodes[0].query( + QUERY, + query_id=query_id, + settings={ + "enable_parallel_replicas": 2, + "max_parallel_replicas": 3, + "prefer_localhost_replica": prefer_localhost_replica, + # The automatic decision must not turn parallel replicas off on this small data, and + # one mark segment per granule lets the reading spread over both replicas. + "automatic_parallel_replicas_mode": 0, + "parallel_replicas_mark_segment_size": 1, + }, + ) + == EXPECTED + ) + + # Every instance with more than one replica read with parallel replicas over its own `default` + # cluster, and the single-node one did not use them at all. + coordinators = parallel_replicas_coordinators(query_id) + for instance, instance_nodes in INSTANCES: + used = any(node.name in coordinators for node in instance_nodes) + assert used == (len(instance_nodes) > 1), ( + f"{instance}: parallel replicas used={used}, coordinators: {coordinators}" + ) From 5d0b2861eac31898165035f3d64bd20541f7df1b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Oct 2026 22:56:10 +0000 Subject: [PATCH 217/296] Backport #121332 to 26.8: Resolve lazily loaded table proxies before casting to an engine type --- ci/jobs/check_style.py | 129 ++++++++++++++++++ src/Analyzer/Resolve/resolveFunction.cpp | 3 +- src/Databases/DatabaseOrdinary.cpp | 16 ++- src/Databases/DatabaseReplicated.cpp | 3 +- src/Databases/DatabaseURL.cpp | 1 + src/Functions/FunctionJoinGet.cpp | 3 +- src/Interpreters/ActionsVisitor.cpp | 5 +- src/Interpreters/BlobStorageLog.cpp | 3 +- .../ClusterProxy/SelectStreamFactory.cpp | 3 +- src/Interpreters/DatabaseCatalog.cpp | 3 +- src/Interpreters/ExpressionAnalyzer.cpp | 3 +- src/Interpreters/IInterpreter.cpp | 3 +- .../InsertDependenciesBuilder.cpp | 12 +- src/Interpreters/InterpreterAlterQuery.cpp | 8 +- src/Interpreters/InterpreterCreateQuery.cpp | 17 ++- src/Interpreters/InterpreterDeleteQuery.cpp | 9 +- src/Interpreters/InterpreterDropQuery.cpp | 3 +- .../InterpreterHypotheticalIndexQuery.cpp | 26 ++-- src/Interpreters/InterpreterInsertQuery.cpp | 6 +- src/Interpreters/InterpreterOptimizeQuery.cpp | 9 +- src/Interpreters/InterpreterSelectQuery.cpp | 3 +- src/Interpreters/InterpreterSystemQuery.cpp | 40 +++--- src/Interpreters/JoinedTables.cpp | 6 +- src/Interpreters/MutationsInterpreter.cpp | 14 +- .../ServerAsynchronousMetrics.cpp | 13 +- src/Planner/CollectSets.cpp | 3 +- src/Planner/Planner.cpp | 3 +- src/Planner/PlannerJoins.cpp | 8 +- src/Planner/PlannerJoinsLogical.cpp | 6 +- .../QueryPlan/ReadFromMergeTree.cpp | 3 +- src/Processors/QueryPlan/ReadFromRemote.cpp | 3 +- .../QueryPlan/SetsSerialization.cpp | 3 +- src/Server/ReplicasStatusHandler.cpp | 8 +- src/Server/TCPHandler.cpp | 4 +- src/Storages/AlterCommands.cpp | 7 +- src/Storages/MergeTree/MergeTreeData.cpp | 12 +- .../StorageFromMergeTreeProjection.cpp | 1 + .../MergeTree/WhatIfIndexEstimator.cpp | 3 +- src/Storages/RocksDB/StorageSystemRocksDB.cpp | 3 +- src/Storages/StorageMergeTree.cpp | 3 +- .../StorageMergeTreeAnalyzeIndexes.cpp | 7 +- .../StorageMergeTreeCodecBlockCounts.cpp | 21 ++- .../StorageMergeTreeCodecBlockCounts.h | 4 +- src/Storages/StorageMergeTreeIndex.cpp | 3 +- src/Storages/StorageMergeTreeTextIndex.cpp | 4 +- src/Storages/StorageProxy.h | 60 ++++++++ src/Storages/StorageReplicatedMergeTree.cpp | 14 +- src/Storages/StorageTableFunction.h | 6 + src/Storages/StorageTableProxy.h | 8 ++ src/Storages/StorageURL.cpp | 2 + src/Storages/System/StatusRequestsPool.cpp | 8 +- .../StorageSystemDroppedTablesParts.cpp | 3 +- src/Storages/System/StorageSystemGraphite.cpp | 7 +- .../System/StorageSystemIcebergFiles.cpp | 3 +- .../System/StorageSystemIcebergHistory.cpp | 3 +- .../System/StorageSystemKafkaConsumers.cpp | 5 +- .../System/StorageSystemMutations.cpp | 7 +- ...torageSystemObjectStorageQueueSettings.cpp | 3 +- .../StorageSystemPartMovesBetweenShards.cpp | 6 +- .../System/StorageSystemPartsBase.cpp | 6 +- src/Storages/System/StorageSystemPartsBase.h | 3 +- src/Storages/System/StorageSystemReplicas.cpp | 6 +- .../System/StorageSystemReplicationQueue.cpp | 6 +- src/Storages/System/StorageSystemTables.cpp | 3 +- src/Storages/Utils.cpp | 1 + .../TableFunctionMergeTreeIndex.cpp | 7 +- .../TableFunctionMergeTreeTextIndex.cpp | 5 +- .../TableFunctionProjection.cpp | 5 +- ...d_merge_prewhere_type_mismatch_2.reference | 6 +- ..._nested_merge_prewhere_type_mismatch_2.sql | 2 +- ...declaration_statistics_lazy_load.reference | 2 +- ...olumn_declaration_statistics_lazy_load.sql | 6 +- ...merge_prefilter_lazy_load_tables.reference | 2 +- ...33_lazy_load_tables_cast_storage.reference | 15 ++ .../05233_lazy_load_tables_cast_storage.sql | 44 ++++++ 75 files changed, 527 insertions(+), 177 deletions(-) create mode 100644 tests/queries/0_stateless/05233_lazy_load_tables_cast_storage.reference create mode 100644 tests/queries/0_stateless/05233_lazy_load_tables_cast_storage.sql diff --git a/ci/jobs/check_style.py b/ci/jobs/check_style.py index 26767e7eb84d1..2015f1d6ecd26 100644 --- a/ci/jobs/check_style.py +++ b/ci/jobs/check_style.py @@ -3,6 +3,7 @@ import math import multiprocessing import os +import pathlib import re import shlex from concurrent.futures import ProcessPoolExecutor @@ -878,6 +879,125 @@ def check_catch_all(files) -> str: return "\n".join(violations) +# Storage classes whose tables can be deferred behind `StorageTableProxy`, which means a pointer +# taken from `DatabaseCatalog` may be the proxy rather than the engine. +DEFERRABLE_STORAGE_CLASSES = ( + "MergeTreeData", + "StorageMergeTree", + "StorageReplicatedMergeTree", + "StorageSharedMergeTree", + "StorageSetOrJoinBase", + "StorageSet", + "StorageJoin", + "StorageSharedSet", + "StorageSharedJoin", + "StorageEmbeddedRocksDB", + "IKeyValueEntity", + "IStorageURLBase", + "IBackgroundOperation", + "StorageWithCommonVirtualColumns", + "StorageLog", + "StorageStripeLog", + "StorageURL", + "StorageObjectStorage", + "StorageKeeperMap", + "StorageMySQL", + "StoragePostgreSQL", + "StorageMongoDB", + "StorageRedis", + "StorageSQLite", + "StorageXDBC", + "StorageHive", + "StorageArrowFlight", + "StorageYTsaurus", + "StorageBigQuery", + "StorageKafka", + "StorageKafka2", + "StorageFileLog", + "StorageRabbitMQ", + "StorageNATS", + "StorageObjectStorageQueue", + "IStreamingStorage", +) + +# Casts on an operand that cannot be a catalog pointer, so no proxy can be in the way. +_NOT_A_CATALOG_POINTER = re.compile( + r"^(?:\*?this\b" + r"|shared_from_this\(\)" + r"|&?\w*(?:snapshot|storage_snapshot)->storage\b" + r"|&?\w*reading->getMergeTreeData\(\)" + r")" +) + + +def _without_comments(text): + """Blanks out comments, keeping the line layout so offsets and line numbers still match.""" + return re.sub( + r"//[^\n]*|/\*.*?\*/", lambda m: re.sub(r"[^\n]", " ", m.group(0)), text, flags=re.S + ) + + +def _cast_operand(text, open_paren): + """The argument of a cast whose '(' is at `open_paren`, or None when unbalanced.""" + depth = 0 + for i in range(open_paren, min(open_paren + 2000, len(text))): + depth += (text[i] == "(") - (text[i] == ")") + if not depth: + return " ".join(text[open_paren + 1 : i].split()) + return None + + +def check_storage_casts(files) -> str: + """Require `castStorage` for casts to an engine that supports deferred loading. + + Such a table lives behind `StorageTableProxy` until its first access and the catalog keeps + handing out that proxy afterwards, so a direct cast fails for the whole life of the table. + """ + types = "|".join(DEFERRABLE_STORAGE_CLASSES) + cast_head = re.compile( + r"\b(?Pdynamic_cast|typeid_cast|dynamic_pointer_cast|static_pointer_cast)\s*<\s*" + r"(?:const\s+)?(?:" + types + r")\s*[*&]?\s*>\s*\(" + ) + # `IStorage::as()` is a `typeid_cast` on the receiver, so the operand is what precedes it. + as_cast = re.compile( + r"(?P[\w.\[\]()]+)(?:->|\.)as\s*<\s*(?:const\s+)?(?:" + types + r")\s*>\s*\(\s*\)" + ) + resolvers = ("castStorage", "resolveStorageProxy", "resolveStorageProxyLoading") + + violations = [] + for path in files: + # The helpers and the proxy itself have to reach the nested storage directly. + if not path.endswith((".cpp", ".h")) or path.endswith( + ("StorageProxy.h", "StorageTableProxy.h", "StorageTableFunction.h") + ): + continue + try: + lines = pathlib.Path(path).read_text(errors="replace").splitlines() + except OSError: + continue + + text = _without_comments("\n".join(lines)) + casts = [(m, m.group("cast"), _cast_operand(text, m.end() - 1)) for m in cast_head.finditer(text)] + casts += [(m, "as", m.group("operand")) for m in as_cast.finditer(text)] + + for match, cast, operand in casts: + if not operand or _NOT_A_CATALOG_POINTER.match(operand) or any(r in operand for r in resolvers): + continue + line = text.count("\n", 0, match.start()) + 1 + # The marker goes on the cast or, when the line is long, the one above it. + if any("NOLINT(storage-cast)" in lines[i] for i in (line - 1, line - 2) if i >= 0): + continue + violations.append( + f"{path}:{line}: {cast} to a deferrable storage engine on `{operand[:60]}`. Such a table " + "is reached through StorageTableProxy, so this cast fails for the whole life of the table " + "and whatever it guards is silently skipped. Use castStorage(ptr, " + "DeferredTable::Load) when the query names this table, or DeferredTable::Skip " + "when this walks every table and must not load one. If the pointer cannot come from " + "DatabaseCatalog, say why in a `/// NOLINT(storage-cast)` comment." + ) + return "\n".join(violations) + + def check_file_names(files): files_set = set() for file in files: @@ -1371,6 +1491,15 @@ def parse_args(): files=cpp_files, ) ) + testname = "storage_casts" + if testpattern.lower() in testname.lower(): + results.append( + run_check_concurrent( + check_name=testname, + check_function=check_storage_casts, + files=cpp_files, + ) + ) testname = "compose_images_from_dockerhub" if testpattern.lower() in testname.lower(): results.append( diff --git a/src/Analyzer/Resolve/resolveFunction.cpp b/src/Analyzer/Resolve/resolveFunction.cpp index aacf493888b7c..750246027f426 100644 --- a/src/Analyzer/Resolve/resolveFunction.cpp +++ b/src/Analyzer/Resolve/resolveFunction.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -1310,7 +1311,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi scope.scope_node->formatASTForErrorMessage()); auto & table_node_typed = table_node->as(); - if (!std::dynamic_pointer_cast(table_node_typed.getStorage())) + if (!castStorage(table_node_typed.getStorage(), DeferredTable::Load)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} table '{}' should have engine StorageJoin. In scope {}", function_name, diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 7bb6c4ac0100b..0f9ee0107929c 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -33,12 +33,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include namespace fs = std::filesystem; @@ -438,6 +440,15 @@ void DatabaseOrdinary::loadTableFromMetadata( } } +/// Background work that deferring would cancel, or nothing to load, so these are never deferred. +static bool isEagerEngine(const String & engine_name) +{ + static const std::unordered_set eager_engines + = {"Distributed", "Buffer", "MaterializedPostgreSQL", "Merge", "Memory"}; + + return eager_engines.contains(engine_name); +} + bool DatabaseOrdinary::shouldLazyLoad(const ASTCreateQuery & query, LoadingStrictnessLevel mode) const { if (!database_metadata_disk_settings[DatabaseMetadataDiskSetting::lazy_load_tables]) @@ -452,6 +463,9 @@ bool DatabaseOrdinary::shouldLazyLoad(const ASTCreateQuery & query, LoadingStric if (query.is_time_series_table) return false; + if (query.storage && query.storage->engine && isEagerEngine(query.storage->engine->name)) + return false; + /// Already handled by `StorageTableFunctionProxy`. if (query.as_table_function) return false; @@ -534,7 +548,7 @@ LoadTaskPtr DatabaseOrdinary::loadTableFromMetadataAsync( void DatabaseOrdinary::restoreMetadataAfterConvertingToReplicated(StoragePtr table, const QualifiedTableName & name) { - auto * rmt = table->as(); + auto rmt = castStorage(table, DeferredTable::Skip); if (!rmt) return; diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 19906928b8c8f..d83f38975ccab 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -2903,7 +2904,7 @@ bool DatabaseReplicated::shouldReplicateQuery(const ContextPtr & query_context, auto table_id = query_context->resolveStorageID(ast, Context::ResolveOrdinary); StoragePtr table = DatabaseCatalog::instance().getTable(table_id, query_context); - return table->as() != nullptr; + return castStorage(table, DeferredTable::Load) != nullptr; }; const auto is_replicated_table = [&](const ASTPtr & ast) diff --git a/src/Databases/DatabaseURL.cpp b/src/Databases/DatabaseURL.cpp index 5ff8e6312e0fa..36874c55942ea 100644 --- a/src/Databases/DatabaseURL.cpp +++ b/src/Databases/DatabaseURL.cpp @@ -132,6 +132,7 @@ class StorageURLDatabaseTable final : public StorageProxy } StoragePtr getNested() const override { return nested; } + StoragePtr tryGetNested() const override { return nested; } String getName() const override { return nested->getName(); } /// Engine classification is used by policy checks (e.g. the `disable_insertion_and_mutation` diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index e3d7b87f97d8e..613b1b1aaf6be 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -168,7 +169,7 @@ getJoin(const ColumnsWithTypeAndName & arguments, ContextPtr context) const auto storage_id = context->resolveStorageID({qualified_name.database, qualified_name.table}); auto table = DatabaseCatalog::instance().getTable(storage_id, std::const_pointer_cast(context)); - auto storage_join = std::dynamic_pointer_cast(table); + auto storage_join = castStorage(table, DeferredTable::Load); if (!storage_join) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Table {} should have engine StorageJoin", join_name); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 8d1fc4ae00d7b..3be625694b4de 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -36,6 +36,7 @@ #include #include +#include #if CLICKHOUSE_CLOUD #include #endif @@ -1842,11 +1843,11 @@ FutureSetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool if (auto set = data.prepared_sets->findStorage(set_key)) return set; #if CLICKHOUSE_CLOUD - if (StorageSharedSet * storage_shared_set = dynamic_cast(table.get())) + if (StorageSharedSet * storage_shared_set = castStorage(table, DeferredTable::Load).get()) return data.prepared_sets->addFromStorage(set_key, right_in_operand, storage_shared_set->getSet(data.getContext()), table_id); #endif - if (StorageSet * storage_set = dynamic_cast(table.get())) + if (StorageSet * storage_set = castStorage(table, DeferredTable::Load).get()) return data.prepared_sets->addFromStorage(set_key, right_in_operand, storage_set->getSet(), table_id); } diff --git a/src/Interpreters/BlobStorageLog.cpp b/src/Interpreters/BlobStorageLog.cpp index 6ed63b50b8f47..2595b66ffc72e 100644 --- a/src/Interpreters/BlobStorageLog.cpp +++ b/src/Interpreters/BlobStorageLog.cpp @@ -14,6 +14,7 @@ #include #include +#include #include namespace DB @@ -103,7 +104,7 @@ static std::string_view normalizePath(std::string_view path) void BlobStorageLog::prepareTable() { SystemLog::prepareTable(); - if (auto merge_tree_table = std::dynamic_pointer_cast(getStorage())) + if (auto merge_tree_table = castStorage(getStorage(), DeferredTable::Skip)) { std::unique_lock lock{prepare_mutex}; const auto & relative_data_path = merge_tree_table->getRelativeDataPath(); diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index a2e6bbca35cd3..b1bb65960ec7c 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -264,7 +265,7 @@ void SelectStreamFactory::createForShardImpl( return; } - const auto * replicated_storage = dynamic_cast(main_table_storage.get()); + const auto * replicated_storage = castStorage(main_table_storage, DeferredTable::Load).get(); if (!replicated_storage) { diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index fc9da0b55af69..a5289588bb1c9 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -1744,7 +1745,7 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) for (const auto & [disk_name, disk] : getContext()->getDisksMap()) { String data_path = getStoreDirPath(table.table_id.uuid); - auto table_merge_tree = std::dynamic_pointer_cast(table.table); + auto table_merge_tree = castStorage(table.table, DeferredTable::Skip); if (!is_disk_eligible_for_search(disk, table_merge_tree) || !disk->existsDirectory(data_path)) continue; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 27359b9199bbf..ab5cb9cb7470c 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -70,6 +70,7 @@ #include #include #include +#include #include #include #include @@ -482,7 +483,7 @@ SetPtr ExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr & subquery_o const auto storage = DatabaseCatalog::instance().getTable(table_id, getContext()); if (storage->getName() != "Set") return nullptr; - const auto storage_set = std::dynamic_pointer_cast(storage); + const auto storage_set = castStorage(storage, DeferredTable::Load); return storage_set->getSet(); } diff --git a/src/Interpreters/IInterpreter.cpp b/src/Interpreters/IInterpreter.cpp index be42c0ca91aff..3c7093d87ef17 100644 --- a/src/Interpreters/IInterpreter.cpp +++ b/src/Interpreters/IInterpreter.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -55,7 +56,7 @@ void IInterpreter::checkStorageSupportsTransactionsIfNeeded(const StoragePtr & s /// Do not allow transactions with replicated tables or MergeTree tables anyway (unless it's a readonly SELECT query) /// because it may try to process transaction on MergeTreeData-level, /// but then fail with a logical error or something on Storage{Replicated,Shared}MergeTree-level. - if (storage->supportsReplication() || dynamic_cast(storage.get()) != nullptr) + if (storage->supportsReplication() || castStorage(storage, DeferredTable::Load) != nullptr) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} (table {}) does not support transactions", storage->getName(), storage->getStorageID().getNameForLogs()); } diff --git a/src/Interpreters/InsertDependenciesBuilder.cpp b/src/Interpreters/InsertDependenciesBuilder.cpp index b1479620e72fc..9dddbbbedf1ba 100644 --- a/src/Interpreters/InsertDependenciesBuilder.cpp +++ b/src/Interpreters/InsertDependenciesBuilder.cpp @@ -542,7 +542,7 @@ static DB::ConstraintsDescription buildConstraints(StorageMetadataPtr metadata, { auto constraints = metadata->getConstraints(); - auto storage_merge_tree = std::dynamic_pointer_cast(storage); + auto storage_merge_tree = castStorage(storage, DeferredTable::Load); if (storage_merge_tree && (storage_merge_tree->merging_params.mode == MergeTreeData::MergingParams::Collapsing || storage_merge_tree->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) @@ -722,7 +722,7 @@ class ExecutingInnerQueryFromViewTransform final : public ExceptionKeepingTransf local_context); bool inner_share_nested_offsets = true; - if (auto * merge_tree = dynamic_cast(inner_storage.get())) + if (auto * merge_tree = castStorage(inner_storage, DeferredTable::Load).get()) inner_share_nested_offsets = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; auto adding_missing_defaults_dag = addMissingDefaults( @@ -790,7 +790,7 @@ bool InsertDependenciesBuilder::storageDeduplicatesBlocksOnInsert(const StorageP /// MergeTree-family engines deduplicate inserted blocks when their (synchronous) deduplication /// window is enabled. This mirrors how `MergeTreeSink` / `ReplicatedMergeTreeSink` compute their /// own `deduplicate` flag. - if (const auto * merge_tree = dynamic_cast(storage.get())) + if (const auto * merge_tree = castStorage(storage, DeferredTable::Load).get()) { const auto merge_tree_settings = merge_tree->getSettings(); if (storage->supportsReplication()) @@ -1448,7 +1448,7 @@ bool InsertDependenciesBuilder::observePath(const DependencyPath & path) const auto & parent = path.parent(1); const auto & current = path.current(); - auto storage = current == init_table_id ? init_storage : DatabaseCatalog::instance().tryGetTable(current, init_context); + auto storage = current == init_table_id ? init_storage : resolveStorageProxyLoading(DatabaseCatalog::instance().tryGetTable(current, init_context)); auto lock = storage ? storage->tryLockForShare(init_context->getInitialQueryId(), init_context->getSettingsRef()[Setting::lock_acquire_timeout]) : nullptr; if (!lock) { @@ -1783,7 +1783,7 @@ Chain InsertDependenciesBuilder::createPreSink(StorageIDMaybeEmpty view_id) cons auto insert_context = insert_contexts.at(view_id); bool inner_share_nested_offsets = true; - if (auto * merge_tree = dynamic_cast(storages.at(inner_table_id).get())) + if (auto * merge_tree = castStorage(storages.at(inner_table_id), DeferredTable::Load).get()) inner_share_nested_offsets = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; /// Widen Enum columns to their target type before adding defaults, so the valid Enum-widening @@ -1877,7 +1877,7 @@ Chain InsertDependenciesBuilder::createSinkImpl(StorageIDMaybeEmpty view_id) con /// but currently we don't have methods for serialization of nested structures "as a whole". { bool skip_nested_validation = false; - if (auto * merge_tree = dynamic_cast(inner_storage.get())) + if (auto * merge_tree = castStorage(inner_storage, DeferredTable::Load).get()) skip_nested_validation = !(*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; if (!skip_nested_validation) result.addSink(std::make_shared(header)); diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index d6fd232999dc0..d9faa8bfa31a5 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -331,7 +332,7 @@ BlockIO runCommandSegments(CommandSegments & segments, const StoragePtr & table, alter_commands->validate(table, context); bool share_nested = true; - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) share_nested = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; alter_commands->prepare(*metadata_snapshot, share_nested); @@ -428,11 +429,14 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) if (table_id) { query_ptr->as().setDatabase(table_id.database_name); - table = DatabaseCatalog::instance().tryGetTable(table_id, getContext()); + /// Resolve once here so every branch below validates against the real structure instead of + /// the columns-only metadata a lazily loaded table reports. + table = resolveStorageProxyLoading(DatabaseCatalog::instance().tryGetTable(table_id, getContext())); } if (!alter.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) { + /// NOLINT(storage-cast): `table` is resolved above. if (table && table->as()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Mutations with ON CLUSTER are not allowed for KeeperMap tables"); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 19de34177dd74..84344019e717a 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -831,7 +832,10 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti { String as_database_name = getContext()->resolveDatabase(create.as_database); getContext()->checkAccess(AccessType::SHOW_COLUMNS, as_database_name, create.as_table); - StoragePtr as_storage = DatabaseCatalog::instance().getTable({as_database_name, create.as_table}, getContext()); + /// A lazily loaded source reports only its columns, so the indices, projections, constraints + /// and comment copied below would silently come out empty. + StoragePtr as_storage = resolveStorageProxyLoading( + DatabaseCatalog::instance().getTable({as_database_name, create.as_table}, getContext())); /// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table. as_storage_lock = as_storage->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); @@ -857,7 +861,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti /// CREATE TABLE AS should copy PRIMARY KEY, ORDER BY, and similar clauses. /// Note: only supports the source table engine is using the new syntax. - if (const auto * merge_tree_data = dynamic_cast(as_storage.get())) + if (const auto * merge_tree_data = castStorage(as_storage, DeferredTable::Load).get()) { if (merge_tree_data->format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { @@ -2452,7 +2456,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, if (!create.attach && getContext()->getSettingsRef()[Setting::database_replicated_allow_only_replicated_engine]) { - bool is_replicated_storage = typeid_cast(res.get()) != nullptr; + bool is_replicated_storage = castStorage(res, DeferredTable::Skip) != nullptr; if (!is_replicated_storage && res->storesDataOnDisk() && database && database->getEngineName() == "Replicated") throw Exception(ErrorCodes::UNKNOWN_STORAGE, "Only tables with a Replicated engine " @@ -2464,7 +2468,9 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, "ATTACH ... FROM ... query is not supported for {} table engine, " "because such tables do not store any data on disk. Use CREATE instead.", res->getName()); - auto * replicated_storage = typeid_cast(res.get()); + /// `res` is the storage this query just built, and for a table function it is a proxy that + /// resolving would run during CREATE. + auto * replicated_storage = castStorage(res, DeferredTable::Skip).get(); if (replicated_storage) { const auto probability = getContext()->getSettingsRef()[Setting::create_replicated_merge_tree_fault_injection_probability]; @@ -3100,7 +3106,8 @@ StoragePtr InterpreterCreateQuery::getValidatedAtomicPopulateSource(const ASTCre if (context->hasQueryContext()) context->getQueryContext()->dropStorageCacheEntry(*ref_dependencies.mv_from_dependency); - auto source = DatabaseCatalog::instance().tryGetTable(*ref_dependencies.mv_from_dependency, context); + auto source = resolveStorageProxyLoading( + DatabaseCatalog::instance().tryGetTable(*ref_dependencies.mv_from_dependency, context)); /// The view's SELECT was validated against the source before the view was published, so the source /// existed then; not finding it now means it was dropped, renamed or exchanged away in the window diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 256113146ad36..8d8b2b07e693c 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -95,7 +96,10 @@ BlockIO InterpreterDeleteQuery::execute() /// metadata is not loaded until the first access. Initialize it now so that /// supportsDelete() and subsequent mutation checks see valid metadata. table->updateExternalDynamicMetadataIfExists(getContext()); - auto metadata_snapshot = table->getInMemoryMetadataPtr(getContext(), false); + /// A lazily loaded table reports only its columns, so the checks below would see no projections + /// and validate the mutation against metadata that has no keys. + auto resolved_table = resolveStorageProxyLoading(table); + auto metadata_snapshot = resolved_table->getInMemoryMetadataPtr(getContext(), false); if (table->supportsDelete()) { @@ -137,7 +141,8 @@ BlockIO InterpreterDeleteQuery::execute() if (metadata_snapshot->hasProjections()) { - if (const auto * merge_tree_data = dynamic_cast(table.get())) + /// `MutateTask` treats THROW like DROP, so missing this check drops the projections. + if (const auto * merge_tree_data = castStorage(resolved_table, DeferredTable::Load).get()) if ((*merge_tree_data->getSettings())[MergeTreeSetting::lightweight_mutation_projection_mode] == LightweightMutationProjectionMode::THROW) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "DELETE query is not allowed for table {} because as it has projections and setting " diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index e3c37585e60b9..9e16ed6cc7224 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -330,7 +331,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, AS TableExclusiveLockHolder table_excl_lock; /// We don't need any lock for ReplicatedMergeTree and for simple MergeTree /// For the rest of tables types exclusive lock is needed - if (!std::dynamic_pointer_cast(table)) + if (!castStorage(table, DeferredTable::Load)) table_excl_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef()[Setting::lock_acquire_timeout]); auto metadata_snapshot = table->getInMemoryMetadataPtr(context_, false); diff --git a/src/Interpreters/InterpreterHypotheticalIndexQuery.cpp b/src/Interpreters/InterpreterHypotheticalIndexQuery.cpp index 4e05d0399fbb2..0cbd5e978c65a 100644 --- a/src/Interpreters/InterpreterHypotheticalIndexQuery.cpp +++ b/src/Interpreters/InterpreterHypotheticalIndexQuery.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -49,8 +50,20 @@ BlockIO InterpreterHypotheticalIndexQuery::execute() auto table_id = context->resolveStorageID(StorageID(query.getDatabase(), query.getTable())); auto table = DatabaseCatalog::instance().getTable(table_id, context); + auto & store = context->getHypotheticalObjectStore(); - const auto * merge_tree = dynamic_cast(table.get()); + /// Dropping only forgets a session entry, so it must not load a lazily loaded table. + if (query.kind == ASTHypotheticalIndexQuery::Drop) + { + /// No access check. a session-local drop leaks nothing (unlike CREATE), and every entry + /// in the session store already passed that check at creation + auto index_name = query.index_name->as().name(); + store.remove(table_id, index_name, query.if_exists); + return {}; + } + + table = resolveStorageProxyLoading(table); + const auto * merge_tree = castStorage(table, DeferredTable::Load).get(); if (!merge_tree) throw Exception( ErrorCodes::NOT_IMPLEMENTED, @@ -66,17 +79,6 @@ BlockIO InterpreterHypotheticalIndexQuery::execute() table_id.getDatabaseName(), table_id.getTableName()); - auto & store = context->getHypotheticalObjectStore(); - - if (query.kind == ASTHypotheticalIndexQuery::Drop) - { - /// No access check. a session-local drop leaks nothing (unlike CREATE), and every entry - /// in the session store already passed that check at creation - auto index_name = query.index_name->as().name(); - store.remove(table_id, index_name, query.if_exists); - return {}; - } - /// CREATE HYPOTHETICAL INDEX const auto & index_ast = query.index_decl->as(); auto metadata = table->getInMemoryMetadataPtr(context, /* bypass_metadata_cache = */ false); diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 9e05cac8341c6..a158eaf3ba07a 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -202,7 +203,8 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query) query.table_id = current_context->resolveStorageID(local_table_id); } - return DatabaseCatalog::instance().getTable(query.table_id, current_context); + /// The insert path reads engine facts the proxy of an unloaded table cannot answer. + return resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(query.table_id, current_context)); } Block InterpreterInsertQuery::getSampleBlock( @@ -1064,7 +1066,7 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica if (query.table_id.empty()) return {}; - StoragePtr dst_storage = DatabaseCatalog::instance().getTable(query.table_id, local_context); + StoragePtr dst_storage = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(query.table_id, local_context)); if (!(dst_storage->isMergeTree() || dst_storage->isDataLake()) || !dst_storage->supportsReplication()) return {}; diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index 756b35a9727e4..be8738e8ac335 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -1,6 +1,7 @@ #include "config.h" #include +#include #include #include #include @@ -47,7 +48,9 @@ BlockIO InterpreterOptimizeQuery::execute() getContext()->checkAccess(getRequiredAccess()); auto table_id = getContext()->resolveStorageID(ast); - StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); + /// Resolve before reading the metadata, so the checks below and `optimizeDryRun` see the real + /// structure rather than the columns-only one a lazily loaded table reports. + StoragePtr table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(table_id, getContext())); checkStorageSupportsTransactionsIfNeeded(table, getContext()); auto metadata_snapshot = table->getInMemoryMetadataPtr(getContext(), false); auto storage_snapshot = table->getStorageSnapshotWithoutData(metadata_snapshot, getContext()); @@ -59,7 +62,7 @@ BlockIO InterpreterOptimizeQuery::execute() throw Exception(ErrorCodes::BAD_ARGUMENTS, "OPTIMIZE MANIFEST is incompatible with FINAL, PARTITION, DEDUPLICATE, CLEANUP, and DRY RUN options"); #if USE_AVRO - auto * object_storage_table = dynamic_cast(table.get()); + auto object_storage_table = castStorage(table, DeferredTable::Skip); if (!object_storage_table) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "OPTIMIZE MANIFEST is only supported for Iceberg tables"); @@ -112,7 +115,7 @@ BlockIO InterpreterOptimizeQuery::execute() if (ast.dry_run) { - auto * merge_tree_data = dynamic_cast(table.get()); + auto * merge_tree_data = castStorage(table, DeferredTable::Load).get(); if (!merge_tree_data) throw Exception(ErrorCodes::BAD_ARGUMENTS, "OPTIMIZE DRY RUN is only supported for MergeTree family tables"); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 59354b87f0fa3..f3589d75b5b19 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1220,7 +1221,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() return true; } - auto storage_merge_tree = std::dynamic_pointer_cast(storage); + auto storage_merge_tree = castStorage(storage, DeferredTable::Load); if (!storage_merge_tree || settings[Setting::parallel_replicas_min_number_of_rows_per_replica] == 0) return false; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 0eda86dc12ab0..a8b7f75ad33da 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -75,6 +75,7 @@ #include #include #include +#include #include #include #include @@ -1374,7 +1375,7 @@ void InterpreterSystemQuery::restoreReplica() const StoragePtr table_ptr = DatabaseCatalog::instance().getTable(table_id, getContext()); - auto * const table_replicated_ptr = dynamic_cast(table_ptr.get()); + auto * const table_replicated_ptr = castStorage(table_ptr, DeferredTable::Load).get(); if (table_replicated_ptr == nullptr) throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs()); @@ -1447,7 +1448,9 @@ StoragePtr InterpreterSystemQuery::doRestartReplica(const StorageID & replica, C return nullptr; } - if (!dynamic_cast(table.get())) + /// The resolved pointer must not outlive this check. `waitDetachedTableNotInUse` below waits + /// for the last reference to the detached table to be released. + if (!castStorage(table, DeferredTable::Load)) { if (throw_on_error) throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), replica.getNameForLogs()); @@ -1626,7 +1629,7 @@ void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context) for (auto it = elem.second->getTablesIterator(getContext()); it->isValid(); it->next()) { - if (dynamic_cast(it->table().get())) + if (castStorage(it->table(), DeferredTable::Skip)) { if (!access_is_granted_globally && !access->isGranted(AccessType::SYSTEM_RESTART_REPLICA, elem.first, it->name())) { @@ -1730,7 +1733,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) DatabasePtr & database = elem.second; for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) { - if (auto * storage_replicated = dynamic_cast(iterator->table().get())) + if (auto * storage_replicated = castStorage(iterator->table(), DeferredTable::Skip).get()) { /// getReplicaPath() is built from getZooKeeperPath(), which strips only a single trailing /// slash, so a table created from "/a///" metadata keeps "/a//replicas/..." and would slip @@ -1775,7 +1778,8 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) bool InterpreterSystemQuery::dropStorageReplica(const String & query_replica, const StoragePtr & storage) { - auto * storage_replicated = dynamic_cast(storage.get()); + /// Dropping a replica from Keeper is what the command asks for, so loading the table is warranted. + auto * storage_replicated = castStorage(storage, DeferredTable::Load).get(); if (!storage_replicated) return false; @@ -2189,7 +2193,8 @@ bool InterpreterSystemQuery::trySyncReplica(StoragePtr table, SyncReplicaMode sy break; } - if (auto * storage_replicated = dynamic_cast(table.get())) + + if (auto * storage_replicated = castStorage(table, DeferredTable::Load).get()) { auto log = getLogger("InterpreterSystemQuery"); LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for current last entry to be processed"); @@ -2237,7 +2242,7 @@ void InterpreterSystemQuery::waitLoadingParts() getContext()->checkAccess(AccessType::SYSTEM_WAIT_LOADING_PARTS, table_id); StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) { LOG_TRACE(log, "Waiting for loading of parts of table {}", table_id.getFullTableName()); merge_tree->waitForOutdatedPartsToBeLoaded(); @@ -2283,7 +2288,7 @@ void InterpreterSystemQuery::restartDisk(const String & disk_name) /// skip_not_loaded: act only on already-loaded tables, do not block on async loading. for (auto it = elem.second->getTablesIterator(getContext(), {}, /*skip_not_loaded=*/ true); it->isValid(); it->next()) { - auto * merge_tree = dynamic_cast(it->table().get()); + auto * merge_tree = castStorage(it->table(), DeferredTable::Skip).get(); if (!merge_tree) continue; @@ -2319,7 +2324,8 @@ namespace MergeTreeData & getMergeTreeWithManualSelector(const StoragePtr & table, const StorageID & table_id, const char * action) { - auto * merge_tree = dynamic_cast(table.get()); + auto resolved = resolveStorageProxyLoading(table); + auto * merge_tree = castStorage(resolved, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Command {} is supported only for MergeTree-family tables, but got: {}", @@ -2413,7 +2419,7 @@ void InterpreterSystemQuery::loadOrUnloadPrimaryKeysImpl(bool load) getContext()->checkAccess(load ? AccessType::SYSTEM_LOAD_PRIMARY_KEY : AccessType::SYSTEM_UNLOAD_PRIMARY_KEY, table_id.database_name, table_id.table_name); StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) { LOG_TRACE(log, "{} primary keys for table {}", load ? "Loading" : "Unloading", table_id.getFullTableName()); load ? merge_tree->loadPrimaryKeys() : merge_tree->unloadPrimaryKeys(); @@ -2433,7 +2439,7 @@ void InterpreterSystemQuery::loadOrUnloadPrimaryKeysImpl(bool load) { for (auto it = database.second->getTablesIterator(getContext()); it->isValid(); it->next()) { - if (auto * merge_tree = dynamic_cast(it->table().get())) + if (auto * merge_tree = castStorage(it->table(), DeferredTable::Skip).get()) { load ? merge_tree->loadPrimaryKeys() : merge_tree->unloadPrimaryKeys(); } @@ -2578,7 +2584,7 @@ void InterpreterSystemQuery::flushObjectStorageQueue(ASTSystemQuery & query) throw Exception(ErrorCodes::BAD_ARGUMENTS, "PATH must be specified for SYSTEM FLUSH OBJECT STORAGE QUEUE"); auto table = DatabaseCatalog::instance().getTable(table_id, context); - auto * queue = dynamic_cast(table.get()); + auto queue = castStorage(table, DeferredTable::Load); if (!queue) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table {} is not an S3Queue or AzureQueue table", table_id.getNameForLogs()); @@ -2639,7 +2645,7 @@ void InterpreterSystemQuery::controlBackgroundActivity(const ASTSystemQuery & qu const bool can_views = access->isGranted(AccessType::SYSTEM_VIEWS, table_id.database_name, table_id.table_name); const bool can_streaming = access->isGranted(AccessType::SYSTEM_STREAMING_ENGINES, table_id.database_name, table_id.table_name); - auto storage = DatabaseCatalog::instance().tryGetTable(table_id, getContext()); + auto storage = resolveStorageProxyLoading(DatabaseCatalog::instance().tryGetTable(table_id, getContext())); const bool is_streaming = storage && storage->isStreamingStorage(); const auto * mv = storage ? dynamic_cast(storage.get()) : nullptr; const bool is_refreshable_view = mv && mv->isRefreshable(); @@ -2721,8 +2727,8 @@ void InterpreterSystemQuery::prewarmMarkCache() getContext()->checkAccess(AccessType::SYSTEM_PREWARM_MARK_CACHE, table_id); - auto table_ptr = DatabaseCatalog::instance().getTable(table_id, getContext()); - auto * merge_tree = dynamic_cast(table_ptr.get()); + auto table_ptr = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(table_id, getContext())); + auto * merge_tree = castStorage(table_ptr, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Command PREWARM MARK CACHE is supported only for MergeTree table, but got: {}", table_ptr->getName()); @@ -2745,8 +2751,8 @@ void InterpreterSystemQuery::prewarmPrimaryIndexCache() getContext()->checkAccess(AccessType::SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, table_id); - auto table_ptr = DatabaseCatalog::instance().getTable(table_id, getContext()); - auto * merge_tree = dynamic_cast(table_ptr.get()); + auto table_ptr = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(table_id, getContext())); + auto * merge_tree = castStorage(table_ptr, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Command PREWARM PRIMARY INDEX CACHE is supported only for MergeTree table, but got: {}", table_ptr->getName()); diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index b1abb5e4a4785..f3d671cf38c15 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -356,7 +357,7 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se if (storage) { - if (auto storage_join = std::dynamic_pointer_cast(storage); storage_join) + if (auto storage_join = castStorage(storage, DeferredTable::Load); storage_join) { table_join->setStorageJoin(storage_join); } @@ -379,11 +380,12 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se return nullptr; } + /// NOLINT(storage-cast): a dictionary, which the catalog never hands out behind a proxy. auto dictionary_kv = std::dynamic_pointer_cast(dictionary); table_join->setStorageJoin(dictionary_kv); } - if (auto storage_kv = std::dynamic_pointer_cast(storage); storage_kv && try_use_direct_join) + if (auto storage_kv = castStorage(storage, DeferredTable::Load); storage_kv && try_use_direct_join) { table_join->setStorageJoin(storage_kv); } diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 548460b6b13f8..f286a7af274b7 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -338,8 +339,9 @@ ASTPtr getPartitionAndPredicateExpressionForMutationCommand( { String partition_id; - auto storage_merge_tree = std::dynamic_pointer_cast(storage); - auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast(storage); + auto resolved_storage = resolveStorageProxyLoading(storage); + auto storage_merge_tree = castStorage(resolved_storage, DeferredTable::Load); + auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast(resolved_storage); if (storage_merge_tree) partition_id = storage_merge_tree->getPartitionIDFromQuery(ASTPtr(alter->partition), context); else if (storage_from_merge_tree_data_part) @@ -367,7 +369,9 @@ ASTPtr getPartitionAndPredicateExpressionForMutationCommand( return predicate_ast; } -MutationsInterpreter::Source::Source(StoragePtr storage_) : storage(std::move(storage_)) +/// A mutation reads and rewrites parts, so it needs the real storage rather than the proxy a lazily +/// loaded table is reached through. +MutationsInterpreter::Source::Source(StoragePtr storage_) : storage(resolveStorageProxyLoading(storage_)) { } @@ -409,7 +413,7 @@ const MergeTreeData * MutationsInterpreter::Source::getMergeTreeData() const if (data) return data; - return dynamic_cast(storage.get()); + return castStorage(storage, DeferredTable::Load).get(); } MergeTreeData::DataPartPtr MutationsInterpreter::Source::getMergeTreeDataPart() const @@ -490,7 +494,7 @@ MutationsInterpreter::MutationsInterpreter( std::move(available_columns_), std::move(context_), std::move(settings_)) { - if (settings.can_execute && !settings.return_mutated_rows && dynamic_cast(source.getStorage().get())) + if (settings.can_execute && !settings.return_mutated_rows && castStorage(source.getStorage(), DeferredTable::Load)) { throw Exception( ErrorCodes::LOGICAL_ERROR, diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 3ca6fc71beb95..c24ca42a48f4f 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -34,6 +34,7 @@ #include #endif +#include #include #include #include @@ -495,11 +496,11 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr if (is_system) ++total_number_of_tables_system; - const auto & table = iterator->table(); + auto table = iterator->table(); if (!table) continue; - if (MergeTreeData * table_merge_tree = dynamic_cast(table.get())) + if (auto table_merge_tree = castStorage(table, DeferredTable::Skip)) { calculateMax(max_part_count_for_partition, table_merge_tree->getMaxPartsCountAndSizeForPartition().first); @@ -541,7 +542,7 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr } } - if (StorageReplicatedMergeTree * table_replicated_merge_tree = typeid_cast(table.get())) + if (StorageReplicatedMergeTree * table_replicated_merge_tree = castStorage(table, DeferredTable::Skip).get()) { StorageReplicatedMergeTree::ReplicatedStatus status; table_replicated_merge_tree->getStatus(status, false); @@ -684,11 +685,7 @@ void ServerAsynchronousMetrics::updateMutationAndDetachedPartsStats() for (auto iterator = db.second->getTablesIterator(getContext(), {}, true); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); - if (!table) - continue; - - if (MergeTreeData * table_merge_tree = dynamic_cast(table.get())) + if (auto table_merge_tree = castStorage(iterator->table(), DeferredTable::Skip)) { for (const auto & detached_part: table_merge_tree->getDetachedParts()) { diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index 2e32f08f8e6fe..01c4d564f1470 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -2,6 +2,7 @@ #include #include +#include #if CLICKHOUSE_CLOUD #include #endif @@ -74,7 +75,7 @@ class CollectSetsVisitor : public InDepthQueryTreeVisitorWithContextas(); - StorageSet * storage_set = second_argument_table != nullptr ? dynamic_cast(second_argument_table->getStorage().get()) : nullptr; + StorageSet * storage_set = second_argument_table != nullptr ? castStorage(second_argument_table->getStorage(), DeferredTable::Load).get() : nullptr; if (storage_set) { diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 5fba025023647..c1f673221fd1d 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -64,6 +64,7 @@ #include #include #include +#include #include #include @@ -277,7 +278,7 @@ FiltersForTableExpressionMap collectFiltersForAnalysis(const QueryTreeNodePtr & const auto * raw = storage_ptr.get(); if (typeid_cast(raw)) return true; - if (parallel_replicas_estimation_enabled && std::dynamic_pointer_cast(storage_ptr)) + if (parallel_replicas_estimation_enabled && castStorage(storage_ptr, DeferredTable::Load)) return true; if (typeid_cast(raw)) return true; diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index 9503f927d31d8..5ac521c6209fe 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -831,7 +832,7 @@ static JoinClausesAndActions buildJoinClausesAndActions( bool is_join_with_special_storage = false; if (const auto * right_table_node = join_node.getRightTableExpressionNode()->as()) { - is_join_with_special_storage = dynamic_cast(right_table_node->getStorage().get()); + is_join_with_special_storage = castStorage(right_table_node->getStorage(), DeferredTable::Load).get(); } for (auto & join_clause : result.join_clauses) @@ -1028,7 +1029,7 @@ void trySetStorageInTableJoin(const QueryTreeNodePtr & table_expression, std::sh else if (auto * table_function = table_expression->as()) storage = table_function->getStorage(); - auto storage_join = std::dynamic_pointer_cast(storage); + auto storage_join = castStorage(storage, DeferredTable::Load); if (storage_join) { table_join->setStorageJoin(storage_join); @@ -1040,8 +1041,9 @@ void trySetStorageInTableJoin(const QueryTreeNodePtr & table_expression, std::sh if (auto storage_dictionary = std::dynamic_pointer_cast(storage); storage_dictionary && storage_dictionary->getDictionary()->getSpecialKeyType() != DictionarySpecialKeyType::Range) + /// NOLINT(storage-cast): a dictionary, which the catalog never hands out behind a proxy. table_join->setStorageJoin(std::dynamic_pointer_cast(storage_dictionary->getDictionary())); - else if (auto storage_key_value = std::dynamic_pointer_cast(storage); storage_key_value) + else if (auto storage_key_value = castStorage(storage, DeferredTable::Load); storage_key_value) table_join->setStorageJoin(storage_key_value); } diff --git a/src/Planner/PlannerJoinsLogical.cpp b/src/Planner/PlannerJoinsLogical.cpp index 1d64aee2bc9f4..d0582e099162a 100644 --- a/src/Planner/PlannerJoinsLogical.cpp +++ b/src/Planner/PlannerJoinsLogical.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -596,18 +597,19 @@ PreparedJoinStorage tryGetStorageInTableJoin(const QueryTreeNodePtr & table_expr const auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); result.column_mapping = table_expression_data.getColumnIdentifierToColumnName(); - result.storage_join = std::dynamic_pointer_cast(storage); + result.storage_join = castStorage(storage, DeferredTable::Load); if (result.storage_join) return result; auto storage_dictionary = std::dynamic_pointer_cast(storage); if (storage_dictionary && storage_dictionary->getDictionary()->getSpecialKeyType() != DictionarySpecialKeyType::Range) { + /// NOLINT(storage-cast): a dictionary, which the catalog never hands out behind a proxy. result.storage_key_value = std::dynamic_pointer_cast(storage_dictionary->getDictionary()); return result; } - result.storage_key_value = std::dynamic_pointer_cast(storage); + result.storage_key_value = castStorage(storage, DeferredTable::Load); if (result.storage_key_value) return result; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index db565127c1525..748dfddbb1cf5 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -6555,7 +6556,7 @@ std::unique_ptr ReadFromMergeTree::deserialize(Deserialization & StorageID table_id(database_name, table_name); auto storage_ptr = DatabaseCatalog::instance().getTable(table_id, ctx.context); - auto * merge_tree = dynamic_cast(storage_ptr.get()); + auto * merge_tree = castStorage(storage_ptr, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} is not a MergeTree table", table_id.getNameForLogs()); diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 80fad44c050e9..12544a025ef4e 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -658,7 +659,7 @@ void ReadFromRemote::addLazyPipe( // has no local storage and reaches a lazy shard only via the failpoint, so it always reads remotely. if (!use_delayed_remote_source && !my_table_func_ptr) { - const auto replicated_storage = std::dynamic_pointer_cast(my_storage); + const auto replicated_storage = castStorage(my_storage, DeferredTable::Load); if (!replicated_storage) { throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected lazy remote read from a non-replicated table: {}", my_storage->getName()); diff --git a/src/Processors/QueryPlan/SetsSerialization.cpp b/src/Processors/QueryPlan/SetsSerialization.cpp index 92a8a932224c2..c4bab839c4cd2 100644 --- a/src/Processors/QueryPlan/SetsSerialization.cpp +++ b/src/Processors/QueryPlan/SetsSerialization.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB { @@ -261,7 +262,7 @@ static void makeSetsFromStorage(std::list sets { Identifier identifier = parseTableIdentifier(set.storage_name, context); auto table_node = resolveTable(identifier, context); - const auto * storage_set = typeid_cast(table_node->getStorage().get()); + const auto * storage_set = castStorage(table_node->getStorage(), DeferredTable::Load).get(); if (!storage_set) throw Exception(ErrorCodes::INCORRECT_DATA, "Table {} is not a StorageSet", set.storage_name); diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 9932a74c2c4e3..7b76374825544 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -68,12 +69,7 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe // If they have some lag it will be reflected as soon as they are load. for (auto iterator = db.second->getTablesIterator(getContext(), {}, true); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); - if (!table) - continue; - - StorageReplicatedMergeTree * table_replicated = dynamic_cast(table.get()); - + auto table_replicated = castStorage(iterator->table(), DeferredTable::Skip); if (!table_replicated) continue; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index c681f104ec357..938759d9205a0 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1750,7 +1751,8 @@ void TCPHandler::processTablesStatusRequest() continue; TableStatus status; - if (auto * replicated_table = dynamic_cast(table.get())) + /// The initiator asks about this table by name, so a lazily loaded replica is loaded to report its delay. + if (auto * replicated_table = castStorage(table, DeferredTable::Load).get()) { status.is_replicated = true; status.absolute_delay = static_cast(replicated_table->getAbsoluteDelay()); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 65f529e85a8a6..693e332bb674d 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -2027,7 +2028,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const const auto virtuals = metadata->virtuals; bool share_nested = true; - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) share_nested = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; auto all_columns = metadata->columns; @@ -2324,7 +2325,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const if (all_columns.hasNested(command.column_name)) { bool skip = false; - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) skip = !(*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; if (!skip) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot rename whole Nested struct"); @@ -2367,7 +2368,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const /// When share_nested_offsets is disabled, dotted-name columns are independent /// and not part of a Nested group, so they can be freely renamed. - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) { if (!(*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]) { diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 96662725e3a88..24b6f10fa50f4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -8473,13 +8474,13 @@ static void assertNotPatchPartition(const MergeTreeData & data, const ASTPtr & p void MergeTreeData::movePartitionToTable(const PartitionCommand & command, ContextPtr query_context) { String dest_database = query_context->resolveDatabase(command.to_database); - auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table}, query_context); + auto dest_storage = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable({dest_database, command.to_table}, query_context)); /// The target table and the source table are the same. if (dest_storage->getStorageID() == this->getStorageID()) return; - auto * dest_storage_merge_tree = dynamic_cast(dest_storage.get()); + auto * dest_storage_merge_tree = castStorage(dest_storage, DeferredTable::Load).get(); if (!dest_storage_merge_tree) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot move partition from table {} to table {} with storage {}", @@ -8616,9 +8617,9 @@ Pipe MergeTreeData::alterPartition( checkPartitionCanBeDropped(command.partition, query_context); auto resolved = query_context->resolveStorageID({command.from_database, command.from_table}); - auto from_storage = DatabaseCatalog::instance().getTable(resolved, query_context); + auto from_storage = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(resolved, query_context)); - auto * from_storage_merge_tree = dynamic_cast(from_storage.get()); + auto * from_storage_merge_tree = castStorage(from_storage, DeferredTable::Load).get(); if (!from_storage_merge_tree) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot replace partition from table {} with storage {} to table {}", @@ -11059,6 +11060,7 @@ void MergeTreeData::checkColumnFilenamesForCollision(const ColumnsDescription & MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & source_table, const StorageMetadataPtr & src_snapshot, const StorageMetadataPtr & my_snapshot) const { + /// NOLINT(storage-cast): a reference, and the `StoragePtr` overload below resolves the proxy. MergeTreeData * src_data = dynamic_cast(&source_table); if (!src_data) throw Exception(ErrorCodes::NOT_IMPLEMENTED, @@ -11118,7 +11120,7 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & sour MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData( const StoragePtr & source_table, const StorageMetadataPtr & src_snapshot, const StorageMetadataPtr & my_snapshot) const { - return checkStructureAndGetMergeTreeData(*source_table, src_snapshot, my_snapshot); + return checkStructureAndGetMergeTreeData(*resolveStorageProxyLoading(source_table), src_snapshot, my_snapshot); } /// must_on_same_disk=false is used only when attach partition; Both for same disk and different disk. diff --git a/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp b/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp index 09786a9b73bad..72fad84b226d0 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp +++ b/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp @@ -28,6 +28,7 @@ StorageFromMergeTreeProjection::StorageFromMergeTreeProjection( StorageID storage_id_, StoragePtr parent_storage_, StorageMetadataPtr parent_metadata_, ProjectionDescriptionRawPtr projection_) : IStorage(storage_id_) , parent_storage(std::move(parent_storage_)) + /// NOLINT(storage-cast): the table function resolves the source table before building this. , merge_tree(dynamic_cast(*parent_storage)) , parent_metadata(std::move(parent_metadata_)) , projection(projection_) diff --git a/src/Storages/MergeTree/WhatIfIndexEstimator.cpp b/src/Storages/MergeTree/WhatIfIndexEstimator.cpp index 5db904a0da9b6..c5cc88c27ad53 100644 --- a/src/Storages/MergeTree/WhatIfIndexEstimator.cpp +++ b/src/Storages/MergeTree/WhatIfIndexEstimator.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -332,7 +333,7 @@ WhatIfResult estimateHypotheticalIndexes( { auto storage = tryResolveSingleTable(select_query, local_context); const auto & store = local_context->getHypotheticalObjectStore(); - if (const auto * mt = dynamic_cast(storage.get())) + if (const auto * mt = castStorage(storage, DeferredTable::Load).get()) { /// Empty table -> ReadNothing, report a zero baseline if (mt->getActivePartsCount() == 0) diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp index d19fd8ed616cf..61eb507f3036e 100644 --- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp +++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -61,7 +62,7 @@ void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr con for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { StoragePtr table = iterator->table(); - RocksDBStoragePtr rocksdb_table = table ? std::dynamic_pointer_cast(table) : nullptr; + RocksDBStoragePtr rocksdb_table = table ? castStorage(table, DeferredTable::Skip) : nullptr; if (!rocksdb_table) continue; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index f2afa34d29580..2c4cb0a58661a 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -3270,7 +3271,7 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con /// Clang's thread-safety analyzer, which cannot track mutex ownership across `std::lock`. void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr local_context) TSA_NO_THREAD_SAFETY_ANALYSIS { - auto dest_table_storage = std::dynamic_pointer_cast(dest_table); + auto dest_table_storage = std::dynamic_pointer_cast(resolveStorageProxyLoading(dest_table)); if (!dest_table_storage) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table {} supports movePartitionToTable only for MergeTree family of table engines. Got {}", diff --git a/src/Storages/StorageMergeTreeAnalyzeIndexes.cpp b/src/Storages/StorageMergeTreeAnalyzeIndexes.cpp index b83639f014682..969ba92a36d3b 100644 --- a/src/Storages/StorageMergeTreeAnalyzeIndexes.cpp +++ b/src/Storages/StorageMergeTreeAnalyzeIndexes.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -128,7 +129,7 @@ class MergeTreeAnalyzeIndexSource final : public ISource, WithContext auto reader_settings = MergeTreeReaderSettings::createForQuery(context, *table_settings, query_info); const auto metadata_snapshot = storage->getInMemoryMetadataPtr(context, false); - const auto * merge_tree_data = dynamic_cast(storage.get()); + const auto * merge_tree_data = castStorage(storage, DeferredTable::Load).get(); if (!merge_tree_data) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeAnalyzeIndexes expected MergeTree table, got: {}", storage->getName()); @@ -305,11 +306,11 @@ StorageMergeTreeAnalyzeIndexes::StorageMergeTreeAnalyzeIndexes( const ASTPtr & predicate_, const OptionalVectorSearchParameters & vector_search_parameters_) : StorageWithCommonVirtualColumns(table_id_) - , source_table(source_table_) + , source_table(resolveStorageProxyLoading(source_table_)) , predicate(predicate_) , vector_search_parameters(vector_search_parameters_) { - const auto * merge_tree_data = dynamic_cast(source_table.get()); + const auto * merge_tree_data = castStorage(source_table, DeferredTable::Load).get(); if (!merge_tree_data) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeAnalyzeIndexes expected MergeTree table, got: {}", source_table->getName()); diff --git a/src/Storages/StorageMergeTreeCodecBlockCounts.cpp b/src/Storages/StorageMergeTreeCodecBlockCounts.cpp index a02ea2d0473f3..6dc20b3c09af0 100644 --- a/src/Storages/StorageMergeTreeCodecBlockCounts.cpp +++ b/src/Storages/StorageMergeTreeCodecBlockCounts.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -313,7 +314,7 @@ void StorageMergeTreeCodecBlockCounts::checkSourceTableAccess(const StoragePtr & context->checkAccess(AccessType::SELECT, source_table->getStorageID(), source_metadata->getColumns().getNamesOfPhysical()); } -StoragePtr StorageMergeTreeCodecBlockCounts::resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context) +std::shared_ptr StorageMergeTreeCodecBlockCounts::resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context) { /// `SHOW TABLES` is the privilege that governs whether the table's existence may be learned, and it is implied /// by a grant on any single column of it, so this only adds a tier below the `SELECT` check on every column @@ -324,11 +325,13 @@ StoragePtr StorageMergeTreeCodecBlockCounts::resolveSourceTable(const StorageID auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); checkSourceTableAccess(source_table, context); - if (!dynamic_cast(source_table.get())) + /// Reading the parts is what the function is for, so a not yet loaded source table is loaded here. + auto merge_tree = castStorage(source_table, DeferredTable::Load); + if (!merge_tree) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Table function mergeTreeCodecBlockCounts expected MergeTree table, got: {}", source_table->getName()); - return source_table; + return merge_tree; } void StorageMergeTreeCodecBlockCounts::read( @@ -344,15 +347,11 @@ void StorageMergeTreeCodecBlockCounts::read( storage_snapshot->check(column_names); /// Under the reader's context, see the constructor. - auto source_table = resolveSourceTable(source_table_id, context); - const auto source_storage_id = source_table->getStorageID(); - - /// A cast to a base class, so not `assert_cast`, which asserts the exact type and would reject every - /// `MergeTree` table. `resolveSourceTable` has already rejected a source table that is not a `MergeTree`. - const auto & merge_tree = dynamic_cast(*source_table); + auto merge_tree = resolveSourceTable(source_table_id, context); + const auto source_storage_id = merge_tree->getStorageID(); /// `system.parts_columns` lists patch parts, so this function does too. - auto data_parts = merge_tree.getDataPartsVectorForInternalUsage( + auto data_parts = merge_tree->getDataPartsVectorForInternalUsage( {MergeTreeData::DataPartState::Active}, {MergeTreeData::DataPartKind::Regular, MergeTreeData::DataPartKind::Patch}); std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); @@ -374,7 +373,7 @@ void StorageMergeTreeCodecBlockCounts::read( } /// The parts reference the source table's MergeTreeData without owning it. - query_plan.addStorageHolder(source_table); + query_plan.addStorageHolder(merge_tree); query_plan.addStep( std::make_unique( diff --git a/src/Storages/StorageMergeTreeCodecBlockCounts.h b/src/Storages/StorageMergeTreeCodecBlockCounts.h index e8852112a0aef..4faa519d942ec 100644 --- a/src/Storages/StorageMergeTreeCodecBlockCounts.h +++ b/src/Storages/StorageMergeTreeCodecBlockCounts.h @@ -6,6 +6,8 @@ namespace DB { +class MergeTreeData; + /// Backing storage for `mergeTreeCodecBlockCounts(database, table)`. One row per (part, column, substream). /// Counts compressed blocks per codec by reading each stream's `.bin` header. Selecting `part_name`/`column`/`substream` is metadata-only. class StorageMergeTreeCodecBlockCounts final : public IStorage @@ -27,7 +29,7 @@ class StorageMergeTreeCodecBlockCounts final : public IStorage /// `SHOW TABLES` on the name, before the catalog is consulted, so that an inaccessible table and a missing one /// answer alike; then `SELECT` on every column, before the engine is examined, so that a user without it cannot /// learn the engine from the `BAD_ARGUMENTS` that rejects a table that is not a `MergeTree`. - static StoragePtr resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context); + static std::shared_ptr resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context); void read( QueryPlan & query_plan, diff --git a/src/Storages/StorageMergeTreeIndex.cpp b/src/Storages/StorageMergeTreeIndex.cpp index 54566034cc773..3c463efbe9352 100644 --- a/src/Storages/StorageMergeTreeIndex.cpp +++ b/src/Storages/StorageMergeTreeIndex.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -294,7 +295,7 @@ StorageMergeTreeIndex::StorageMergeTreeIndex( , with_marks(with_marks_) , with_minmax(with_minmax_) { - const auto * merge_tree = dynamic_cast(source_table.get()); + const auto * merge_tree = castStorage(source_table, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeIndex expected MergeTree table, got: {}", source_table->getName()); diff --git a/src/Storages/StorageMergeTreeTextIndex.cpp b/src/Storages/StorageMergeTreeTextIndex.cpp index 2ebf5b4a57ce7..4ca65f7c9cc10 100644 --- a/src/Storages/StorageMergeTreeTextIndex.cpp +++ b/src/Storages/StorageMergeTreeTextIndex.cpp @@ -1,4 +1,5 @@ +#include #include #include @@ -362,6 +363,7 @@ void ReadFromMergeTreeTextIndex::applyFilters(ActionDAGNodes added_filter_nodes) void ReadFromMergeTreeTextIndex::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { /// Taken at read time: the storage outlives the query in a table created from the function before that was forbidden. + /// NOLINT(storage-cast): the table function resolves the source table before building this. auto data_parts = dynamic_cast(*storage->source_table).getDataPartsVectorForInternalUsage(); std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); auto filtered_parts = VirtualColumnUtils::filterDataPartsWithExpression(data_parts, virtual_columns_filter); @@ -410,7 +412,7 @@ StorageMergeTreeTextIndex::StorageMergeTreeTextIndex( , source_table(source_table_) , text_index(std::move(text_index_)) { - if (!dynamic_cast(source_table.get())) + if (!castStorage(source_table, DeferredTable::Load)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeTextIndex expected MergeTree table, got: {}", source_table->getName()); StorageInMemoryMetadata storage_metadata; diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index bccc4ffeae5e4..2dc249080a2cf 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -16,6 +16,14 @@ class StorageProxy : public IStorage virtual StoragePtr getNested() const = 0; + /// The wrapped storage if it already exists, or null. Never creates it, so an observer + /// iterating every table cannot trigger a load. + virtual StoragePtr tryGetNested() const { return nullptr; } + + /// Whether the proxy only defers the creation of the storage and adds no behaviour of its own, + /// so that an operation may run on the wrapped storage directly. + virtual bool isLazyStandIn() const { return false; } + String getName() const override { return "Proxy"; } bool isRemote() const override { return getNested()->isRemote(); } @@ -194,5 +202,57 @@ class StorageProxy : public IStorage }; +/// The storage an operation should run on: the lazy stand-in is replaced by the storage it wraps +/// once that exists, while the other proxies add behaviour of their own and are kept. +inline StoragePtr resolveStorageProxy(const StoragePtr & storage) +{ + const auto * proxy = dynamic_cast(storage.get()); + if (!proxy || !proxy->isLazyStandIn()) + return storage; + auto nested = proxy->tryGetNested(); + return nested ? nested : storage; +} + +/// Same, but creates the wrapped storage when it does not exist yet. For operations that name a +/// table explicitly, where loading it is the expected cost of the operation. +inline StoragePtr resolveStorageProxyLoading(const StoragePtr & storage) +{ + const auto * proxy = dynamic_cast(storage.get()); + return proxy && proxy->isLazyStandIn() ? proxy->getNested() : storage; +} + +/// Proxies stack: a lazily loaded `URL` table is a `StorageTableProxy` over a `StorageURLSchemeDispatch` +/// over the real storage. The bound only guards against a cycle. +constexpr size_t max_storage_proxy_depth = 16; + +/// What a cast does with a table that is not loaded yet. +enum class DeferredTable : uint8_t +{ + /// Load it. For an operation that names the table, where loading is its expected cost. + Load, + /// Leave it unloaded, so the cast yields null and the caller skips it. For an observer that + /// walks every table and must not turn a listing into a load. + Skip, +}; + +/// The single way to cast a catalog pointer to a concrete engine type. A lazily loaded table is +/// reached through `StorageTableProxy`, so a direct cast fails even once the table is loaded. +template +std::shared_ptr castStorage(const StoragePtr & storage, DeferredTable deferred_table) +{ + /// The type test looks through every layer, whichever wrappers sit on top of the engine. + StoragePtr resolved = storage; + for (size_t depth = 0; depth < max_storage_proxy_depth && resolved; ++depth) + { + const auto * proxy = dynamic_cast(resolved.get()); + if (!proxy) + break; + auto nested = deferred_table == DeferredTable::Load ? proxy->getNested() : proxy->tryGetNested(); + if (!nested) + break; + resolved = nested; + } + return std::dynamic_pointer_cast(resolved); +} } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 33a9ce1173d22..5637e4431ebe3 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -75,6 +75,7 @@ #include #include #include +#include #include #include #include @@ -3207,7 +3208,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) auto clone_data_parts_from_source_table = [&] () -> size_t { - source_table = DatabaseCatalog::instance().tryGetTable(source_table_id, getContext()); + /// Leaving this proxied would make the checks below read the source as not replicated. + source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().tryGetTable(source_table_id, getContext())); if (!source_table) { LOG_DEBUG(log, "Can't use {} as source table for REPLACE PARTITION command. It does not exist.", source_table_id.getNameForLogs()); @@ -3339,7 +3341,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) /// However, it's quite dangerous, because part may appear in source table. /// So we enqueue it for check only if no replicas of source table have part either. bool need_check = true; - if (auto * replicated_src_table = typeid_cast(source_table.get())) + if (auto * replicated_src_table = castStorage(source_table, DeferredTable::Load).get()) { String src_replica = replicated_src_table->findReplicaHavingPart(part_desc->src_part_name, false); if (!src_replica.empty()) @@ -3403,7 +3405,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) throw Exception(ErrorCodes::UNFINISHED, "Checksums of {} is suddenly changed", part_desc->src_table_part->name); /// Don't do hardlinks in case of zero-copy at any side (defensive programming) - bool source_zero_copy_enabled = (*dynamic_cast(source_table.get())->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; + bool source_zero_copy_enabled = (*castStorage(source_table, DeferredTable::Load)->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; bool our_zero_copy_enabled = (*storage_settings_ptr)[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; IDataPartStorage::ClonePartParams clone_params @@ -9203,7 +9205,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( const auto zookeeper = getZooKeeper(); const bool zero_copy_enabled = (*storage_settings_ptr)[MergeTreeSetting::allow_remote_fs_zero_copy_replication] - || (*dynamic_cast(source_table.get())->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; + || (*src_data.getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; using Entry = std::unique_ptr; std::vector entries(partitions.size()); @@ -9531,7 +9533,7 @@ std::unique_ptr StorageReplicatedMergeTree::rep void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr query_context) { auto component_guard = Coordination::setCurrentComponent("StorageReplicatedMergeTree::movePartitionToTable"); - auto dest_table_storage = std::dynamic_pointer_cast(dest_table); + auto dest_table_storage = castStorage(dest_table, DeferredTable::Load); if (!dest_table_storage) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table {} supports movePartitionToTable only for ReplicatedMergeTree family of table engines. " @@ -9656,7 +9658,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta /// Don't do hardlinks in case of zero-copy at any side (defensive programming) bool zero_copy_enabled = (*storage_settings_ptr)[MergeTreeSetting::allow_remote_fs_zero_copy_replication] - || (*dynamic_cast(dest_table.get())->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; + || (*dest_table_storage->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; IDataPartStorage::ClonePartParams clone_params { diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 70697cfcdacaa..443333d82584e 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -35,6 +35,12 @@ class StorageTableFunctionProxy final : public StorageProxy setInMemoryMetadata(cached_metadata); } + StoragePtr tryGetNested() const override + { + std::lock_guard lock{nested_mutex}; + return nested; + } + StoragePtr getNestedImpl() const { std::lock_guard lock{nested_mutex}; diff --git a/src/Storages/StorageTableProxy.h b/src/Storages/StorageTableProxy.h index 26f69e01992ac..11f5c0b1130ab 100644 --- a/src/Storages/StorageTableProxy.h +++ b/src/Storages/StorageTableProxy.h @@ -52,6 +52,14 @@ class StorageTableProxy final : public StorageProxy return IStorage::getInMemoryMetadataPtr(context_, bypass_metadata_cache); } + StoragePtr tryGetNested() const override + { + std::lock_guard lock{nested_mutex}; + return nested; + } + + bool isLazyStandIn() const override { return true; } + StoragePtr getNested() const override { std::lock_guard lock{nested_mutex}; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 5ebb3cec411e5..cd59a8c92aa71 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -2286,6 +2286,7 @@ class StorageURLSchemeDispatch final : public StorageProxy } StoragePtr getNested() const override { return nested; } + StoragePtr tryGetNested() const override { return nested; } /// The table was created with `ENGINE = URL(...)`; report it as such for consistency with /// `SHOW CREATE TABLE` and `system.tables`, even though reads/writes go to the delegate. String getName() const override { return "URL"; } @@ -2537,6 +2538,7 @@ static StoragePtr tryDispatchURLEngineByScheme(const StorageFactory::Arguments & /// `format = auto` that would force re-inference (and external I/O) on every `ATTACH`/restart. if (const auto * file = typeid_cast(delegate_storage.get())) resolved_format = file->getFormatName(); + /// NOLINT(storage-cast): the delegate is created right here, it never comes from the catalog. else if (const auto * object_storage = typeid_cast(delegate_storage.get())) resolved_format = object_storage->getFormatName(); else diff --git a/src/Storages/System/StatusRequestsPool.cpp b/src/Storages/System/StatusRequestsPool.cpp index f4d657612dd83..44402fc10eda0 100644 --- a/src/Storages/System/StatusRequestsPool.cpp +++ b/src/Storages/System/StatusRequestsPool.cpp @@ -2,17 +2,21 @@ #include #include +#include namespace DB { StoragePtr resolveStatusRequestTable(const StorageID & storage_id) { + /// The status is read by casting to the engine, so a lazily loaded table has to be resolved to + /// the storage behind its proxy. A table with no storage yet has no status to report. /// Resolution by UUID does not depend on the current table name, so it survives renames. if (storage_id.hasUUID()) - return DatabaseCatalog::instance().tryGetByUUID(storage_id.uuid).second; + return resolveStorageProxy(DatabaseCatalog::instance().tryGetByUUID(storage_id.uuid).second); - return DatabaseCatalog::instance().tryGetTable(storage_id, Context::getGlobalContextInstance()); + return resolveStorageProxy( + DatabaseCatalog::instance().tryGetTable(storage_id, Context::getGlobalContextInstance())); } DatabasePtr resolveStatusRequestDatabase(const String & database_name) diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.cpp b/src/Storages/System/StorageSystemDroppedTablesParts.cpp index 6e008205e2b40..7b45899dd5e76 100644 --- a/src/Storages/System/StorageSystemDroppedTablesParts.cpp +++ b/src/Storages/System/StorageSystemDroppedTablesParts.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -42,7 +43,7 @@ StoragesDroppedInfoStream::StoragesDroppedInfoStream(std::optional f String database_name = storage->getStorageID().getDatabaseName(); String table_name = storage->getStorageID().getTableName(); String engine_name = storage->getName(); - if (!dynamic_cast(storage.get())) + if (!castStorage(storage, DeferredTable::Skip)) continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name)) diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index 0bc4f21566213..0d079dd1fb23c 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -52,11 +53,7 @@ static StorageSystemGraphite::Configs getConfigs(ContextPtr context) for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); - if (!table) - continue; - - const MergeTreeData * table_data = dynamic_cast(table.get()); + auto table_data = castStorage(iterator->table(), DeferredTable::Skip); if (!table_data) continue; diff --git a/src/Storages/System/StorageSystemIcebergFiles.cpp b/src/Storages/System/StorageSystemIcebergFiles.cpp index c428bd784e6f3..c1ce1d7d99f4f 100644 --- a/src/Storages/System/StorageSystemIcebergFiles.cpp +++ b/src/Storages/System/StorageSystemIcebergFiles.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -221,7 +222,7 @@ class SystemIcebergFilesSource : public ISource if (!lock) return false; - auto * object_storage_table = dynamic_cast(storage.get()); + auto object_storage_table = castStorage(storage, DeferredTable::Skip); if (!object_storage_table || !object_storage_table->isIcebergStorage()) return false; diff --git a/src/Storages/System/StorageSystemIcebergHistory.cpp b/src/Storages/System/StorageSystemIcebergHistory.cpp index e9cd3b66eb289..f51cfadafc449 100644 --- a/src/Storages/System/StorageSystemIcebergHistory.cpp +++ b/src/Storages/System/StorageSystemIcebergHistory.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -202,7 +203,7 @@ void StorageSystemIcebergHistory::fillData( // Table was dropped while acquiring the lock, skipping table continue; - if (auto * object_storage_table = dynamic_cast(storage.get())) + if (auto * object_storage_table = castStorage(storage, DeferredTable::Skip).get()) { add_history_record(database_name, table_name, object_storage_table); } diff --git a/src/Storages/System/StorageSystemKafkaConsumers.cpp b/src/Storages/System/StorageSystemKafkaConsumers.cpp index 2dfdc0304e31a..04731fcb31df7 100644 --- a/src/Storages/System/StorageSystemKafkaConsumers.cpp +++ b/src/Storages/System/StorageSystemKafkaConsumers.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -276,9 +277,9 @@ void StorageSystemKafkaConsumers::fillData(MutableColumns & res_columns, Context for (auto it = db.second->getTablesIterator(context); it->isValid(); it->next()) { StoragePtr storage = it->table(); - if (auto * kafka_table = dynamic_cast(storage.get())) + if (auto kafka_table = castStorage(storage, DeferredTable::Skip)) handle_table(it, *kafka_table); - else if (auto * kafka_2_table = dynamic_cast(storage.get())) + else if (auto kafka_2_table = castStorage(storage, DeferredTable::Skip)) handle_table(it, *kafka_2_table); } } diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 08b9ac0360d1f..095673f0f43c0 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -88,13 +89,10 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr c for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); + auto table = castStorage(iterator->table(), DeferredTable::Skip); if (!table) continue; - if (!dynamic_cast(table.get())) - continue; - if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; @@ -142,6 +140,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr c std::vector statuses; { const IStorage * storage = merge_tree_tables[database][table].get(); + /// NOLINT(storage-cast): `merge_tree_tables` is filled with already resolved storages. if (const auto * merge_tree = dynamic_cast(storage)) statuses = merge_tree->getMutationsStatus(); } diff --git a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp index 0894963bc0191..ec3f099ba390c 100644 --- a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp +++ b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace DB @@ -66,7 +67,7 @@ void StorageSystemObjectStorageQueueSettings::fillData( for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { StoragePtr storage = iterator->table(); - if (auto * queue_table = dynamic_cast(storage.get())) + if (auto queue_table = castStorage(storage, DeferredTable::Skip)) { add_table(iterator, *queue_table); } diff --git a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp index b1aa1c8a263d3..47ee9ea2873a1 100644 --- a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp +++ b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -70,11 +71,9 @@ void StorageSystemPartMovesBetweenShards::fillData(MutableColumns & res_columns, for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); + auto table = castStorage(iterator->table(), DeferredTable::Skip); if (!table) continue; - if (!dynamic_cast(table.get())) - continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; replicated_tables[db.first][iterator->name()] = table; @@ -119,6 +118,7 @@ void StorageSystemPartMovesBetweenShards::fillData(MutableColumns & res_columns, String database = (*col_database_to_filter)[i].safeGet(); String table = (*col_table_to_filter)[i].safeGet(); + /// NOLINT(storage-cast): `replicated_tables` is filled with already resolved storages. auto moves = dynamic_cast(*replicated_tables[database][table]).getPartMovesBetweenShardsEntries(); for (auto & entry : moves) diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 38d2ced83ce84..6cc120847304f 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -161,7 +162,7 @@ StoragesInfoStream::StoragesInfoStream(std::optional filter_by_datab for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { String table_name = iterator->name(); - StoragePtr storage = iterator->table(); + auto storage = castStorage(iterator->table(), DeferredTable::Skip); if (!storage) continue; @@ -175,9 +176,6 @@ StoragesInfoStream::StoragesInfoStream(std::optional filter_by_datab storage_uuid = hash.get128(); } - if (!dynamic_cast(storage.get())) - continue; - if (check_access_for_tables_in_db && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name)) continue; diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index fb65818780237..382211b678649 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -2,6 +2,7 @@ #include #include +#include #include @@ -75,7 +76,7 @@ class StoragesInfoStreamBase info.engine = info.storage->getName(); - info.data = dynamic_cast(info.storage.get()); + info.data = castStorage(info.storage, DeferredTable::Skip).get(); if (!info.data) throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown engine {}", info.engine); diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 0161fbc3dcaca..73e10da25be49 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -193,12 +194,9 @@ void StorageSystemReplicas::readImpl( const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); + auto table = castStorage(iterator->table(), DeferredTable::Skip); if (!table) continue; - - if (!dynamic_cast(table.get())) - continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; replicated_tables[db.first][iterator->name()] = table; diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index 27e1660c4607d..ad26b8e612e7e 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -89,11 +90,9 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, Conte for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); + auto table = castStorage(iterator->table(), DeferredTable::Skip); if (!table) continue; - if (!dynamic_cast(table.get())) - continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; replicated_tables[db.first][iterator->name()] = table; @@ -141,6 +140,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, Conte String database = (*col_database_to_filter)[i].safeGet(); String table = (*col_table_to_filter)[i].safeGet(); + /// NOLINT(storage-cast): `replicated_tables` is filled with already resolved storages. dynamic_cast(*replicated_tables[database][table]).getQueue(queue, replica_name); for (size_t j = 0, queue_size = queue.size(); j < queue_size; ++j) diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 4e2b8faedca96..cf8a79439f11d 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -981,7 +982,7 @@ class TablesBlockSource final : public ISource ++res_index; } - auto table_merge_tree = std::dynamic_pointer_cast(table); + auto table_merge_tree = castStorage(table, DeferredTable::Skip); if (columns_mask[src_index++]) { if (table_merge_tree) diff --git a/src/Storages/Utils.cpp b/src/Storages/Utils.cpp index 7143ca6cf76da..d5922374d39c9 100644 --- a/src/Storages/Utils.cpp +++ b/src/Storages/Utils.cpp @@ -25,6 +25,7 @@ namespace DB { return {CurrentMetrics::AttachedDictionary}; } + /// NOLINT(storage-cast): runs under the database lock, and attach and detach must count a proxy alike. if (typeid_cast(storage.get()) != nullptr) { return {CurrentMetrics::AttachedTable, CurrentMetrics::AttachedReplicatedTable}; diff --git a/src/TableFunctions/TableFunctionMergeTreeIndex.cpp b/src/TableFunctions/TableFunctionMergeTreeIndex.cpp index f6b5de4fe4555..4d5b07f6ea739 100644 --- a/src/TableFunctions/TableFunctionMergeTreeIndex.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeIndex.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -150,10 +151,10 @@ static NameSet getAllPossibleStreamNames( ColumnsDescription TableFunctionMergeTreeIndex::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + auto source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(source_table_id, context)); auto metadata_snapshot = source_table->getInMemoryMetadataPtr(context, false); - const auto * merge_tree = dynamic_cast(source_table.get()); + const auto * merge_tree = castStorage(source_table, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function mergeTreeIndex expected MergeTree table, got: {}", source_table->getName()); @@ -204,7 +205,7 @@ StoragePtr TableFunctionMergeTreeIndex::executeImpl( ColumnsDescription /*cached_columns*/, bool is_insert_query) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + auto source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(source_table_id, context)); auto columns = getActualTableStructure(context, is_insert_query); StorageID storage_id(getDatabaseName(), table_name); diff --git a/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp b/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp index 29f2a251d9e12..fbd90bd9464de 100644 --- a/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -117,7 +118,7 @@ std::pair TableFunctionMergeTreeTextIndex::resolv /// Otherwise the errors below would reveal the engine and the indexes of a table the user cannot see. context->checkAccess(AccessType::SHOW_TABLES, source_database, source_table); - auto source_table_ptr = DatabaseCatalog::instance().getTable(StorageID{source_database, source_table}, context); + auto source_table_ptr = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(StorageID{source_database, source_table}, context)); auto metadata_snapshot = source_table_ptr->getInMemoryMetadataPtr(context, false); const auto & index_desc = metadata_snapshot->getSecondaryIndices().getByName(source_index_name); @@ -127,7 +128,7 @@ std::pair TableFunctionMergeTreeTextIndex::resolv "Got index '{}' of type '{}', expected 'text'", source_index_name, index_desc.type); - const auto * merge_tree = dynamic_cast(source_table_ptr.get()); + const auto * merge_tree = castStorage(source_table_ptr, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeTextIndex expected MergeTree table, got: {}", source_table_ptr->getName()); diff --git a/src/TableFunctions/TableFunctionProjection.cpp b/src/TableFunctions/TableFunctionProjection.cpp index 7b6f252b6b8b7..bcf88e1dc970b 100644 --- a/src/TableFunctions/TableFunctionProjection.cpp +++ b/src/TableFunctions/TableFunctionProjection.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -65,7 +66,7 @@ void TableFunctionMergeTreeProjection::parseArguments(const ASTPtr & ast_functio ColumnsDescription TableFunctionMergeTreeProjection::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + auto source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(source_table_id, context)); auto metadata_snapshot = source_table->getInMemoryMetadataPtr(context, false); if (!metadata_snapshot->getProjections().has(projection_name)) @@ -85,7 +86,7 @@ StoragePtr TableFunctionMergeTreeProjection::executeImpl( ColumnsDescription /*cached_columns*/, bool /* is_insert_query */) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + auto source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(source_table_id, context)); auto metadata_snapshot = source_table->getInMemoryMetadataPtr(context, false); ProjectionDescriptionRawPtr projection = &metadata_snapshot->getProjections().get(projection_name); diff --git a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference index 670c62cb8acba..0259a7a6a9049 100644 --- a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference +++ b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference @@ -30,10 +30,10 @@ -- an explicit PREWHERE through both views keeps working -- 10 10 --- re-attached tables are lazy proxies -- -lazy_inner TableProxy +-- only the re-attached MergeTree is a lazy proxy -- +lazy_inner Merge lazy_leaf TableProxy -lazy_outer TableProxy +lazy_outer Merge -- the proxy must still reject the mismatched column, not abort -- -- a matching column still supports PREWHERE through the proxy -- 10 diff --git a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql index 412183279c357..b0e1d21115efb 100644 --- a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql +++ b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql @@ -133,7 +133,7 @@ ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; -- as the style check requires (a `{...:String}` parameter is not recognized by it). USE {CLICKHOUSE_DATABASE_1:Identifier}; -SELECT '-- re-attached tables are lazy proxies --'; +SELECT '-- only the re-attached MergeTree is a lazy proxy --'; SELECT name, engine FROM system.tables WHERE database = currentDatabase() ORDER BY name; SELECT '-- the proxy must still reject the mismatched column, not abort --'; diff --git a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference index 494e4eb3e2075..1809b85a1aac3 100644 --- a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference +++ b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference @@ -1,5 +1,5 @@ t_lazy TableProxy -t_lazy_memory TableProxy +t_lazy_log TableProxy CREATE TABLE default_1.t_lazy\n(\n `key` UInt64,\n `v` Float64 STATISTICS(tdigest),\n `s` String TTL d + toIntervalMonth(1),\n `d` DateTime\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 TableProxy CREATE TABLE default_1.t_lazy\n(\n `key` UInt64,\n `v` Float64 STATISTICS(tdigest),\n `s` String TTL d + toIntervalMonth(1),\n `d` DateTime,\n `v2` Float64 STATISTICS(uniq)\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql index de3fda3e11857..80ff346f1857b 100644 --- a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql +++ b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql @@ -14,7 +14,7 @@ DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = Atomic SETTINGS lazy_load_tables = 1; CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy (key UInt64, v Float64, s String, d DateTime) ENGINE = MergeTree ORDER BY key; -CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy_memory (x UInt64) ENGINE = Memory; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy_log (x UInt64) ENGINE = Log; DETACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; @@ -38,7 +38,7 @@ SHOW CREATE TABLE t_lazy; -- The proxy forwards the nested answer rather than a blanket `true`: a lazily loaded engine without -- statistics support is still rejected. -ALTER TABLE t_lazy_memory MODIFY COLUMN x UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } -ALTER TABLE t_lazy_memory ADD COLUMN y UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_lazy_log MODIFY COLUMN x UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_lazy_log ADD COLUMN y UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference index 9f607859a6e8f..586c62c104083 100644 --- a/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference +++ b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference @@ -1,4 +1,4 @@ -TableProxy +Distributed 3 t05045_leaf 3 diff --git a/tests/queries/0_stateless/05233_lazy_load_tables_cast_storage.reference b/tests/queries/0_stateless/05233_lazy_load_tables_cast_storage.reference new file mode 100644 index 0000000000000..e780d1a59a2ea --- /dev/null +++ b/tests/queries/0_stateless/05233_lazy_load_tables_cast_storage.reference @@ -0,0 +1,15 @@ +listing skips unloaded tables and loads none of them +0 +0 +t_local 1 +t_replicated 1 +t_sync 1 +listing shows a table once it is loaded +10 +10 +1 +2 +a replica command loads the table it names +t_local 0 +t_replicated 0 +t_sync 0 diff --git a/tests/queries/0_stateless/05233_lazy_load_tables_cast_storage.sql b/tests/queries/0_stateless/05233_lazy_load_tables_cast_storage.sql new file mode 100644 index 0000000000000..86f4b69611100 --- /dev/null +++ b/tests/queries/0_stateless/05233_lazy_load_tables_cast_storage.sql @@ -0,0 +1,44 @@ +-- Tags: zookeeper, no-replicated-database +-- no-replicated-database: `DETACH DATABASE` / `ATTACH DATABASE` of an `Atomic` database +-- with the `lazy_load_tables` setting. + +-- In a database with `lazy_load_tables = 1`, an unloaded table is a `StorageTableProxy`, and a cast of +-- the catalog pointer to the engine type sees only the proxy. The system tables and the replica +-- commands resolve the proxy through `castStorage` instead: a listing skips an unloaded table and +-- shows it once it is loaded, while a command that names the table loads it. +-- https://github.com/ClickHouse/ClickHouse/issues/117420 + +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = Atomic SETTINGS lazy_load_tables = 1; + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_replicated (n UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/{table}', 'r1') ORDER BY n; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_local (n UInt64) ENGINE = MergeTree ORDER BY n; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_sync (n UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/{table}', 'r1') ORDER BY n; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.t_replicated SELECT number FROM numbers(10); +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.t_local SELECT number FROM numbers(10); + +-- Re-attach the database so the tables become unloaded lazy proxies. +DETACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; + +-- The `system.tables` filter is spelled with `currentDatabase()` because the style check only +-- recognizes that form. Reading the engine name does not load a lazy table. +USE {CLICKHOUSE_DATABASE_1:Identifier}; + +SELECT 'listing skips unloaded tables and loads none of them'; +SELECT count() FROM system.replicas WHERE database = currentDatabase(); +SELECT count() FROM system.parts WHERE database = currentDatabase(); +SELECT name, engine = 'TableProxy' AS unloaded FROM system.tables WHERE database = currentDatabase() ORDER BY name; + +SELECT 'listing shows a table once it is loaded'; +SELECT count() FROM t_replicated; +SELECT count() FROM t_local; +SELECT count() FROM system.replicas WHERE database = currentDatabase(); +SELECT count() FROM system.parts WHERE database = currentDatabase() AND active; + +SELECT 'a replica command loads the table it names'; +SYSTEM SYNC REPLICA t_sync; +SELECT name, engine = 'TableProxy' AS unloaded FROM system.tables WHERE database = currentDatabase() ORDER BY name; + +USE {CLICKHOUSE_DATABASE:Identifier}; +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; From e539df4e9dfa364709660b3337ecda17432432e6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 01:42:12 +0000 Subject: [PATCH 218/296] Backport #120931 to 26.8: Revert "Revert "Do not allocate from peer-declared sizes when reading settings from the Native protocol"" --- src/Core/BaseSettings.cpp | 2 +- src/Core/Field.cpp | 8 +- src/Core/SettingsFields.cpp | 14 +-- src/Core/tests/gtest_settings.cpp | 153 ++++++++++++++++++++++++++++++ src/IO/ReadHelpers.cpp | 5 +- src/IO/ReadHelpers.h | 5 + 6 files changed, 171 insertions(+), 16 deletions(-) diff --git a/src/Core/BaseSettings.cpp b/src/Core/BaseSettings.cpp index cbedfdb85d0f0..da6718854c747 100644 --- a/src/Core/BaseSettings.cpp +++ b/src/Core/BaseSettings.cpp @@ -28,7 +28,7 @@ void BaseSettingsHelpers::writeString(std::string_view str, WriteBuffer & out) String BaseSettingsHelpers::readString(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); return str; } diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 12ab3ae32e44f..5e9998838ce8a 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -439,7 +439,7 @@ static Field getBinaryValue(UInt8 type, ReadBuffer & buf) case Field::Types::String: { std::string value; - readStringBinary(value, buf); + readStringBinaryGrowing(value, buf); return value; } case Field::Types::Array: @@ -469,8 +469,8 @@ static Field getBinaryValue(UInt8 type, ReadBuffer & buf) case Field::Types::AggregateFunctionState: { AggregateFunctionStateData value; - readStringBinary(value.name, buf); - readStringBinary(value.data, buf); + readStringBinaryGrowing(value.name, buf); + readStringBinaryGrowing(value.data, buf); return value; } case Field::Types::Bool: @@ -597,7 +597,7 @@ void readBinary(Object & x, ReadBuffer & buf) UInt8 type = 0; String key; readBinary(type, buf); - readBinary(key, buf); + readStringBinaryGrowing(key, buf); x[key] = getBinaryValue(type, buf); } } diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 0849ead597917..001edbfeb6290 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -247,7 +247,7 @@ void SettingFieldNumber::readBinary(ReadBuffer & in) { static_assert(std::is_floating_point_v); String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = ::DB::parseFromString(str); } } @@ -461,7 +461,7 @@ void SettingFieldString::writeBinary(WriteBuffer & out) const void SettingFieldString::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = std::move(str); } @@ -544,7 +544,7 @@ void SettingFieldChar::writeBinary(WriteBuffer & out) const void SettingFieldChar::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = stringToChar(str); } @@ -557,7 +557,7 @@ void SettingFieldURI::writeBinary(WriteBuffer & out) const void SettingFieldURI::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = Poco::URI{str}; } @@ -570,7 +570,7 @@ void SettingFieldEnumHelpers::writeBinary(std::string_view str, WriteBuffer & ou String SettingFieldEnumHelpers::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); return str; } @@ -582,7 +582,7 @@ void SettingFieldTimezone::writeBinary(WriteBuffer & out) const void SettingFieldTimezone::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = std::move(str); } @@ -615,7 +615,7 @@ void SettingFieldCustom::writeBinary(WriteBuffer & out) const void SettingFieldCustom::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); parseFromString(str); } diff --git a/src/Core/tests/gtest_settings.cpp b/src/Core/tests/gtest_settings.cpp index fb18ce76337b3..7a75f2e11052f 100644 --- a/src/Core/tests/gtest_settings.cpp +++ b/src/Core/tests/gtest_settings.cpp @@ -1,18 +1,26 @@ #include +#include +#include #include +#include +#include #include #include #include #include #include #include +#include #include +#include #include +#include namespace DB::ErrorCodes { + extern const int ATTEMPT_TO_READ_AFTER_EOF; extern const int INCORRECT_DATA; } @@ -287,3 +295,148 @@ GTEST_TEST(SettingsTier, GetTierDecodesEveryEncoding) EXPECT_EQ(BaseSettingsHelpers::getTier(private_preview | Flags::CUSTOM), SettingsTierType::PRIVATE_PREVIEW); EXPECT_EQ(BaseSettingsHelpers::getTier(private_preview | Flags::HOT_RELOAD), SettingsTierType::PRIVATE_PREVIEW); } + +namespace +{ + +/// Restores the global hard limit and the operator-new throw threshold on scope exit. +struct MemoryLimitGuard +{ + Int64 prev_hard_limit; + MemoryLimitGuard() : prev_hard_limit(total_memory_tracker.getHardLimit()) {} + ~MemoryLimitGuard() + { + CurrentMemoryTracker::setMinAllocationSizeBytesToThrow(0); + total_memory_tracker.setHardLimit(prev_hard_limit); + } +}; + +/// Makes the next allocation of a megabyte or more overshoot the global limit, while smaller +/// allocations (the exception being built, the settings object's own bookkeeping) still succeed. +void armMemoryLimitOnAllocationsOfAtLeastAMegabyte() +{ + MainThreadStatus::getInstance(); + CurrentThread::flushUntrackedMemory(); + CurrentMemoryTracker::setMinAllocationSizeBytesToThrow(1ULL << 20); + total_memory_tracker.setHardLimit(total_memory_tracker.get() + 1024); +} + +} + +GTEST_TEST(Settings, ADeclaredLengthOnTheSettingsWireIsNotAnAllocation) +{ + /// Each entry declares a large string length on the settings wire and then sends no payload, so + /// a read that grows with the bytes that actually arrive runs out of buffer immediately, while + /// one that sizes its destination from the declared length allocates before it can fail. + constexpr size_t declared_size = 64 * 1024 * 1024; + + /// `writeBinary(const Map &)` writes a fixed width element count, then one tagged field per + /// element, and puts no type constraint on an element. + const auto write_map_element_tag = [](WriteBuffer & out, Field::Types::Which field_type) + { + BaseSettingsHelpers::writeString("additional_table_filters", out); + writeBinary(static_cast(1), out); + writeBinary(static_cast(field_type), out); + }; + + struct Case + { + std::string_view covers; + SettingsWriteFormat format; + std::function write_wire; + }; + + const std::vector cases = { + {"the setting name, i.e. every BaseSettingsHelpers::readString carrier", + SettingsWriteFormat::STRINGS_WITH_FLAGS, + [&](WriteBuffer & out) { writeVarUInt(declared_size, out); }}, + {"SettingFieldString::readBinary", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + BaseSettingsHelpers::writeString("log_comment", out); + writeVarUInt(declared_size, out); + }}, + {"SettingFieldNumber::readBinary", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + BaseSettingsHelpers::writeString("totals_auto_threshold", out); + writeVarUInt(declared_size, out); + }}, + {"a String inside a Map setting", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + write_map_element_tag(out, Field::Types::String); + writeVarUInt(declared_size, out); + }}, + {"an AggregateFunctionState name inside a Map setting", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + write_map_element_tag(out, Field::Types::AggregateFunctionState); + writeVarUInt(declared_size, out); + }}, + {"an Object key inside a Map setting", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + write_map_element_tag(out, Field::Types::Object); + writeBinary(static_cast(1), out); + writeBinary(static_cast(Field::Types::String), out); + writeVarUInt(declared_size, out); + }}, + }; + + for (const auto & test_case : cases) + { + SCOPED_TRACE(test_case.covers); + + /// Build the wire and the settings object before clamping, so only the read runs clamped. + WriteBufferFromOwnString out; + test_case.write_wire(out); + const std::string wire = out.str(); + Settings settings; + ReadBufferFromString in(wire); + + try + { + MemoryLimitGuard guard; + armMemoryLimitOnAllocationsOfAtLeastAMegabyte(); + settings.read(in, test_case.format); + ADD_FAILURE() << "reading a declared length of " << declared_size << " did not throw"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF) << e.displayText(); + } + } +} + +GTEST_TEST(Settings, WellFormedSettingsStillReadUnderAClampedMemoryLimit) +{ + /// The in-range control: the clamp itself does not make a read fail, and the empty string end + /// marker still terminates the loop. + Settings sent; + sent.set("log_comment", "hello"); + sent.set("max_block_size", UInt64(4096)); + sent.set("totals_auto_threshold", 0.25); + + WriteBufferFromOwnString out; + sent.write(out, SettingsWriteFormat::STRINGS_WITH_FLAGS); + const std::string wire = out.str(); + + Settings settings; + ReadBufferFromString in(wire); + { + MemoryLimitGuard guard; + armMemoryLimitOnAllocationsOfAtLeastAMegabyte(); + ASSERT_NO_THROW(settings.read(in, SettingsWriteFormat::STRINGS_WITH_FLAGS)); + } + + ASSERT_EQ(settings.get("log_comment"), Field(String("hello"))); + ASSERT_EQ(settings.get("max_block_size"), Field(UInt64(4096))); + ASSERT_EQ(settings.get("totals_auto_threshold"), Field(0.25)); + ASSERT_TRUE(in.eof()); +} diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 34637470a0034..f347a259cc881 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -2071,10 +2071,7 @@ bool trySkipJSONField(ReadBuffer & buf, std::string_view name_of_field, const Fo } -/// The same as `readStringBinary`, but the string grows as the bytes arrive instead of being resized -/// to the declared size first, so that a size declared by the peer cannot become an allocation on -/// its own when the payload never follows. -static void readStringBinaryGrowing(String & s, ReadBuffer & buf, size_t max_string_size = DEFAULT_MAX_STRING_SIZE) +void readStringBinaryGrowing(String & s, ReadBuffer & buf, size_t max_string_size) { size_t size = 0; readVarUInt(size, buf); diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 173ebc3f9d6f7..755861fa88c38 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -171,6 +171,11 @@ inline void skipStringBinary(ReadBuffer & buf) buf.ignore(size); } +/// The same as `readStringBinary`, but the string grows as the bytes arrive instead of being resized +/// to the declared size first, so that a size declared by the peer cannot become an allocation on +/// its own when the payload never follows. +void readStringBinaryGrowing(String & s, ReadBuffer & buf, size_t max_string_size = DEFAULT_MAX_STRING_SIZE); + /// For historical reasons we store IPv6 as a String inline void readIPv6Binary(IPv6 & ip, ReadBuffer & buf) { From 69053fa9f8375839e365d12aae461f1ba7137c96 Mon Sep 17 00:00:00 2001 From: "Nihal Z. Miaji" Date: Thu, 8 Oct 2026 11:42:23 +0800 Subject: [PATCH 219/296] Update 04366_filter_push_down_limit_by.reference --- .../0_stateless/04366_filter_push_down_limit_by.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference b/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference index 30402fed27e44..fa1f28c186664 100644 --- a/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference +++ b/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference @@ -2,7 +2,7 @@ 0 1 0 -1 +0 0 0 0 From 904c5629453c232306b70be03c900cab7ea06949 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 07:19:56 +0000 Subject: [PATCH 220/296] Backport #123900 to 26.8: Faster short `LowCardinality` range inserts from a different dictionary --- src/Columns/ColumnLowCardinality.cpp | 20 +++ src/Columns/ColumnUnique.h | 78 +++++++-- src/Columns/IColumnUnique.h | 3 + src/Columns/tests/gtest_low_cardinality.cpp | 159 ++++++++++++++++++ ...hash_join_low_cardinality_many_matches.xml | 11 ++ 5 files changed, 254 insertions(+), 17 deletions(-) create mode 100644 tests/performance/hash_join_low_cardinality_many_matches.xml diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index d927a82bad64d..b40c289a53c69 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; extern const int INCORRECT_DATA; + extern const int PARAMETER_OUT_OF_BOUND; } void throwUnexpectedLowCardinalityIndexType(size_t size) @@ -30,6 +31,9 @@ void throwUnexpectedLowCardinalityIndexType(size_t size) namespace { + /// The range path starts to win around 64 rows when there are only a few distinct short keys. + constexpr size_t max_rows_to_translate_individually = 64; + void checkColumn(const IColumn & column) { if (!dynamic_cast(&column)) @@ -214,6 +218,22 @@ void ColumnLowCardinality::doInsertRangeFrom(const IColumn & src, size_t start, { compactIfSharedDictionary(); + if (length <= max_rows_to_translate_individually) + { + const IColumn & src_indexes = low_cardinality_src->getIndexes(); + if (start > src_indexes.size() || length > src_indexes.size() - start) + throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Parameters start = {}, length = {} are out of bound in " + "ColumnLowCardinality::insertRangeFrom method (size() = {}).", start, length, src_indexes.size()); + + std::array positions; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) - only the first `length` entries are written before read + for (size_t i = 0; i < length; ++i) + positions[i] = src_indexes.getUInt(start + i); + getDictionary().uniqueInsertRowsFrom(*low_cardinality_src->getDictionary().getNestedColumn(), {positions.data(), length}); + for (size_t i = 0; i < length; ++i) + idx.insertIndex(positions[i]); + return; + } + /// TODO: Support native insertion from other unique column. It will help to avoid null map creation. auto sub_idx = IColumn::mutate(low_cardinality_src->getIndexes().cut(start, length)); diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index c7234f3fb12c0..447a52cdbe0c9 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -63,6 +63,7 @@ class ColumnUnique final : public COWHelper rows) override; MutableColumnPtr uniqueInsertRangeFrom(const IColumn & src, size_t start, size_t length) override; IColumnUnique::IndexesWithOverflow uniqueInsertRangeWithOverflow(const IColumn & src, size_t start, size_t length, size_t max_dictionary_size) override; @@ -229,6 +230,12 @@ class ColumnUnique final : public COWHelper(column_holder.get()); } const ColumnType * getRawColumnPtr() const { return assert_cast(column_holder.get()); } + /// `src` must be `ColumnType` or `Nullable(ColumnType)`. + std::pair getValuesAndNullMap(const IColumn & src) const; + + /// The NULL and default value rules of `uniqueInsertRangeFrom`. Returns nothing for any other value. + ALWAYS_INLINE std::optional getSpecialValueIndex(const ColumnType & src, const NullMap * null_map, size_t row) const; + template MutableColumnPtr uniqueInsertRangeImpl( const IColumn & src, @@ -444,6 +451,57 @@ size_t ColumnUnique::uniqueInsertFrom(const IColumn & src, size_t n) return uniqueInsertData(ref.data(), ref.size()); } +template +void ColumnUnique::uniqueInsertRowsFrom(const IColumn & src, std::span rows) +{ + const auto [src_column, null_map] = getValuesAndNullMap(src); + UInt64 previous_row = std::numeric_limits::max(); + UInt64 previous_index = 0; + for (UInt64 & row : rows) + { + if (row != previous_row) + { + previous_row = row; + if (auto special_index = getSpecialValueIndex(*src_column, null_map, row)) + previous_index = *special_index; + else + previous_index = uniqueInsertFrom(*src_column, row); + } + row = previous_index; + } +} + +template +std::pair ColumnUnique::getValuesAndNullMap(const IColumn & src) const +{ + const ColumnType * src_column = nullptr; + const NullMap * null_map = nullptr; + + if (const auto * nullable_column = checkAndGetColumn(&src)) + { + src_column = typeid_cast(&nullable_column->getNestedColumn()); + null_map = &nullable_column->getNullMapData(); + } + else + src_column = typeid_cast(&src); + + if (src_column == nullptr) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Invalid column type for ColumnUnique::insertRangeFrom. " + "Expected {}, got {}", column_holder->getName(), src.getName()); + + return {src_column, null_map}; +} + +template +std::optional ColumnUnique::getSpecialValueIndex(const ColumnType & src, const NullMap * null_map, size_t row) const +{ + if (null_map && (*null_map)[row]) + return getNullValueIndex(); + if (getRawColumnPtr()->compareAt(getNestedTypeDefaultValueIndex(), row, src, 1) == 0) + return getNestedTypeDefaultValueIndex(); + return std::nullopt; +} + template size_t ColumnUnique::uniqueInsertData(const char * pos, size_t length) { @@ -657,8 +715,6 @@ MutableColumnPtr ColumnUnique::uniqueInsertRangeImpl( ReverseIndex * secondary_index, size_t max_dictionary_size) { - const ColumnType * src_column = nullptr; - const NullMap * null_map = nullptr; auto & positions = positions_column->getData(); auto update_position = [&](UInt64 & next_position) -> MutableColumnPtr @@ -692,17 +748,7 @@ MutableColumnPtr ColumnUnique::uniqueInsertRangeImpl( return nullptr; }; - if (const auto * nullable_column = checkAndGetColumn(&src)) - { - src_column = typeid_cast(&nullable_column->getNestedColumn()); - null_map = &nullable_column->getNullMapData(); - } - else - src_column = typeid_cast(&src); - - if (src_column == nullptr) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Invalid column type for ColumnUnique::insertRangeFrom. " - "Expected {}, got {}", column_holder->getName(), src.getName()); + const auto [src_column, null_map] = getValuesAndNullMap(src); auto column = getRawColumnPtr(); @@ -724,10 +770,8 @@ MutableColumnPtr ColumnUnique::uniqueInsertRangeImpl( { auto row = start + num_added_rows; - if (null_map && (*null_map)[row]) - positions[num_added_rows] = static_cast(getNullValueIndex()); - else if (column->compareAt(getNestedTypeDefaultValueIndex(), row, *src_column, 1) == 0) - positions[num_added_rows] = static_cast(getNestedTypeDefaultValueIndex()); + if (auto special_index = getSpecialValueIndex(*src_column, null_map, row)) + positions[num_added_rows] = static_cast(*special_index); else { auto ref = src_column->getDataAt(row); diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h index 31baf6a282a4b..42a35c50202f9 100644 --- a/src/Columns/IColumnUnique.h +++ b/src/Columns/IColumnUnique.h @@ -44,6 +44,9 @@ class IColumnUnique : public IColumn virtual bool tryUniqueInsert(const Field & x, size_t & index) = 0; virtual size_t uniqueInsertFrom(const IColumn & src, size_t n) = 0; + /// Inserts rows `rows` of `src` with the rules of `uniqueInsertRangeFrom`, which differ from `uniqueInsertFrom` for NULL + /// and for values that compare equal to the default, such as `-0.0`, and replaces each row number with its index. + virtual void uniqueInsertRowsFrom(const IColumn & src, std::span rows) = 0; /// Appends range of elements from other column. /// Could be used to concatenate columns. virtual MutableColumnPtr uniqueInsertRangeFrom(const IColumn & src, size_t start, size_t length) = 0; diff --git a/src/Columns/tests/gtest_low_cardinality.cpp b/src/Columns/tests/gtest_low_cardinality.cpp index 7656d4fc96769..5ece835fbc1dc 100644 --- a/src/Columns/tests/gtest_low_cardinality.cpp +++ b/src/Columns/tests/gtest_low_cardinality.cpp @@ -3,9 +3,16 @@ #include #include +#include +#include #include +#include +#include +#include +#include + using namespace DB; template @@ -111,3 +118,155 @@ TEST(ColumnLowCardinality, EmptyDictionaryEmptyIndexes) ASSERT_EQ(column->size(), 0); } + +namespace +{ + +MutableColumnPtr makeLowCardinality(const DataTypePtr & type, const std::vector & values) +{ + auto column = std::make_shared(type)->createColumn(); + for (const auto & value : values) + column->insert(value); + return column; +} + +/// 1000 rows drawn from `keys`, with runs of repeated values. +std::vector makeSourceValues(const std::vector & keys, UInt64 seed) +{ + pcg64 rng(seed); + std::vector values; + while (values.size() < 1000) + { + const Field & key = keys[rng() % keys.size()]; + for (size_t repeat = 1 + rng() % 3; repeat > 0 && values.size() < 1000; --repeat) + values.push_back(key); + } + return values; +} + +/// A range short enough for the per-row path must give the same rows and the same dictionary prefix +/// as a range of the same rows that is long enough for the range path. +void checkShortRangeMatchesRangePath(const IColumn & source, const std::function & make_destination) +{ + const auto & source_lc = assert_cast(source); + const auto & source_keys = *source_lc.getDictionary().getNestedNotNullableColumn(); + constexpr size_t long_length = 600; + for (size_t start : {0, 1, 7, 300}) + { + for (size_t length = 1; length <= 256; ++length) + { + auto short_range = make_destination(); + auto long_range = make_destination(); + const size_t offset = short_range->size(); + short_range->insertRangeFrom(source, start, length); + long_range->insertRangeFrom(source, start, long_length); + + const auto & short_lc = assert_cast(*short_range); + const auto & long_lc = assert_cast(*long_range); + ASSERT_EQ(short_range->size(), offset + length); + for (size_t row = offset; row < offset + length; ++row) + { + SCOPED_TRACE(fmt::format("start {}, length {}, row {}", start, length, row)); + ASSERT_EQ(short_lc.getIndexes().getUInt(row), long_lc.getIndexes().getUInt(row)); + ASSERT_EQ(short_lc.isNullAt(row), long_lc.isNullAt(row)); + if (!short_lc.isNullAt(row)) + ASSERT_EQ(short_lc.getDataAt(row), long_lc.getDataAt(row)); + + /// Both paths apply the same NULL and default rules, so check those against the source too. + const size_t source_row = start + row - offset; + const auto & dictionary = short_lc.getDictionary(); + const size_t default_index = dictionary.getNestedTypeDefaultValueIndex(); + if (source.isNullAt(source_row)) + ASSERT_EQ(short_lc.getIndexes().getUInt(row), dictionary.getNullValueIndex()); + else if (dictionary.getNestedNotNullableColumn()->compareAt( + default_index, source_lc.getIndexes().getUInt(source_row), source_keys, 1) == 0) + ASSERT_EQ(short_lc.getIndexes().getUInt(row), default_index); + } + + const auto & short_keys = *short_lc.getDictionary().getNestedNotNullableColumn(); + const auto & long_keys = *long_lc.getDictionary().getNestedNotNullableColumn(); + ASSERT_LE(short_keys.size(), long_keys.size()); + for (size_t key = 0; key < short_keys.size(); ++key) + ASSERT_EQ(short_keys.getDataAt(key), long_keys.getDataAt(key)) << "start " << start << ", length " << length; + } + } +} + +} + +TEST(ColumnLowCardinality, ShortRangeFromDifferentDictionaryMatchesRangePath) +{ + const auto string_type = std::make_shared(); + const auto nullable_string_type = std::make_shared(string_type); + + std::vector string_keys{Field(""), Field("a"), Field("b"), Field("c"), Field("d"), Field("e"), Field(std::string(40, 'f'))}; + auto string_source = makeLowCardinality(string_type, makeSourceValues(string_keys, 1)); + auto make_string_destination = [&] { return makeLowCardinality(string_type, {Field("b"), Field("x"), Field(""), Field("a"), Field("y")}); }; + + { + SCOPED_TRACE("LowCardinality(String)"); + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*string_source, make_string_destination)); + } + { + SCOPED_TRACE("LowCardinality(Nullable(String))"); + std::vector nullable_keys = string_keys; + nullable_keys.push_back(Field()); + auto source = makeLowCardinality(nullable_string_type, makeSourceValues(nullable_keys, 2)); + auto make_destination = [&] { return makeLowCardinality(nullable_string_type, {Field("b"), Field(), Field("x")}); }; + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*source, make_destination)); + } + { + SCOPED_TRACE("LowCardinality(String) into LowCardinality(Nullable(String))"); + auto make_destination = [&] { return makeLowCardinality(nullable_string_type, {Field("b"), Field(), Field("x")}); }; + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*string_source, make_destination)); + } + { + SCOPED_TRACE("Shared source dictionary"); + const auto & string_source_lc = assert_cast(*string_source); + auto source = ColumnLowCardinality::create(string_source_lc.getDictionaryPtr(), string_source_lc.getIndexesPtr(), /* is_shared = */ true); + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*source, make_string_destination)); + } + { + /// The keys are set directly, so that the dictionary holds `-0.0` next to `0.0` and two NaN payloads. + SCOPED_TRACE("LowCardinality(Float64)"); + auto keys = ColumnFloat64::create(); + for (Float64 key : {0.0, -0.0, 1.5, std::bit_cast(0x7ff8000000000001ULL), std::bit_cast(0xfff8000000000002ULL)}) + keys->insertValue(key); + ColumnPtr dictionary = DataTypeLowCardinality::createColumnUnique(DataTypeFloat64(), std::move(keys)); + auto indexes = ColumnUInt8::create(); + pcg64 rng(3); + for (size_t row = 0; row < 1000; ++row) + indexes->insertValue(static_cast(rng() % 5)); + auto source = ColumnLowCardinality::create(dictionary, std::move(indexes), /* is_shared = */ false); + auto make_destination = [&] { return makeLowCardinality(std::make_shared(), {Field(2.5), Field(1.5)}); }; + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*source, make_destination)); + } + { + SCOPED_TRACE("LowCardinality(UInt64), index type grows"); + const auto uint64_type = std::make_shared(); + std::vector keys; + for (UInt64 key = 0; key < 300; ++key) + keys.push_back(Field(key * 10)); + auto source = makeLowCardinality(uint64_type, makeSourceValues(keys, 4)); + std::vector destination_values; + for (UInt64 value = 1; value <= 250; ++value) + destination_values.push_back(Field(value * 7)); + auto make_destination = [&] { return makeLowCardinality(uint64_type, destination_values); }; + ASSERT_EQ(assert_cast(*make_destination()).getSizeOfIndexType(), sizeof(UInt8)); + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*source, make_destination)); + + auto destination = make_destination(); + destination->insertRangeFrom(*source, 0, 32); + ASSERT_EQ(assert_cast(*destination).getSizeOfIndexType(), sizeof(UInt16)); + } + { + SCOPED_TRACE("Out of bound range"); + auto destination = make_string_destination(); + const size_t size_before = destination->size(); + const size_t dictionary_size_before = assert_cast(*destination).getDictionary().size(); + EXPECT_THROW(destination->insertRangeFrom(*string_source, string_source->size() - 1, 2), Exception); + EXPECT_THROW(destination->insertRangeFrom(*string_source, std::numeric_limits::max(), 1), Exception); + EXPECT_EQ(destination->size(), size_before); + EXPECT_EQ(assert_cast(*destination).getDictionary().size(), dictionary_size_before); + } +} diff --git a/tests/performance/hash_join_low_cardinality_many_matches.xml b/tests/performance/hash_join_low_cardinality_many_matches.xml new file mode 100644 index 0000000000000..8c6d625176024 --- /dev/null +++ b/tests/performance/hash_join_low_cardinality_many_matches.xml @@ -0,0 +1,11 @@ + + + + 8 + 0 + + + SELECT count() FROM (SELECT number % 100 AS k FROM numbers(4000)) AS a INNER JOIN (SELECT intDiv(number, 4000) AS k, toLowCardinality(toString(number % 7)) AS lc FROM numbers(400000)) AS b ON a.k = b.k WHERE NOT ignore(b.lc) SETTINGS join_algorithm = 'hash' + SELECT count() FROM (SELECT number % 100 AS k FROM numbers(4000)) AS a INNER JOIN (SELECT intDiv(number, 4000) AS k, toLowCardinality(toString(number % 7)) AS lc FROM numbers(400000)) AS b ON a.k = b.k WHERE NOT ignore(b.lc) SETTINGS join_algorithm = 'parallel_hash' + From 3786c6e7391b740450317076a0178beed0f2a148 Mon Sep 17 00:00:00 2001 From: "Nihal Z. Miaji" Date: Thu, 8 Oct 2026 09:36:17 +0000 Subject: [PATCH 221/296] Size filtered `DISTINCT` chunks by the number of selected rows --- src/Processors/Transforms/DistinctTransform.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Transforms/DistinctTransform.cpp b/src/Processors/Transforms/DistinctTransform.cpp index 5d693a474108f..8ac959c1a3380 100644 --- a/src/Processors/Transforms/DistinctTransform.cpp +++ b/src/Processors/Transforms/DistinctTransform.cpp @@ -394,7 +394,7 @@ void DistinctTransform::transform(Chunk & chunk) else { for (auto & column : columns) - column = column->filter(filter, -1); + column = column->filter(filter, num_selected); chunk.setColumns(std::move(columns), num_selected); } From c95d078aee85b749d9d6033bf581536bfc4ce6c6 Mon Sep 17 00:00:00 2001 From: Vladimir Cherkasov Date: Thu, 8 Oct 2026 12:56:14 +0200 Subject: [PATCH 222/296] fix test --- .../05057_explain_hide_secrets_without_privilege.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.reference b/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.reference index 16088c81eacda..d068effff2986 100644 --- a/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.reference +++ b/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.reference @@ -1,4 +1,4 @@ -Output: hex(encrypt(\'aes-128-ecb\', secret, [HIDDEN])) +Output: hex(encrypt(\'aes-128-ecb\', [HIDDEN], [HIDDEN])) ReadFromMemoryStorage Output: secret From f229a6339c22d4e89f12313d79f573a6e3c2d9e2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 13:04:32 +0000 Subject: [PATCH 223/296] Backport #124220 to 26.8: Use skip indexes through a `CAST` that drops `Nullable` --- .../optimizeDirectReadFromTextIndex.cpp | 10 +++- src/Storages/MergeTree/RPNBuilder.cpp | 37 ++++++------ src/Storages/MergeTree/RPNBuilder.h | 10 ++-- ...ap_lowcardinality_subcolumn_cast.reference | 8 ++- ...ndex_map_lowcardinality_subcolumn_cast.sql | 11 ++-- ...bloom_filter_json_subcolumn_cast.reference | 2 + ...05317_bloom_filter_json_subcolumn_cast.sql | 4 +- ...ossless_conversion_drop_nullable.reference | 23 ++++++++ ...ndex_lossless_conversion_drop_nullable.sql | 59 +++++++++++++++++++ 9 files changed, 130 insertions(+), 34 deletions(-) create mode 100644 tests/queries/0_stateless/05338_skip_index_lossless_conversion_drop_nullable.reference create mode 100644 tests/queries/0_stateless/05338_skip_index_lossless_conversion_drop_nullable.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp index 765a0ffde1dca..617fd50849313 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp @@ -572,6 +572,14 @@ class TextIndexDAGReplacer ActionsDAGWithInversionPushDown canonical_dag(&function_node, context, /* boolean_context */ false); const auto & canonical_node = canonical_dag.predicate ? *canonical_dag.predicate : function_node; + /// The index is analyzed under a `CAST` that drops `Nullable` and throws on NULL. Direct read replaces or + /// short-circuits the predicate, so `NOT hasToken(CAST(s, 'String'), 'a')` would return the NULL row + /// instead of throwing. Use the index only to skip granules then. + const bool drops_nullable = std::ranges::any_of(canonical_node.children, [](const auto * argument) + { + return unwrapLosslessConversion(argument, /*allow_drop_nullable=*/ false) != unwrapLosslessConversion(argument); + }); + NameSet used_index_columns; std::vector selected_conditions; @@ -596,7 +604,7 @@ class TextIndexDAGReplacer /// Use direct read only when enabled and the entry is direct-read-eligible (has `index`) and has no /// patched parts. Otherwise just inject the tokenizer/preprocessor/postprocessor (no virtual column), /// same as None mode. - if (!direct_read_from_text_index || !info.index || info.has_patched_parts + if (!direct_read_from_text_index || !info.index || info.has_patched_parts || drops_nullable || search_query->getDirectReadMode() == TextIndexDirectReadMode::None) { selected_conditions.emplace_back(search_query, index_name, String{}, &info, is_index_analyzed); diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 8580eedffbfc7..7153007436ded 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -585,34 +585,31 @@ RPNBuilderTreeNode RPNBuilderFunctionTreeNode::getArgumentAt(size_t index) const namespace { -/// Whether converting a value of type `from` to type `to` never changes it and never throws. -/// `Nullable` cannot be dropped, because it may throw on NULL. -bool isLosslessConversion(const DataTypePtr & from, const DataTypePtr & to) +/// Whether converting a value of type `from` to type `to` never changes a non-NULL value. +/// Dropping `Nullable` throws on NULL. Such a query may not throw if the index skips the granules +/// with NULL, which is accepted. +bool isLosslessConversion(const DataTypePtr & from, const DataTypePtr & to, bool allow_drop_nullable) { auto from_type = removeLowCardinality(from); auto to_type = removeLowCardinality(to); - if (to_type->isNullable()) - { - from_type = removeNullable(from_type); - to_type = removeNullable(to_type); - } - else if (from_type->isNullable()) - { + if (!allow_drop_nullable && from_type->isNullable() && !to_type->isNullable()) return false; - } + + from_type = removeNullable(from_type); + to_type = removeNullable(to_type); if (from_type->equals(*to_type)) return true; const auto * from_array = typeid_cast(from_type.get()); const auto * to_array = typeid_cast(to_type.get()); - return from_array && to_array && isLosslessConversion(from_array->getNestedType(), to_array->getNestedType()); + return from_array && to_array && isLosslessConversion(from_array->getNestedType(), to_array->getNestedType(), allow_drop_nullable); } } -bool isLosslessConversionFunction(const ActionsDAG::Node & node) +bool isLosslessConversionFunction(const ActionsDAG::Node & node, bool allow_drop_nullable) { if (node.type != ActionsDAG::ActionType::FUNCTION || !node.function_base) return false; @@ -626,10 +623,10 @@ bool isLosslessConversionFunction(const ActionsDAG::Node & node) if (!is_cast && !is_wrapper) return false; - return isLosslessConversion(node.children.front()->result_type, node.result_type); + return isLosslessConversion(node.children.front()->result_type, node.result_type, allow_drop_nullable); } -RPNBuilderTreeNode unwrapLosslessConversion(const RPNBuilderTreeNode & node) +RPNBuilderTreeNode unwrapLosslessConversion(const RPNBuilderTreeNode & node, bool allow_drop_nullable) { if (!node.isFunction()) return node; @@ -637,20 +634,20 @@ RPNBuilderTreeNode unwrapLosslessConversion(const RPNBuilderTreeNode & node) const auto function = node.toFunctionNode(); const auto * function_dag_node = function.getDAGNode(); - if (!function_dag_node || !isLosslessConversionFunction(*function_dag_node)) + if (!function_dag_node || !isLosslessConversionFunction(*function_dag_node, allow_drop_nullable)) return node; - return unwrapLosslessConversion(function.getArgumentAt(0)); + return unwrapLosslessConversion(function.getArgumentAt(0), allow_drop_nullable); } -const ActionsDAG::Node * unwrapLosslessConversion(const ActionsDAG::Node * node) +const ActionsDAG::Node * unwrapLosslessConversion(const ActionsDAG::Node * node, bool allow_drop_nullable) { const auto * node_without_alias = getNodeWithoutAlias(node); - if (!isLosslessConversionFunction(*node_without_alias)) + if (!isLosslessConversionFunction(*node_without_alias, allow_drop_nullable)) return node; - return unwrapLosslessConversion(node_without_alias->children.front()); + return unwrapLosslessConversion(node_without_alias->children.front(), allow_drop_nullable); } template diff --git a/src/Storages/MergeTree/RPNBuilder.h b/src/Storages/MergeTree/RPNBuilder.h index 04d502c5286ea..9b8227a6b3487 100644 --- a/src/Storages/MergeTree/RPNBuilder.h +++ b/src/Storages/MergeTree/RPNBuilder.h @@ -169,13 +169,15 @@ class RPNBuilderFunctionTreeNode : public RPNBuilderTreeNode }; /// Whether the node is `CAST`, `_CAST`, `toNullable` or `toLowCardinality` whose conversion never changes -/// the value and never throws: `LowCardinality` added or dropped and `Nullable` added, at any depth of `Array`. -bool isLosslessConversionFunction(const ActionsDAG::Node & node); +/// a non-NULL value: `LowCardinality` or `Nullable` added or dropped, at any depth of `Array`. +/// Dropping `Nullable` throws on NULL, but an index may skip such granules and the query will not throw. +/// Pass `allow_drop_nullable = false` when the predicate is replaced and must still throw, e.g. by direct read from a text index. +bool isLosslessConversionFunction(const ActionsDAG::Node & node, bool allow_drop_nullable = true); /// Strips lossless conversions (see above) from the node. Indexes are analyzed on the expression under them. /// The AST form carries no types and is returned as is. -RPNBuilderTreeNode unwrapLosslessConversion(const RPNBuilderTreeNode & node); -const ActionsDAG::Node * unwrapLosslessConversion(const ActionsDAG::Node * node); +RPNBuilderTreeNode unwrapLosslessConversion(const RPNBuilderTreeNode & node, bool allow_drop_nullable = true); +const ActionsDAG::Node * unwrapLosslessConversion(const ActionsDAG::Node * node, bool allow_drop_nullable = true); /** RPN Builder build stack of reverse polish notation elements (RPNElements) required for index analysis. * diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference index 97d96022f0203..dbf3e42ddd848 100644 --- a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference @@ -65,8 +65,12 @@ idx 1 idx 2 scan 1 scan 2 --- Nullable column: dropping Nullable is not looked through, the cast still throws on the NULL row -0 +-- Nullable column: dropping Nullable is looked through, the cast does not throw if the granule with NULL is skipped +Granules: 2/2 +Name: idx +Granules: 1/2 +idx 1 +idx 2 -- Array column: a cast that only wraps the elements is looked through Array(Nullable) Granules: 2/2 Array(Nullable) Name: idx diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql index 9c2f4b4409a6e..9a5833c380f14 100644 --- a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql @@ -4,8 +4,8 @@ -- `arrayElement` on a `Map(K, LowCardinality(V))` returns `V`, while the subcolumn `m.key_` is `LowCardinality(V)`. -- Hence `optimize_functions_to_subcolumns` rewrites `m['key'] = 'value'` into `_CAST(m.key_, 'V') = 'value'`, -- and the text index has to look through the cast: for an index on `mapValues(m)`. --- The same applies to every conversion that cannot change a value or throw: adding or dropping `LowCardinality`, --- adding `Nullable` (also as `toNullable`, which `join_use_nulls` emits in pushed-down filters), at any depth of `Array`. +-- The same applies to every conversion that does not change a non-NULL value: adding or dropping `LowCardinality` or +-- `Nullable` (adding also as `toNullable`, which `join_use_nulls` emits in pushed-down filters), at any depth of `Array`. SET explain_query_plan_default = 'legacy'; SET enable_analyzer = 1; @@ -127,9 +127,10 @@ SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE h SELECT 'idx', id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; SELECT 'scan', id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; -SELECT '-- Nullable column: dropping Nullable is not looked through, the cast still throws on the NULL row'; -SELECT count() FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'network')) WHERE explain LIKE '%Name: idx%'; -SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'network'); -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT '-- Nullable column: dropping Nullable is looked through, the cast does not throw if the granule with NULL is skipped'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } DROP TABLE tab_ns; diff --git a/tests/queries/0_stateless/05317_bloom_filter_json_subcolumn_cast.reference b/tests/queries/0_stateless/05317_bloom_filter_json_subcolumn_cast.reference index c92c37c608996..bbc3c38b2cb65 100644 --- a/tests/queries/0_stateless/05317_bloom_filter_json_subcolumn_cast.reference +++ b/tests/queries/0_stateless/05317_bloom_filter_json_subcolumn_cast.reference @@ -15,4 +15,6 @@ Granules: 1/2 1 -- Nullable(String) path, cast to String Granules: 2/2 +Name: index_document_typename +Granules: 1/2 1 diff --git a/tests/queries/0_stateless/05317_bloom_filter_json_subcolumn_cast.sql b/tests/queries/0_stateless/05317_bloom_filter_json_subcolumn_cast.sql index 81ac466c33abf..62a67f2bbf2d0 100644 --- a/tests/queries/0_stateless/05317_bloom_filter_json_subcolumn_cast.sql +++ b/tests/queries/0_stateless/05317_bloom_filter_json_subcolumn_cast.sql @@ -1,5 +1,5 @@ --- A `bloom_filter` index on a typed JSON path is used when the path is cast to `Nullable`, which cannot change a --- value or throw. A cast that drops `Nullable` throws on NULL, so the index is not used for it. +-- A `bloom_filter` index on a typed JSON path is used when the path is cast to or from `Nullable`, which does not +-- change a non-NULL value. SET explain_query_plan_default = 'legacy'; SET enable_analyzer = 1; diff --git a/tests/queries/0_stateless/05338_skip_index_lossless_conversion_drop_nullable.reference b/tests/queries/0_stateless/05338_skip_index_lossless_conversion_drop_nullable.reference new file mode 100644 index 0000000000000..ebc2ae5c57bb0 --- /dev/null +++ b/tests/queries/0_stateless/05338_skip_index_lossless_conversion_drop_nullable.reference @@ -0,0 +1,23 @@ +-- bloom_filter +Granules: 4/4 +Name: idx_bf +Granules: 1/4 +5 +1 +5 +1 +5 +5 +1 +5 +-- text +Granules: 4/4 +Name: idx_text +Granules: 1/4 +5 +-- the granule with NULL is skipped, so the query does not throw +5 +-- the text index is not read directly instead of a cast that throws on NULL +0 +5 +1 diff --git a/tests/queries/0_stateless/05338_skip_index_lossless_conversion_drop_nullable.sql b/tests/queries/0_stateless/05338_skip_index_lossless_conversion_drop_nullable.sql new file mode 100644 index 0000000000000..e56a75a9e0c7a --- /dev/null +++ b/tests/queries/0_stateless/05338_skip_index_lossless_conversion_drop_nullable.sql @@ -0,0 +1,59 @@ +-- Skip indexes look through a `CAST` that drops `Nullable` from the indexed column. It does not change a non-NULL +-- value, and it throws on NULL. A query that would throw does not throw if the granules with NULL are skipped. +-- The text index is not read directly instead of such a predicate, otherwise `NOT` would return the NULL row. + +SET explain_query_plan_default = 'legacy'; +SET enable_analyzer = 1; +SET use_skip_indexes = 1; +SET use_skip_indexes_on_data_read = 0; +SET use_query_condition_cache = 0; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + id UInt32, + s_bf Nullable(String), + s_lc LowCardinality(Nullable(String)), + s_text Nullable(String), + INDEX idx_bf s_bf TYPE bloom_filter GRANULARITY 1, + INDEX idx_lc s_lc TYPE bloom_filter GRANULARITY 1, + INDEX idx_text s_text TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +-- One part with four granules of two rows. The last granule holds NULL. +INSERT INTO tab SELECT + number, + if(number < 6, concat('bf', toString(number)), NULL), + if(number < 6, concat('lc', toString(number)), NULL), + if(number < 6, concat('text', toString(number), ' tail'), NULL) +FROM numbers(8); + +SELECT '-- bloom_filter'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab WHERE CAST(s_bf, 'String') = 'bf5') WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT id FROM tab WHERE CAST(s_bf, 'String') = 'bf5' SETTINGS force_data_skipping_indices = 'idx_bf'; +SELECT id FROM tab WHERE CAST(s_bf, 'String') IN ('bf1', 'bf5') ORDER BY id SETTINGS force_data_skipping_indices = 'idx_bf'; +SELECT id FROM tab WHERE has(['bf1', 'bf5'], CAST(s_bf, 'String')) ORDER BY id SETTINGS force_data_skipping_indices = 'idx_bf'; +SELECT id FROM tab WHERE CAST(s_lc, 'String') = 'lc5' SETTINGS force_data_skipping_indices = 'idx_lc'; +SELECT id FROM tab WHERE CAST(s_lc, 'LowCardinality(String)') IN ('lc1', 'lc5') ORDER BY id SETTINGS force_data_skipping_indices = 'idx_lc'; + +SELECT '-- text'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab WHERE hasToken(CAST(s_text, 'String'), 'text5')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT id FROM tab WHERE hasToken(CAST(s_text, 'String'), 'text5') SETTINGS force_data_skipping_indices = 'idx_text'; + +SELECT '-- the granule with NULL is skipped, so the query does not throw'; +SELECT id FROM tab WHERE CAST(s_bf, 'String') = 'bf5'; +SELECT id FROM tab WHERE CAST(s_bf, 'String') = 'bf5' SETTINGS use_skip_indexes = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } + +SELECT '-- the text index is not read directly instead of a cast that throws on NULL'; +SET use_skip_indexes_on_data_read = 1; +SET query_plan_direct_read_from_text_index = 1; +SELECT countIf(position(explain, '__text_index_') > 0) > 0 FROM (EXPLAIN actions = 1 SELECT id FROM tab WHERE hasToken(CAST(s_text, 'String'), 'text5')); +SELECT id FROM tab WHERE hasToken(CAST(s_text, 'String'), 'text5') SETTINGS force_data_skipping_indices = 'idx_text'; +SELECT id FROM tab WHERE NOT hasToken(CAST(s_text, 'String'), 'text5'); -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT countIf(position(explain, '__text_index_') > 0) > 0 FROM (EXPLAIN actions = 1 SELECT id FROM tab WHERE hasToken(s_text, 'text5')); + +DROP TABLE tab; From a59626a3c73d634b575b37be28709ee9467806f3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 13:23:38 +0000 Subject: [PATCH 224/296] Backport #123514 to 26.8: Fix mv source engine checks --- src/Analyzer/Resolve/IdentifierResolver.cpp | 24 +++++----- ..._sample_in_subquery_issue_123270.reference | 10 +++++ ..._final_sample_in_subquery_issue_123270.sql | 45 +++++++++++++++++++ 3 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 tests/queries/0_stateless/05241_mv_source_final_sample_in_subquery_issue_123270.reference create mode 100644 tests/queries/0_stateless/05241_mv_source_final_sample_in_subquery_issue_123270.sql diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 73cd1c3c40a99..e64cd591cd545 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -305,17 +305,6 @@ std::shared_ptr IdentifierResolver::tryResolveTableIdentifier(const I StorageID storage_id(database_name, table_name); storage_id = context->resolveStorageID(storage_id); - - /// The view source carries the inserted block and its types. For a MV, return this source - /// directly as a table node instead of swapping it later for the storage from the catalog - /// which may have been changed by a concurrent ALTER (the MV types must match the - /// snapshot at the start of the INSERT, not the current types). - /// For an inner query of an ordinary view, keep the normal flow that resolves from the catalog. - if (auto view_source = context->getViewSource(); - view_source && !context->isViewInnerQuery() - && view_source->getStorageID().getFullNameNotQuoted() == storage_id.getFullNameNotQuoted()) - return std::make_shared(view_source, context); - bool is_temporary_table = storage_id.getDatabaseName() == DatabaseCatalog::TEMPORARY_DATABASE; StoragePtr storage; @@ -374,7 +363,18 @@ std::shared_ptr IdentifierResolver::tryResolveTableIdentifier(const I if (!storage_lock) storage_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); storage->updateExternalDynamicMetadataIfExists(context); - const auto metadata_snapshot = storage->getInMemoryMetadataPtr(context, false); + auto metadata_snapshot = storage->getInMemoryMetadataPtr(context, false); + + /// The view source holds the block pushed to a materialized view with the types it was + /// converted to; the catalog table may have been altered since. Take the types from it, but + /// keep the catalog storage so engine checks (FINAL, SAMPLE, `_shard_num`) see the real table. + /// `replaceStorageInQueryTree` swaps the view source in for execution after the passes. + /// The inner query of an ordinary view reads the table itself. + if (auto view_source = context->getViewSource(); + view_source && !context->isViewInnerQuery() + && view_source->getStorageID().getFullNameNotQuoted() == storage_id.getFullNameNotQuoted()) + metadata_snapshot = view_source->getInMemoryMetadataPtr(context, false); + auto storage_snapshot = storage->getStorageSnapshot(metadata_snapshot, context); /// Pass the user-requested storage_id explicitly instead of letting the /// TableNode ctor read storage->getStorageID(), which can be mutated by diff --git a/tests/queries/0_stateless/05241_mv_source_final_sample_in_subquery_issue_123270.reference b/tests/queries/0_stateless/05241_mv_source_final_sample_in_subquery_issue_123270.reference new file mode 100644 index 0000000000000..fc672dca15332 --- /dev/null +++ b/tests/queries/0_stateless/05241_mv_source_final_sample_in_subquery_issue_123270.reference @@ -0,0 +1,10 @@ +in_final 1 1 +in_final 2 1 +join_final 1 2 +join_final 2 2 +join_sample 3 1 +join_table_final 1 1 +join_table_final 2 1 +scalar_final 1 2 +scalar_final 2 2 +scalar_sample 3 1 diff --git a/tests/queries/0_stateless/05241_mv_source_final_sample_in_subquery_issue_123270.sql b/tests/queries/0_stateless/05241_mv_source_final_sample_in_subquery_issue_123270.sql new file mode 100644 index 0000000000000..9304f1bfa19d0 --- /dev/null +++ b/tests/queries/0_stateless/05241_mv_source_final_sample_in_subquery_issue_123270.sql @@ -0,0 +1,45 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/123270 +-- FINAL and SAMPLE on the source table inside a subquery, JOIN or IN of a materialized view query +-- are validated against the real table engine, not against the pushed block. + +DROP TABLE IF EXISTS mv_final; +DROP TABLE IF EXISTS mv_sample; +DROP TABLE IF EXISTS mv_join_final; +DROP TABLE IF EXISTS mv_join_table_final; +DROP TABLE IF EXISTS mv_in_final; +DROP TABLE IF EXISTS mv_join_sample; +DROP TABLE IF EXISTS dst; +DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS src_sample; + +CREATE TABLE src (k UInt32) ENGINE = ReplacingMergeTree ORDER BY k; +CREATE TABLE src_sample (k UInt32) ENGINE = MergeTree ORDER BY intHash32(k) SAMPLE BY intHash32(k); +CREATE TABLE dst (name String, k UInt32, c UInt64) ENGINE = MergeTree ORDER BY (name, k); + +CREATE MATERIALIZED VIEW mv_final TO dst AS + SELECT 'scalar_final' AS name, k, (SELECT count() FROM src FINAL) AS c FROM src; +CREATE MATERIALIZED VIEW mv_sample TO dst AS + SELECT 'scalar_sample' AS name, k, (SELECT count() FROM src_sample SAMPLE 1) AS c FROM src_sample; +CREATE MATERIALIZED VIEW mv_join_final TO dst AS + SELECT 'join_final' AS name, s.k AS k, t.c AS c FROM src AS s CROSS JOIN (SELECT count() AS c FROM src FINAL) AS t; +CREATE MATERIALIZED VIEW mv_join_table_final TO dst AS + SELECT 'join_table_final' AS name, s.k AS k, count() AS c FROM src AS s INNER JOIN src AS t FINAL ON s.k = t.k GROUP BY s.k; +CREATE MATERIALIZED VIEW mv_in_final TO dst AS + SELECT 'in_final' AS name, k, toUInt64(k IN (SELECT k FROM src FINAL)) AS c FROM src; +CREATE MATERIALIZED VIEW mv_join_sample TO dst AS + SELECT 'join_sample' AS name, s.k AS k, t.c AS c FROM src_sample AS s CROSS JOIN (SELECT count() AS c FROM src_sample SAMPLE 1) AS t; + +INSERT INTO src VALUES (1), (2); +INSERT INTO src_sample VALUES (3); + +SELECT * FROM dst ORDER BY name, k; + +DROP TABLE mv_final; +DROP TABLE mv_sample; +DROP TABLE mv_join_final; +DROP TABLE mv_join_table_final; +DROP TABLE mv_in_final; +DROP TABLE mv_join_sample; +DROP TABLE dst; +DROP TABLE src; +DROP TABLE src_sample; From f70969964ef75c44d5fdedf256933b272b030e55 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:48:12 +0000 Subject: [PATCH 225/296] Fix build of the #123796 backport: no tryConvertFieldToTypeExact on this branch tryConvertFieldToTypeExact comes from #121078, which is not on this branch. At this call site only a widening conversion (DateTime or lower-scale DateTime64 to DateTime64) reaches it, which is exact or overflows to Null, so the strict tryConvertFieldToType gives the same result. In 05330, the "implied condition is not duplicated" case compares `a` with a DateTime64 constant instead of a DateTime one: on this branch the existing `a < ` conjunct hits TYPE_MISMATCH in tryConvertToColumnType (#117903, fixed on master by #121078), so it is not seen as implying the derived bound. Co-Authored-By: Claude Opus 5.5 --- src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp | 2 +- ...0_optimize_and_compare_chain_time_point_constant_scale.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index d29e5a3172217..c2e65ca35229c 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -2212,7 +2212,7 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont converted = tryConvertFieldToType( constant_node->getValue(), *other_type, constant_type.get(), getFormatSettings(getContext())); else if (isLowerScaleTimePoint(constant_type, other_type)) - converted = tryConvertFieldToTypeExact(constant_node->getValue(), *other_type, constant_type.get()); + converted = tryConvertFieldToType(constant_node->getValue(), *other_type, constant_type.get(), {}, /*strict=*/true); if (converted.isNull()) return {}; diff --git a/tests/queries/0_stateless/05330_optimize_and_compare_chain_time_point_constant_scale.sql b/tests/queries/0_stateless/05330_optimize_and_compare_chain_time_point_constant_scale.sql index 9035b2b919c9d..8bc3a532ba159 100644 --- a/tests/queries/0_stateless/05330_optimize_and_compare_chain_time_point_constant_scale.sql +++ b/tests/queries/0_stateless/05330_optimize_and_compare_chain_time_point_constant_scale.sql @@ -84,12 +84,12 @@ SELECT 'overflowing constant stays unchained', SELECT 'implied condition is not duplicated', (SELECT count() FROM (EXPLAIN QUERY TREE SELECT * FROM values('a DateTime64(6, \'UTC\'), b DateTime64(6, \'UTC\')', ('2020-01-01 00:00:00', '2020-01-01 00:00:01')) - WHERE a < b AND b < toDateTime('2020-01-01 00:00:02', 'UTC') AND a < toDateTime('2020-01-01 00:00:02', 'UTC') + WHERE a < b AND b < toDateTime('2020-01-01 00:00:02', 'UTC') AND a < toDateTime64('2020-01-01 00:00:02', 6, 'UTC') SETTINGS optimize_and_compare_chain = 1) WHERE explain LIKE '%function_name: less,%') = (SELECT count() FROM (EXPLAIN QUERY TREE SELECT * FROM values('a DateTime64(6, \'UTC\'), b DateTime64(6, \'UTC\')', ('2020-01-01 00:00:00', '2020-01-01 00:00:01')) - WHERE a < b AND b < toDateTime('2020-01-01 00:00:02', 'UTC') AND a < toDateTime('2020-01-01 00:00:02', 'UTC') + WHERE a < b AND b < toDateTime('2020-01-01 00:00:02', 'UTC') AND a < toDateTime64('2020-01-01 00:00:02', 6, 'UTC') SETTINGS optimize_and_compare_chain = 0) WHERE explain LIKE '%function_name: less,%'); -- The derived condition carries a typed constant, which is sent to the shards as is. From 54295a9bd3d64af6536ec149901c5ad5e74cd07c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 15:10:41 +0000 Subject: [PATCH 226/296] Backport #124049 to 26.8: Validate RoaringBitmap data when reading `groupBitmap` state --- .../AggregateFunctionGroupBitmapData.h | 5 ++++ ...roupbitmap_invalid_run_container.reference | 2 ++ ...5241_groupbitmap_invalid_run_container.sql | 24 +++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 tests/queries/0_stateless/05241_groupbitmap_invalid_run_container.reference create mode 100644 tests/queries/0_stateless/05241_groupbitmap_invalid_run_container.sql diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index 8b48a527edfbc..9ff950fbc3782 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -138,6 +138,11 @@ class RoaringBitmapWithSmallSet : private boost::noncopyable readStringGrowing(buf, size, in); roaring_bitmap = std::make_shared(RoaringBitmap::readSafe(buf.data(), size)); + + /// `readSafe` does not validate structure. Reject malformed bitmaps before any set operation. + const char * reason = nullptr; + if (!roaring_bitmap->internal_validate(&reason)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid roaring bitmap in groupBitmap state: {}", reason ? reason : "unknown reason"); } else throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown type of roaring bitmap"); diff --git a/tests/queries/0_stateless/05241_groupbitmap_invalid_run_container.reference b/tests/queries/0_stateless/05241_groupbitmap_invalid_run_container.reference new file mode 100644 index 0000000000000..5878ba4722554 --- /dev/null +++ b/tests/queries/0_stateless/05241_groupbitmap_invalid_run_container.reference @@ -0,0 +1,2 @@ +1000 +1000 diff --git a/tests/queries/0_stateless/05241_groupbitmap_invalid_run_container.sql b/tests/queries/0_stateless/05241_groupbitmap_invalid_run_container.sql new file mode 100644 index 0000000000000..acc36ecd8250d --- /dev/null +++ b/tests/queries/0_stateless/05241_groupbitmap_invalid_run_container.sql @@ -0,0 +1,24 @@ +-- Malformed groupBitmap states (a run container with n_runs = 0) must be rejected at deserialization, not crash on a set operation. +SELECT CAST(unhex('010b3b30000001000000000000'), 'AggregateFunction(groupBitmap, UInt32)'); -- { serverError INCORRECT_DATA } + +SELECT groupBitmapMerge(s) FROM +( + SELECT CAST(unhex('010b3b30000001000000000000'), 'AggregateFunction(groupBitmap, UInt32)') AS s + UNION ALL + SELECT CAST(unhex('010b3b30000001000000000000'), 'AggregateFunction(groupBitmap, UInt32)') AS s +); -- { serverError INCORRECT_DATA } + +SELECT bitmapCardinality(bitmapOr( + CAST(unhex('010b3b30000001000000000000'), 'AggregateFunction(groupBitmap, UInt32)'), + CAST(unhex('010b3b30000001000000000000'), 'AggregateFunction(groupBitmap, UInt32)') +)); -- { serverError INCORRECT_DATA } + +SELECT groupBitmapMerge(s) FROM +( + SELECT CAST(unhex('01170100000000000000000000003b30000001000000000000'), 'AggregateFunction(groupBitmap, UInt64)') AS s + UNION ALL + SELECT CAST(unhex('01170100000000000000000000003b30000001000000000000'), 'AggregateFunction(groupBitmap, UInt64)') AS s +); -- { serverError INCORRECT_DATA } + +SELECT groupBitmapMerge(s) FROM (SELECT groupBitmapState(number::UInt32) AS s FROM numbers(1000)); +SELECT groupBitmapMerge(s) FROM (SELECT groupBitmapState(number::UInt64) AS s FROM numbers(1000)); From 3655aa6bce78062c995e17c432933601b49c1cfd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 15:12:48 +0000 Subject: [PATCH 227/296] Backport #124043 to 26.8: Bump `CRoaring` to v5.2.3 --- contrib/croaring | 2 +- .../AggregateFunctionGroupNumericIndexedVectorDataBSI.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/croaring b/contrib/croaring index 342463b31b909..f2b1a55bcfa49 160000 --- a/contrib/croaring +++ b/contrib/croaring @@ -1 +1 @@ -Subproject commit 342463b31b909737bd6295c69b2f8e0ed9497424 +Subproject commit f2b1a55bcfa49ec986539b05b603e26a3adcc95c diff --git a/src/AggregateFunctions/AggregateFunctionGroupNumericIndexedVectorDataBSI.h b/src/AggregateFunctions/AggregateFunctionGroupNumericIndexedVectorDataBSI.h index a4dc03e067aed..4c08615eae280 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupNumericIndexedVectorDataBSI.h +++ b/src/AggregateFunctions/AggregateFunctionGroupNumericIndexedVectorDataBSI.h @@ -891,7 +891,7 @@ class BSINumericIndexedVector UInt64 row = 0; UInt64 col = val & 0x3f; #if defined(__BMI2__) && !defined(__e2k__) - ASM_SHIFT_RIGHT(val, shift, row); + CROARING_ASM_SHIFT_RIGHT(val, shift, row); #else row = val >> shift; #endif @@ -1006,7 +1006,7 @@ class BSINumericIndexedVector UInt64 tmp_offset = 0; UInt64 p = bit_buffer[i][j]; #if defined(__BMI2__) && !defined(__e2k__) - ASM_SHIFT_RIGHT(p, shift, tmp_offset); + CROARING_ASM_SHIFT_RIGHT(p, shift, tmp_offset); #else tmp_offset = p >> shift; #endif From 3eba0317af35410ae6090baade5f23a4503be8f8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 16:43:06 +0000 Subject: [PATCH 228/296] Update autogenerated version to 26.8.21.10 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 215c265dcc12a..457d01977a754 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54533) +SET(VERSION_REVISION 54534) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 21) -SET(VERSION_GITHASH 68170c6e7043e16504bb80174ee032a196a1aa63) -SET(VERSION_DESCRIBE v26.8.21.1-lts) -SET(VERSION_STRING 26.8.21.1) +SET(VERSION_PATCH 22) +SET(VERSION_GITHASH e9b8ef33ed7e4d28376d0c75ea01ae7ccc8859eb) +SET(VERSION_DESCRIBE v26.8.22.1-lts) +SET(VERSION_STRING 26.8.22.1) # end of autochange From b1492e037e54d657228e60bb8761e8e1591262af Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 17:25:41 +0000 Subject: [PATCH 229/296] Backport #124235 to 26.8: Create `mongocxx::instance` before any `mongocxx::uri` and forward MongoDB driver logs --- src/Dictionaries/MongoDBDictionarySource.h | 2 -- src/Processors/Sources/MongoDBSource.h | 2 -- src/Storages/StorageMongoDB.h | 6 +++-- .../tests/gtest_mongodb_instance_order.cpp | 22 +++++++++++++++++++ 4 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 src/Storages/tests/gtest_mongodb_instance_order.cpp diff --git a/src/Dictionaries/MongoDBDictionarySource.h b/src/Dictionaries/MongoDBDictionarySource.h index c286df62a515c..e5b82a13576f0 100644 --- a/src/Dictionaries/MongoDBDictionarySource.h +++ b/src/Dictionaries/MongoDBDictionarySource.h @@ -54,8 +54,6 @@ class MongoDBDictionarySource final : public IDictionarySource std::string toString() const override; private: - MongoDBInstanceHolder & instance_holder = MongoDBInstanceHolder::instance(); - const DictionaryStructure dict_struct; const std::shared_ptr configuration; SharedHeader sample_block; diff --git a/src/Processors/Sources/MongoDBSource.h b/src/Processors/Sources/MongoDBSource.h index 445c64c29e1dd..3c97e8579b9dc 100644 --- a/src/Processors/Sources/MongoDBSource.h +++ b/src/Processors/Sources/MongoDBSource.h @@ -32,8 +32,6 @@ class MongoDBSource final : public ISource String getName() const override { return "MongoDB"; } private: - MongoDBInstanceHolder & instance_holder = MongoDBInstanceHolder::instance(); - static void insertDefaultValue(IColumn & column, const IColumn & sample_column); void insertValue(IColumn & column, const size_t & idx, const DataTypePtr & type, const std::string & name, const bsoncxx::document::element & value); diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index c0a3e2131a751..65c6e5a508d7d 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -60,6 +60,10 @@ class MongoDBInstanceHolder final : public boost::noncopyable struct MongoDBConfiguration { + /// Declared first: the driver requires the instance to exist before any other driver object is created, + /// and a URI parsed on another thread while the instance installs its log handler races in `mongoc_log`. + MongoDBInstanceHolder & instance_holder = MongoDBInstanceHolder::instance(); + std::unique_ptr uri; String collection; std::unordered_set oid_fields = {"_id"}; @@ -110,8 +114,6 @@ class StorageMongoDB final : public StorageWithCommonVirtualColumns size_t num_streams) override; private: - MongoDBInstanceHolder & instance_holder = MongoDBInstanceHolder::instance(); - std::optional visitWhereConstant( const ContextPtr & context, const ConstantNode * const_node, diff --git a/src/Storages/tests/gtest_mongodb_instance_order.cpp b/src/Storages/tests/gtest_mongodb_instance_order.cpp new file mode 100644 index 0000000000000..d091c27686983 --- /dev/null +++ b/src/Storages/tests/gtest_mongodb_instance_order.cpp @@ -0,0 +1,22 @@ +#include +#include + +#if USE_MONGODB + +#include + +#include + +using namespace DB; + +/// The driver requires `mongocxx::instance` to exist before any other driver object, and the URI of a +/// configuration is built right after the configuration itself. `mongocxx::instance::current` creates a +/// default instance when none exists, after which the holder's instance can never be created. +TEST(StorageMongoDB, ConfigurationCreatesDriverInstanceFirst) +{ + MongoDBConfiguration configuration; + mongocxx::instance::current(); + EXPECT_NO_THROW(MongoDBInstanceHolder::instance()); +} + +#endif From 38afea9d6883a9c414b3012b893f8bcb702ec6c6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 18:54:23 +0000 Subject: [PATCH 230/296] Backport #118577 to 26.8: Write the empty statistics flag for bucketed Maps in JSON shared data --- src/DataTypes/Serializations/ISerialization.h | 3 +- .../Serializations/SerializationDynamic.cpp | 2 +- .../Serializations/SerializationMap.cpp | 3 +- .../Serializations/SerializationObject.cpp | 2 +- .../SerializationObjectSharedData.cpp | 4 +- .../gtest_map_bucketed_serialization.cpp | 1 + .../tests/gtest_object_serialization.cpp | 4 +- .../MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../MergeTree/MergeTreeReaderWide.cpp | 2 +- ..._bucketed_map_without_statistics.reference | 2 + ...d_data_bucketed_map_without_statistics.sql | 17 ++++ ..._shared_data_legacy_bucketed_map.reference | 9 ++ ...54_json_shared_data_legacy_bucketed_map.sh | 84 +++++++++++++++++++ ...d_map_compact_no_substream_marks.reference | 10 +++ ...ucketed_map_compact_no_substream_marks.sql | 27 ++++++ 15 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference create mode 100644 tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql create mode 100644 tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference create mode 100755 tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh create mode 100644 tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference create mode 100644 tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index 9ec18aafc98d7..b94270507e95b 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -503,7 +503,8 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro bool native_format = false; const FormatSettings * format_settings{}; - bool object_and_dynamic_read_statistics = false; + /// Whether the stream contains statistics for `Dynamic`, `JSON`, and `Map`. + bool read_statistics = false; /// Callback that should be called when new dynamic subcolumns are discovered during prefix deserialization. StreamCallback dynamic_subcolumns_callback; diff --git a/src/DataTypes/Serializations/SerializationDynamic.cpp b/src/DataTypes/Serializations/SerializationDynamic.cpp index c7ab75fecf621..bb35ed3545615 100644 --- a/src/DataTypes/Serializations/SerializationDynamic.cpp +++ b/src/DataTypes/Serializations/SerializationDynamic.cpp @@ -463,7 +463,7 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationDynamic::deserializeD auto variant_type = std::make_shared(variants); /// Read statistics. - if (settings.object_and_dynamic_read_statistics) + if (settings.read_statistics) { bool has_statistics = true; /// In V3 version we have additional flag that indicates if we have statistics or not. diff --git a/src/DataTypes/Serializations/SerializationMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp index 0175dcf28f477..d1abfe9b84063 100644 --- a/src/DataTypes/Serializations/SerializationMap.cpp +++ b/src/DataTypes/Serializations/SerializationMap.cpp @@ -868,7 +868,8 @@ SerializationMap::deserializeBucketsInfoStatePrefix(DeserializeBinaryBulkSetting /// Read statistics if any. ColumnMap::StatisticsPtr statistics; bool have_statistics = false; - readBinary(have_statistics, *stream); + if (settings.read_statistics) + readBinary(have_statistics, *stream); if (have_statistics) { auto mutable_statistics = std::make_shared(); diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index a2304073915ef..35ad4488a2880 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -827,7 +827,7 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb } /// Read statistics if needed. - if (settings.object_and_dynamic_read_statistics) + if (settings.read_statistics) { bool has_statistics = true; /// In V3 version we have additional flag that indicates if we have statistics or not. diff --git a/src/DataTypes/Serializations/SerializationObjectSharedData.cpp b/src/DataTypes/Serializations/SerializationObjectSharedData.cpp index f7799e8363749..fbbb4e5117d0d 100644 --- a/src/DataTypes/Serializations/SerializationObjectSharedData.cpp +++ b/src/DataTypes/Serializations/SerializationObjectSharedData.cpp @@ -992,7 +992,7 @@ std::shared_ptr SerializationO throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot read paths from object shared data with ADVANCED serialization version because seek_stream_to_mark_callback is not initialized"); DeserializeBinaryBulkSettings deserialization_settings; - deserialization_settings.object_and_dynamic_read_statistics = false; + deserialization_settings.read_statistics = false; deserialization_settings.position_independent_encoding = true; deserialization_settings.use_specialized_prefixes_and_suffixes_substreams = true; deserialization_settings.data_part_type = MergeTreeDataPartType::Compact; @@ -1216,7 +1216,7 @@ void SerializationObjectSharedData::deserializeBinaryBulkWithMultipleStreams( throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty stream for object shared data data"); DeserializeBinaryBulkSettings deserialization_settings; - deserialization_settings.object_and_dynamic_read_statistics = false; + deserialization_settings.read_statistics = false; deserialization_settings.position_independent_encoding = true; deserialization_settings.use_specialized_prefixes_and_suffixes_substreams = true; deserialization_settings.data_part_type = MergeTreeDataPartType::Compact; diff --git a/src/DataTypes/Serializations/tests/gtest_map_bucketed_serialization.cpp b/src/DataTypes/Serializations/tests/gtest_map_bucketed_serialization.cpp index ded350e2755bd..b0fd5436ba94b 100644 --- a/src/DataTypes/Serializations/tests/gtest_map_bucketed_serialization.cpp +++ b/src/DataTypes/Serializations/tests/gtest_map_bucketed_serialization.cpp @@ -92,6 +92,7 @@ ColumnPtr deserializeRange( std::map> buffers; ISerialization::DeserializeBinaryBulkSettings settings; + settings.read_statistics = true; settings.getter = [&](const ISerialization::SubstreamPath & path) -> ReadBuffer * { auto name = ISerialization::getSubcolumnNameForStream(path); diff --git a/src/DataTypes/Serializations/tests/gtest_object_serialization.cpp b/src/DataTypes/Serializations/tests/gtest_object_serialization.cpp index 9f286e714185f..6cc6bb3d34ed7 100644 --- a/src/DataTypes/Serializations/tests/gtest_object_serialization.cpp +++ b/src/DataTypes/Serializations/tests/gtest_object_serialization.cpp @@ -188,7 +188,7 @@ TEST(ObjectSerialization, FlattenAndBucketSharedDataPaths) } /// The shared-data-paths statistics count in the ObjectStructure prefix is only read when -/// `object_and_dynamic_read_statistics` is enabled -- the MergeTree part read path, which the +/// `read_statistics` is enabled -- the MergeTree part read path, which the /// `Native` input format (covered by 04350_json_native_too_many_paths) never reaches, since it /// always leaves statistics disabled. A corrupted count there must be rejected with a clean /// `INCORRECT_DATA` error instead of escaping as an uncaught `std::bad_alloc` / @@ -212,7 +212,7 @@ TEST(ObjectSerialization, TooManySharedDataPathsStatistics) ReadBufferFromString structure_stream(structure_bytes); ISerialization::DeserializeBinaryBulkSettings settings; - settings.object_and_dynamic_read_statistics = true; + settings.read_statistics = true; settings.getter = [&](const ISerialization::SubstreamPath & path) -> ReadBuffer * { if (!path.empty() && path.back().type == ISerialization::Substream::ObjectStructure) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 4e23d4af576c8..5a6f4d5ec810c 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -501,7 +501,7 @@ void MergeTreeReaderCompact::readPrefix( { ISerialization::DeserializeBinaryBulkSettings deserialize_settings; deserialize_settings.getter = buffer_getter; - deserialize_settings.object_and_dynamic_read_statistics = true; + deserialize_settings.read_statistics = true; deserialize_settings.use_specialized_prefixes_and_suffixes_substreams = true; deserialize_settings.data_part_type = MergeTreeDataPartType::Compact; deserialize_settings.check_stream_exists_callback = std::move(check_stream_exists_callback); diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 71bd9c2aeaa1d..7480fb55a34e8 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -388,7 +388,7 @@ void MergeTreeReaderWide::deserializePrefix( if (!deserialize_state_map.contains(name)) { ISerialization::DeserializeBinaryBulkSettings deserialize_settings; - deserialize_settings.object_and_dynamic_read_statistics = true; + deserialize_settings.read_statistics = true; deserialize_settings.prefixes_prefetch_callback = prefixes_prefetch_callback; deserialize_settings.data_part_type = MergeTreeDataPartType::Wide; deserialize_settings.prefixes_deserialization_thread_pool = settings.use_prefixes_deserialization_thread_pool ? &getMergeTreePrefixesDeserializationThreadPool().get() : nullptr; diff --git a/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference new file mode 100644 index 0000000000000..23a4dd0460b91 --- /dev/null +++ b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference @@ -0,0 +1,2 @@ +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 diff --git a/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql new file mode 100644 index 0000000000000..b6da283b5a30f --- /dev/null +++ b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS json_shared_bucketed_map; +CREATE TABLE json_shared_bucketed_map (j JSON(max_dynamic_paths = 0)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, + object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; + +-- Preserve runtime `Map` types in shared data instead of parsing them as `JSON` objects. +INSERT INTO json_shared_bucketed_map +SELECT * FROM format(RowBinary, 'j JSON(max_dynamic_paths = 0)', concat( + unhex('02016d'), formatRowNoNewline('RowBinary', CAST(map('k', toUInt64(1)) AS Dynamic)), + unhex('0174'), formatRowNoNewline('RowBinary', CAST(map('k', CAST(tuple(toUInt64(2)) AS Tuple(a UInt64))) AS Dynamic)))) +SETTINGS input_format_binary_read_json_as_string = 0; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map; +DROP TABLE json_shared_bucketed_map; diff --git a/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference new file mode 100644 index 0000000000000..c0412f2f07f69 --- /dev/null +++ b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference @@ -0,0 +1,9 @@ +legacy {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +subcolumns 1 2 +rewritten {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +rewritten {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +rewritten_types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +rewritten_types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +rewritten_subcolumns 1 2 +rewritten_subcolumns 1 2 diff --git a/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh new file mode 100755 index 0000000000000..250255f61c6c9 --- /dev/null +++ b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +workdir=$(mktemp -d "${CLICKHOUSE_TMP}/legacy_bucketed_map.XXXXXX") +trap 'rm -rf "$workdir"' EXIT + +part_dir=$($CLICKHOUSE_LOCAL --path "$workdir" --multiquery --query " +CREATE DATABASE legacy ENGINE = Atomic; +CREATE TABLE legacy.maps (j JSON(max_dynamic_paths = 0)) ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, object_shared_data_buckets_for_wide_part = 1, + object_shared_data_serialization_version = 'advanced', object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; +SELECT concat(data_paths[1], 'detached') FROM system.tables WHERE database = 'legacy' AND name = 'maps'; +") + +# A real part written by `91ffe225686`, before the `Map` statistics flag fix. +# Its `JSON` paths contain a scalar-valued map, a tuple-valued map, an array of maps, +# and nested `JSON` with a typed map. The original checksums are preserved. +base64 -d <<'PART' | tar -xz -C "$part_dir" +H4sIAAAAAAACA+2dC1xTZR/H/88uCAPGuKggyBnjfnGcjbHVwAuQiKAvKOYFL/M4DjjG2NzFFF8TZ1YWKb4mFZWa19Twfs8LlWaW +mn3MlKQsw8osyxKxlHyfcREbL5Kk6/PG+fP5c7bnPDuDnfN9fv/zPP/nGVVYqJLgHzJWPZlWa00WnUlsnmaG+2kkNrlM1rjFZr+V +SiTxLY+byiVS/ANCEhxgFpOZMuK3h65pt8+6ME9v1FFm4VTaaNLoi5RCGa9GGlHzTsw3PK2HRvpm6VpeWRAHINEJIGJ7bMPERK5F +CLKHe7qo9ZYis+2iQYWvPLrvYsWif6/NHjH6qg/3FhAFYv2kAlptVpkmU0Y6V5VLmSkxKZ6kKdqM6q/nT/vyswn1J9YcSytccXQo +qmYVlK6Nrtnks/jy+uf2Hxq4MDlIrTNqpZKFCeveebVhe2JS9NaNGaPy9IhcYnqqoUqjOtl74OHqd4feXBFmoMyTTSodZdSabEeP +5Y8a+fCVt3Oszz1Xs9DtVGUKGrDph4u3prI2JGbIwumpaet1EbH7L9WlFtef4wVL5jWUplecR2Q4O+XXiidQ6pYX+nHUJxPGy0yW +SSazkaZ0+Ni0meoPu9/6rvTY5+8EdU8jx1jmRKMStE5mrdW+lH/inNPHwaeq30by+dcuxyU2nC8r2MT6IfOx1z9B5Gfe39euNrjt +PNTfOVlYNWhdCD6mRW22GGnxLv2xwWs+OxW8e/uR+m3y0hikfHKxC582+FVQO5c3HK6dfDxs3LqjledWpRSPSTdHxZhX7EbkmuqD +xMhvT7zoOq4+su+aeetD69lffJm+tWfygB9Ljg96a8uyq0+jw/zLyHWPmjv24KrxIp/gyafdw6VB/cOXjebnJWzPKrpILeKOQuTg +fUPklQt+HZRbYTj1oSafE70P7SutOzL0lbjggzOnhJZRMfNRlZPp4rZr+r2/sG5d20bNGu5l6CN96lzx3FTltZGhR7+oEzUs6o/I +0d5HNvxkGLUy3fW9L1eGv3QjWq03TNcU5dLT6MDdt0JHu/SbM2H4FHddlalShTi1vsTwOYMGX3z/2WLvHFVlTOz7F3aeRiO2na6Y +5bv924ANpYhc+tGwZd25I/fWLqwRu1xY9l6wSVMsWh22hS94yn/t4z3XnzlFTV2CnB+pk29OfDOv/KvYHhz2hrAjoT8W0nM5Jfuz +066oImmoy0bknJXcA+IXT6Dw6yn9vjq57EbIVKrQkjvj95k3V30SMS6px2+e/kPY/VBO+tmNF57amPr4LzsneM0LoN8JW7t1EBF7 +xJrWMOXlgzt68MsRmX3deu2DC5l9e6uEmYqjFR498LVGuqQcTRqx+jU9v+bH1Kpbe/6D/CYK8tMjjYIVV13o4Qecglf4Sqd5sA6s +OfjuFf+80yvm9Yh0QeRzr08K+eCL1B17Jpiq4ycVcr1MtFFDFWqKKTPmT1xg0heVIn7Np0OvVl3iZb+gnpWdPPN98MTNlVljMmvU +JrGBUmvp3JnCa+WXE2azo+ZXBhz8RfbYSSCFZSKXg5+wdwtOqKoozazjvDMN3d1HH4ezgzyDz/7+hK9PtwHXdG877dSjEJczEb9Z +vy8vzXvi93OvocWuN+ddmf4E6r3Uj31MMbH3pT5k2sbZXFXkk4fcNxWnb80/vdOTzXd/5v2Zhn9HH3h+9kR3cenKXl9bXCaNfkG8 +txsw9v9mVKv+6wstuqL7r/4d6j8pIePs9F8iJxWM/jtE/5vOehv1l/AkwuZ9St7EgonC9OzMf0XoqGmq3OlFlE6jVjUqbl8yksdA +9E/iX9Ua8tyvpqAj/uXxpB3/MlIuZ/h3JP+tZ72dNkBC3lkHNxfNu4S4cVDyXFoD/ZbQ9s6yPwT/HdewbdrdeUek3/5b3P5L/0SV +Do7V9H52NwJ3+Qttca8Yx6q06W4Vmo7aFB/ftaItVsU1HMF/823cfX+PjvjHEYC9/kvjZQz/jjAJI4GM/jfyn0vnUZZCs0qt1xmM +tMkmAfhxLq3+y21Ch/xLpHb8x8fFkQz/jrCUzEcGpkQMyZFFMix0bf7vEovZelQfIP9Sub3+x0viGf4dYnU//7rwg17Jv+V88/YG +Z/7uN6yv4cIxCEDEAQSTUGB4dw5ygxAAFxiAELAE4OwNiA98AZBaLbsnyPALsoHjDATKAQ6XBiS3lWk9gfsw7CChxFmIkA4fZgpw +AiEcWCnQzRUgA4bhF2q53cDpIcCffi0XurHAQwDunAKYhcK741dRBGJxttmOXaZlMaz+ffw3DsM8OP1vy79MwsT/DrEDLxd3E+5a +Mfdqgzr55bOGp8uCbCcMmsb4hORM/ABwQzCGDbIElBvDjmGQ6Vr82w2sPhD+4xV2/Mvj45n+P4cY73vOl19r3IddMC/32FNzY5WV +wIUDsBOYeo7/WF/wf14AJQGIQYXhv5NhwL3zryCljP47xIZ51c9S7npxcPaOmxR33YWsMuIP+q9rfIJwi8AewQJXphXoovz/j+GP +Px8NdMi/wo5/qUSmiGP4d4Sdqv6uUP3KvKHjL8/3/Djb72NrLC4sadZ/n6gkFAm9NnmAEvqyPME/i4SoXk5CKFnG3I53ef7/dDRw +z/ovlSjimPwfh1gw/WrKXD79rXGO7nPjykR2O/pfwuh/F+W/NVW504MAHfMvt7//jyOZ/H+H2OTPzeneIqeowOvl9XUlZ16wBuJC +JfYDiIMopEN6ZMb898xiUOnq/Hd6EKBT/X/M+J9DbMc2S0jK6vMj4XzGkszKNcnt6L+S0f+uyn9rOnBnA4CO+FfI2/Avk0kZ/h1h +xkM3i1csP5Q/XPkWh+hZ0d/6DWZ8UDeA3wa6eolbcv1vi0B38VTKqKGKzKpcjUltVBmMdJ5mWoQ4yWikpkcMpQwR2Wajpig/Rvjo +4CKzXBYZKZ5kUWtpsy3XNU/P7w9E0EPgorflx5LBInDxJEMgVhIZArwpodKBWnq6KRqIkCjwJMJCwf1naEp/9RZnN16UIwUITYRe +fB8geEmoR2yQOAGBhxK5EAESYLE8USwp6g1sZx4iegcDjyACwcnjJvQguuNfS/uImyYxCNkInoxsvtxns4hoMa6qk+GjhCWAG7sf +ImLCwd05DRHhfcCHiIwAAWscIpKjgMfxQGgGa24LHEvY7AVsQp4CBKQD0VcBxId39Jcn9gNi6W1+6kDE7QusEiAysLP2g9OTkNV4 +N81DHgsgiCPiLsD32Xn481QugN5JtuzfgRwPd0SgBGAdgwXNycIJj4OINZvlUc1R+kWlc2aNsBgK6QgqwMmLCFWCK7uSTUSHgZvz +G2wiLAa8bf+LBysSZeKPl+qDXLMaT1+n+O9sANDx+L99/p9crmD6/x1iVoP/Vs/NG1W/V2uTZebNu8pEf9D/uQAP2fQftwhsARwu +21zKZTSzi+p/p5MAOsz/U9jP/1fI45j7f4fY6C+una5LLR9SFRqgXPW9ppu1EhcOYLeM/7v7gn93AfgHYg/CHon9YeyDsWcIgBgr +gAD/fBL89bhgGvYZ2OdgfxZ7Ka7wvK3COlxhCy7Yh/1d7Eewf4S9Gvun2M/jirUCGBLlgyggAgSQ6B+TBf4yvKsv9gHY07BnYR8m +gMAxAmC6I/42/u89Crhn/ZfiewKGf4fYpde/nm/ps8u4ZvFiQn7959p29B+3CIz+d0H+285VfAD6T8bZ9/8rFBKm/88h9tKM2Rmv +8/38d32ay/Ne9F2dlY8LObYYABCLzfDB8G/P/wPQ/zbj/6REzui/QyxqWEjJI9ffvGB6duYJ3ZGk3Hb6/zlM/3/X5b9xMYO/MAew +Q/5lbfr/pCTT/+cQ26eY8uxDN5555b3JWz/wTq32s3rhQmfsJRwGDob/O/jv/BzAToz/y+RM/O8Q+zAzZc1UN/HhUeQU+TwqJrYd +/Xdm9L/r8t80Bvvg8v8krftu5//FM/G/Qwyd+sFwK+BmTv76uOQfx1zyto7AhTnYt/jbpv6zENKyMf/JeMvhwSLg29YDQFrkAVX+ +d8ztj4BsHS7mBoDBq2nqfiyUMHME/lH8OzL/Ty5l8n8cYnuPT7x16mrGo5+9B6J0oe/GdvQ/h9H/rsP/X8/4v0f+5fbrf8YpSCb+ +d4iVHH80nDtjufbkL2NufHd5A88qxIV+2INt6//8hHAAwEF6LPIImbGqMw1AV+L/ry37cw/82/f/yZj1vxxkVpEwo/zGlZyf3Hzf +9Y38obzt+P8AAFfcFPgBJ4sFWw6mMMz8M/lvmderal7/+f4tBXzv6//GyZj7f8cYySDA8N/If9uvn7lv11hH+b8S+/6/OBwBMPw7 +wmaImlf4FynHjo8RGYx6A5VPmWmVebqBNqn+cFG0KINJZdarimiTmc5tqiZS4pCRjhHd7TUi5QyRjjKIlJIYkalxjhB+ODNG1Lzf +9oSh8W/l3/6LpcBR/Eva5P9LFcz8PwdZyx29V+v5VxWIbU9MLWt9Dmve+rEeL6nuNSZogrKwr0Xv52NNwoUDsSdzQIAP9LObX7vf +EoSCIBg8nYEjgDIms4Axxhhj7O+3/wKOKbQwAHgAAA== +PART + +$CLICKHOUSE_LOCAL --path "$workdir" --multiquery <<'SQL' +ALTER TABLE legacy.maps ATTACH PART 'all_1_1_0'; +SELECT 'legacy', j.m, j.t, j.a, j.o FROM legacy.maps; +SELECT 'types', dynamicType(j.m), dynamicType(j.t), dynamicType(j.a), dynamicType(j.o) FROM legacy.maps; +SELECT 'subcolumns', j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM legacy.maps; +INSERT INTO legacy.maps SELECT j FROM legacy.maps; +OPTIMIZE TABLE legacy.maps FINAL; +SELECT 'rewritten', j.m, j.t, j.a, j.o FROM legacy.maps; +SELECT 'rewritten_types', dynamicType(j.m), dynamicType(j.t), dynamicType(j.a), dynamicType(j.o) FROM legacy.maps; +SELECT 'rewritten_subcolumns', j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM legacy.maps; +SQL diff --git a/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference new file mode 100644 index 0000000000000..f037c389e431f --- /dev/null +++ b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference @@ -0,0 +1,10 @@ +Compact +{"m":{"k":1},"t":{"k":{"a":2}}} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 +{"m":{"k":1},"t":{"k":{"a":2}}} +{"m":{"k":1},"t":{"k":{"a":2}}} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 +1 2 diff --git a/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql new file mode 100644 index 0000000000000..ec084ba7434c1 --- /dev/null +++ b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS json_shared_bucketed_map_compact; +CREATE TABLE json_shared_bucketed_map_compact (j JSON(max_dynamic_paths = 0)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000, + write_marks_for_substreams_in_compact_parts = 0, + object_shared_data_serialization_version = 'advanced', + map_serialization_version = 'with_buckets', + object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; + +-- Preserve runtime `Map` types in shared data instead of parsing them as `JSON` objects. +INSERT INTO json_shared_bucketed_map_compact +SELECT * FROM format(RowBinary, 'j JSON(max_dynamic_paths = 0)', concat( + unhex('02016d'), formatRowNoNewline('RowBinary', CAST(map('k', toUInt64(1)) AS Dynamic)), + unhex('0174'), formatRowNoNewline('RowBinary', CAST(map('k', CAST(tuple(toUInt64(2)) AS Tuple(a UInt64))) AS Dynamic)))) +SETTINGS input_format_binary_read_json_as_string = 0; +SELECT part_type FROM system.parts WHERE database = currentDatabase() AND table = 'json_shared_bucketed_map_compact' AND active; +SELECT j FROM json_shared_bucketed_map_compact; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map_compact; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map_compact; +INSERT INTO json_shared_bucketed_map_compact SELECT j FROM json_shared_bucketed_map_compact; +OPTIMIZE TABLE json_shared_bucketed_map_compact FINAL; +SELECT j FROM json_shared_bucketed_map_compact; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map_compact; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map_compact; +DROP TABLE json_shared_bucketed_map_compact; From 785b5781b3d88b656227eaa742f13ddf75e89629 Mon Sep 17 00:00:00 2001 From: Vladimir Cherkasov Date: Thu, 8 Oct 2026 20:55:30 +0200 Subject: [PATCH 231/296] fix test --- .../05057_explain_hide_secrets_without_privilege.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.reference b/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.reference index d068effff2986..66eeb1d456f29 100644 --- a/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.reference +++ b/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.reference @@ -45,7 +45,7 @@ Positions: 1 Positions: 3 2 ReadFromMemoryStorage [\n {\n "Plan": {\n "Node Type": "Expression",\n "Node Id": "Expression_25",\n "Description": "(Project names + (Projection + (Change column names to column identifiers + (Convert VIEW subquery result to VIEW table structure + (Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers)))))))))",\n "Expression": {\n "Inputs": [\n {\n "Name": "secret",\n "Type": "String"\n }\n ],\n "Actions": [\n {\n "Node Type": "INPUT",\n "Result Type": "String",\n "Result Name": "secret",\n "Arguments": [0],\n "Removed Arguments": [],\n "Result": 0\n },\n {\n "Node Type": "COLUMN",\n "Result Type": "String",\n "Result Name": "\'aes-128-ecb\'_String",\n "Column": "Const(String)",\n "Arguments": [],\n "Removed Arguments": [],\n "Result": 1\n },\n {\n "Node Type": "COLUMN",\n "Result Type": "String",\n "Result Name": "[HIDDEN id: 1]_String",\n "Column": "Const(String)",\n "Arguments": [],\n "Removed Arguments": [],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "__table2.secret",\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 3\n },\n {\n "Node Type": "FUNCTION",\n "Result Type": "String",\n "Result Name": "encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)",\n "Function": "encrypt",\n "Compiled": false,\n "Arguments": [1, 3, 2],\n "Removed Arguments": [1, 3, 2],\n "Result": 0\n },\n {\n "Node Type": "FUNCTION",\n "Result Type": "String",\n "Result Name": "hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String))",\n "Function": "hex",\n "Compiled": false,\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "encrypted_secret",\n "Arguments": [2],\n "Removed Arguments": [2],\n "Result": 0\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "__table1.encrypted_secret",\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "encrypted_secret",\n "Arguments": [2],\n "Removed Arguments": [2],\n "Result": 0\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "__table1.encrypted_secret",\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "encrypted_secret",\n "Arguments": [2],\n "Removed Arguments": [2],\n "Result": 0\n }\n ],\n "Outputs": [\n {\n "Name": "encrypted_secret",\n "Type": "String"\n }\n ],\n "Positions": [0]\n },\n "Plans": [\n {\n "Node Type": "ReadFromMemoryStorage",\n "Node Id": "ReadFromMemoryStorage_6"\n }\n ]\n }\n }\n] -Output: hex(encrypt(\'aes-128-ecb\', secret, [HIDDEN])) +Output: hex(encrypt(\'aes-128-ecb\', [HIDDEN], [HIDDEN])) ReadFromMemoryStorage Header: secret String From 955cc391273a87221c71a8bf9d845db56bfd9e17 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 20:34:37 +0000 Subject: [PATCH 232/296] Backport #112304 to 26.8: Supporting lazily replicated arrays on arrayElement and arraySlice (reduces memory usage and improves performance) --- .../DataTypeLowCardinalityHelpers.cpp | 8 + src/Functions/GatherUtils/GatherUtils.h | 7 + src/Functions/GatherUtils/IArraySource.h | 2 + src/Functions/GatherUtils/Selectors.h | 10 + src/Functions/GatherUtils/Sources.h | 94 +++++++++ .../GatherUtils/createArraySource.cpp | 56 ++++++ .../GatherUtils/sliceDynamicOffsetBounded.cpp | 6 + .../sliceDynamicOffsetUnbounded.cpp | 6 + .../sliceFromLeftConstantOffsetBounded.cpp | 6 + .../sliceFromLeftConstantOffsetUnbounded.cpp | 6 + .../sliceFromRightConstantOffsetBounded.cpp | 6 + .../sliceFromRightConstantOffsetUnbounded.cpp | 6 + src/Functions/IFunction.cpp | 16 ++ src/Functions/IFunctionAdaptors.h | 1 + src/Functions/array/arrayElement.cpp | 181 ++++++++++++++++++ src/Functions/array/arraySlice.cpp | 13 +- .../lazy_replicated_array_functions.xml | 15 ++ ...y_replicated_array_slice_element.reference | 171 +++++++++++++++++ ...lazy_replicated_array_slice_element.sql.j2 | 126 ++++++++++++ 19 files changed, 733 insertions(+), 3 deletions(-) create mode 100644 tests/performance/lazy_replicated_array_functions.xml create mode 100644 tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference create mode 100644 tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 diff --git a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp index 13542bf0d001e..c9e04ed517156 100644 --- a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp +++ b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -147,6 +148,13 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column) { res = column_low_cardinality->convertToFullColumn(); } + else if (const auto * column_replicated = typeid_cast(column.get())) + { + const auto & nested = column_replicated->getNestedColumn(); + auto nested_no_lc = recursiveRemoveLowCardinality(nested); + if (nested.get() != nested_no_lc.get()) + res = ColumnReplicated::create(nested_no_lc, column_replicated->getIndexesColumn()); + } if (res != column) { diff --git a/src/Functions/GatherUtils/GatherUtils.h b/src/Functions/GatherUtils/GatherUtils.h index d880b8393adc8..d02cc8e31ac5f 100644 --- a/src/Functions/GatherUtils/GatherUtils.h +++ b/src/Functions/GatherUtils/GatherUtils.h @@ -28,6 +28,11 @@ * Sink - allows to build result column by copying Slices into it. */ +namespace DB +{ +class ColumnReplicated; +} + namespace DB::GatherUtils { @@ -41,6 +46,8 @@ enum class ArraySearchType : uint8_t }; std::unique_ptr createArraySource(const ColumnArray & col, bool is_const, size_t total_rows); +/// A source that reads a lazily replicated array without materializing it, only for slice* algorithms +std::unique_ptr createArraySourceFromReplicated(const ColumnReplicated & col); std::unique_ptr createValueSource(const IColumn & col, bool is_const, size_t total_rows); std::unique_ptr createArraySink(ColumnArray & col, size_t column_size); diff --git a/src/Functions/GatherUtils/IArraySource.h b/src/Functions/GatherUtils/IArraySource.h index ee3e0da004a84..e07776b84cac4 100644 --- a/src/Functions/GatherUtils/IArraySource.h +++ b/src/Functions/GatherUtils/IArraySource.h @@ -23,6 +23,8 @@ struct IArraySource virtual size_t getColumnSize() const = 0; virtual bool isConst() const { return false; } virtual bool isNullable() const { return false; } + /// True for ReplicatedSource: the source reads a lazily replicated array (ColumnReplicated) + virtual bool isReplicated() const { return false; } virtual void accept(ArraySourceVisitor &) { diff --git a/src/Functions/GatherUtils/Selectors.h b/src/Functions/GatherUtils/Selectors.h index 1e7ee42d5ecb3..ac42b98bdefed 100644 --- a/src/Functions/GatherUtils/Selectors.h +++ b/src/Functions/GatherUtils/Selectors.h @@ -63,6 +63,16 @@ struct ArraySourceSelector template static void select(IArraySource & source, Args && ... args) { + /// ReplicatedSource follows the non-virtual iteration methods of its base source, so it must be dispatched to the algorithm + /// with its concrete type. Only selectors that declare `supports_replicated_source = true` do that + constexpr bool supports_replicated_source = requires { requires Base::supports_replicated_source; }; + if constexpr (!supports_replicated_source) + { + if (source.isReplicated()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "{} does not support lazily replicated array sources", demangle(typeid(Base).name())); + } + ArraySourceSelectorVisitor visitor(source, args ...); source.accept(visitor); } diff --git a/src/Functions/GatherUtils/Sources.h b/src/Functions/GatherUtils/Sources.h index 1a2a98b2aff89..d4de1ad4708ce 100644 --- a/src/Functions/GatherUtils/Sources.h +++ b/src/Functions/GatherUtils/Sources.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -238,6 +239,99 @@ struct ConstSource : public Base #pragma clang diagnostic pop + +/// ReplicatedSource makes the pair (nested array column, replication indexes) look like a flat source of logical rows. +/// Base is an array source: NumericArraySource, GenericArraySource, or NullableArraySource +/// +/// nested_row: the unreplicated data, a block of 3 array have 3 nested_rows (0, 1, 2) +/// logical_row: a row_index into the column as consumers see it: +/// +/// i.e. +/// nested column (3 rows): A B C +/// replication_indexes: [0, 0, 1, 2, 2, 2] +/// logical view (6 rows): A A B C C C +template +struct ReplicatedSource : public Base +{ + using Slice = typename Base::Slice; + using SinkType = typename Base::SinkType; + using Base::row_num; + using Base::prev_offset; + using Base::offsets; + + const ColumnIndex & replication_indexes; + size_t total_rows; + size_t logical_row = 0; + + template + ReplicatedSource(const ColumnType & col_, const ColumnIndex & replication_indexes_) + : Base(col_) + , replication_indexes(replication_indexes_) + , total_rows(replication_indexes_.size()) + { + positionAtCurrentRow(); + } + + template + ReplicatedSource(const ColumnType & col_, const NullMap & null_map_, const ColumnIndex & replication_indexes_) + : Base(col_, null_map_) + , replication_indexes(replication_indexes_) + , total_rows(replication_indexes_.size()) + { + positionAtCurrentRow(); + } + + void accept(ArraySourceVisitor & visitor) override + { + visitor.visit(*this); + } + + void next() + { + ++logical_row; + positionAtCurrentRow(); + } + + bool isEnd() const + { + return logical_row == total_rows; + } + + size_t rowNum() const + { + return logical_row; + } + + size_t getSizeForReserve() const override + { + /// A proportional estimate: the exact size would need an extra pass over the indexes. + size_t nested_rows = offsets.size(); + return nested_rows == 0 ? 0 : Base::getSizeForReserve() / nested_rows * total_rows; + } + + size_t getColumnSize() const override + { + return total_rows; + } + + bool isReplicated() const override + { + return true; + } + +private: + /// Point the base source at the nested row of the current logical row. + void positionAtCurrentRow() + { + if (logical_row == total_rows) + return; + ssize_t nested_row = replication_indexes.getIndexAt(logical_row); + row_num = nested_row; + /// `offsets[-1]` is a guaranteed zero (`PaddedPODArray` left padding), same as `ColumnArray::offsetAt`. + prev_offset = offsets[nested_row - 1]; + } +}; + struct StringSource { using Slice = NumericArraySlice; diff --git a/src/Functions/GatherUtils/createArraySource.cpp b/src/Functions/GatherUtils/createArraySource.cpp index 1e080b53ccbec..523437d9dbd9b 100644 --- a/src/Functions/GatherUtils/createArraySource.cpp +++ b/src/Functions/GatherUtils/createArraySource.cpp @@ -1,8 +1,15 @@ +#include #include #include #include #include + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace DB::GatherUtils { /// Creates IArraySource from ColumnArray @@ -54,6 +61,38 @@ struct ArraySourceCreator<> } }; +template +struct ReplicatedArraySourceCreator; + +template +struct ReplicatedArraySourceCreator +{ + static std::unique_ptr create(const ColumnArray & col, const NullMap * null_map, const ColumnIndex & replication_indexes) + { + using ColVecType = ColumnVectorOrDecimal; + + if (typeid_cast(&col.getData())) + { + if (null_map) + return std::make_unique>>>(col, *null_map, replication_indexes); + return std::make_unique>>(col, replication_indexes); + } + + return ReplicatedArraySourceCreator::create(col, null_map, replication_indexes); + } +}; + +template <> +struct ReplicatedArraySourceCreator<> +{ + static std::unique_ptr create(const ColumnArray & col, const NullMap * null_map, const ColumnIndex & replication_indexes) + { + if (null_map) + return std::make_unique>>(col, *null_map, replication_indexes); + return std::make_unique>(col, replication_indexes); + } +}; + } std::unique_ptr createArraySource(const ColumnArray & col, bool is_const, size_t total_rows) @@ -66,4 +105,21 @@ std::unique_ptr createArraySource(const ColumnArray & col, bool is } return Creator::create(col, nullptr, is_const, total_rows); } + +std::unique_ptr createArraySourceFromReplicated(const ColumnReplicated & col) +{ + const auto * nested_array = typeid_cast(col.getNestedColumn().get()); + if (!nested_array) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "createArraySourceFromReplicated expects ColumnReplicated over ColumnArray, got {}", + col.getNestedColumn()->getName()); + + using Creator = TypeListChangeRoot; + if (const auto * column_nullable = typeid_cast(&nested_array->getData())) + { + auto column = ColumnArray::create(column_nullable->getNestedColumnPtr(), nested_array->getOffsetsPtr()); + return Creator::create(*column, &column_nullable->getNullMapData(), col.getIndexes()); + } + return Creator::create(*nested_array, nullptr, col.getIndexes()); +} } diff --git a/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp b/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp index 0c5b7f78a815f..560c68cbb1685 100644 --- a/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp @@ -12,6 +12,8 @@ namespace struct SliceDynamicOffsetBoundedSelectArraySource : public ArraySourceSelector { + static constexpr bool supports_replicated_source = true; + template static void selectSource(bool is_const, bool is_nullable, Source && source, const IColumn & offset_column, const IColumn & length_column, ColumnArray::MutablePtr & result) @@ -31,6 +33,8 @@ struct SliceDynamicOffsetBoundedSelectArraySource : public ArraySourceSelector &>(source), sink, offset_column, length_column); + else if (source.isReplicated()) + sliceDynamicOffsetBounded(static_cast &>(source), sink, offset_column, length_column); else sliceDynamicOffsetBounded(static_cast(source), sink, offset_column, length_column); } @@ -41,6 +45,8 @@ struct SliceDynamicOffsetBoundedSelectArraySource : public ArraySourceSelector &>(source), sink, offset_column, length_column); + else if (source.isReplicated()) + sliceDynamicOffsetBounded(static_cast &>(source), sink, offset_column, length_column); else sliceDynamicOffsetBounded(source, sink, offset_column, length_column); } diff --git a/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp index 976ec69514795..d109f49248d46 100644 --- a/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp @@ -13,6 +13,8 @@ namespace struct SliceDynamicOffsetUnboundedSelectArraySource : public ArraySourceSelector { + static constexpr bool supports_replicated_source = true; + template static void selectSource(bool is_const, bool is_nullable, Source && source, const IColumn & offset_column, ColumnArray::MutablePtr & result) { @@ -31,6 +33,8 @@ struct SliceDynamicOffsetUnboundedSelectArraySource if (is_const) sliceDynamicOffsetUnbounded(static_cast &>(source), sink, offset_column); + else if (source.isReplicated()) + sliceDynamicOffsetUnbounded(static_cast &>(source), sink, offset_column); else sliceDynamicOffsetUnbounded(static_cast(source), sink, offset_column); } @@ -41,6 +45,8 @@ struct SliceDynamicOffsetUnboundedSelectArraySource if (is_const) sliceDynamicOffsetUnbounded(static_cast &>(source), sink, offset_column); + else if (source.isReplicated()) + sliceDynamicOffsetUnbounded(static_cast &>(source), sink, offset_column); else sliceDynamicOffsetUnbounded(source, sink, offset_column); } diff --git a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp index 905a554c59cfb..a6235381b12bb 100644 --- a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp @@ -13,6 +13,8 @@ namespace struct SliceFromLeftConstantOffsetBoundedSelectArraySource : public ArraySourceSelector { + static constexpr bool supports_replicated_source = true; + template static void selectSource(bool is_const, bool is_nullable, Source && source, size_t & offset, ssize_t & length, ColumnArray::MutablePtr & result) { @@ -31,6 +33,8 @@ struct SliceFromLeftConstantOffsetBoundedSelectArraySource if (is_const) sliceFromLeftConstantOffsetBounded(static_cast &>(source), sink, offset, length); + else if (source.isReplicated()) + sliceFromLeftConstantOffsetBounded(static_cast &>(source), sink, offset, length); else sliceFromLeftConstantOffsetBounded(static_cast(source), sink, offset, length); } @@ -41,6 +45,8 @@ struct SliceFromLeftConstantOffsetBoundedSelectArraySource if (is_const) sliceFromLeftConstantOffsetBounded(static_cast &>(source), sink, offset, length); + else if (source.isReplicated()) + sliceFromLeftConstantOffsetBounded(static_cast &>(source), sink, offset, length); else sliceFromLeftConstantOffsetBounded(source, sink, offset, length); } diff --git a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp index 32c93604712c5..fb6dee57920bf 100644 --- a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp @@ -13,6 +13,8 @@ namespace struct SliceFromLeftConstantOffsetUnboundedSelectArraySource : public ArraySourceSelector { + static constexpr bool supports_replicated_source = true; + template static void selectSource(bool is_const, bool is_nullable, Source && source, size_t & offset, ColumnArray::MutablePtr & result) { @@ -31,6 +33,8 @@ struct SliceFromLeftConstantOffsetUnboundedSelectArraySource if (is_const) sliceFromLeftConstantOffsetUnbounded(static_cast &>(source), sink, offset); + else if (source.isReplicated()) + sliceFromLeftConstantOffsetUnbounded(static_cast &>(source), sink, offset); else sliceFromLeftConstantOffsetUnbounded(static_cast(source), sink, offset); } @@ -41,6 +45,8 @@ struct SliceFromLeftConstantOffsetUnboundedSelectArraySource if (is_const) sliceFromLeftConstantOffsetUnbounded(static_cast &>(source), sink, offset); + else if (source.isReplicated()) + sliceFromLeftConstantOffsetUnbounded(static_cast &>(source), sink, offset); else sliceFromLeftConstantOffsetUnbounded(source, sink, offset); } diff --git a/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp b/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp index c012981792627..a7a822e5d0d24 100644 --- a/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp @@ -13,6 +13,8 @@ namespace struct SliceFromRightConstantOffsetBoundedSelectArraySource : public ArraySourceSelector { + static constexpr bool supports_replicated_source = true; + template static void selectSource(bool is_const, bool is_nullable, Source && source, size_t & offset, ssize_t & length, ColumnArray::MutablePtr & result) { @@ -31,6 +33,8 @@ struct SliceFromRightConstantOffsetBoundedSelectArraySource if (is_const) sliceFromRightConstantOffsetBounded(static_cast &>(source), sink, offset, length); + else if (source.isReplicated()) + sliceFromRightConstantOffsetBounded(static_cast &>(source), sink, offset, length); else sliceFromRightConstantOffsetBounded(static_cast(source), sink, offset, length); } @@ -41,6 +45,8 @@ struct SliceFromRightConstantOffsetBoundedSelectArraySource if (is_const) sliceFromRightConstantOffsetBounded(static_cast &>(source), sink, offset, length); + else if (source.isReplicated()) + sliceFromRightConstantOffsetBounded(static_cast &>(source), sink, offset, length); else sliceFromRightConstantOffsetBounded(source, sink, offset, length); } diff --git a/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp index 4187446958d7e..b0fe25afc62f4 100644 --- a/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp @@ -13,6 +13,8 @@ namespace struct SliceFromRightConstantOffsetUnboundedSelectArraySource : public ArraySourceSelector { + static constexpr bool supports_replicated_source = true; + template static void selectSource(bool is_const, bool is_nullable, Source && source, size_t & offset, ColumnArray::MutablePtr & result) { @@ -31,6 +33,8 @@ struct SliceFromRightConstantOffsetUnboundedSelectArraySource if (is_const) sliceFromRightConstantOffsetUnbounded(static_cast &>(source), sink, offset); + else if (source.isReplicated()) + sliceFromRightConstantOffsetUnbounded(static_cast &>(source), sink, offset); else sliceFromRightConstantOffsetUnbounded(static_cast(source), sink, offset); } @@ -41,6 +45,8 @@ struct SliceFromRightConstantOffsetUnboundedSelectArraySource if (is_const) sliceFromRightConstantOffsetUnbounded(static_cast &>(source), sink, offset); + else if (source.isReplicated()) + sliceFromRightConstantOffsetUnbounded(static_cast &>(source), sink, offset); else sliceFromRightConstantOffsetUnbounded(source, sink, offset); } diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index ba856de66b72f..2744f60e68fd2 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -217,6 +217,22 @@ ColumnPtr IExecutableFunction::defaultImplementationForNulls( if (null_presence.has_nullable) { + /// A `Nullable`-typed argument may still be wrapped in `ColumnReplicated` when the function + /// opted out of the default implementation for replicated columns (e.g. `arrayElement`). + for (const auto & arg : args) + { + if (arg.type->isNullable() && typeid_cast(arg.column.get())) + { + ColumnsWithTypeAndName materialized_args = args; + for (auto & materialized_arg : materialized_args) + { + if (materialized_arg.type->isNullable()) + materialized_arg.column = materialized_arg.column->convertToFullColumnIfReplicated(); + } + return defaultImplementationForNulls(materialized_args, result_type, input_rows_count, dry_run); + } + } + const bool result_is_nullable = result_type->isNullable(); if (!result_is_nullable) diff --git a/src/Functions/IFunctionAdaptors.h b/src/Functions/IFunctionAdaptors.h index 4b50b559d9729..110a20b714838 100644 --- a/src/Functions/IFunctionAdaptors.h +++ b/src/Functions/IFunctionAdaptors.h @@ -25,6 +25,7 @@ class FunctionToExecutableFunctionAdaptor final : public IExecutableFunction bool useDefaultImplementationForConstants() const final { return function->useDefaultImplementationForConstants(); } bool useDefaultImplementationForLowCardinalityColumns() const final { return function->useDefaultImplementationForLowCardinalityColumns(); } bool useDefaultImplementationForSparseColumns() const final { return function->useDefaultImplementationForSparseColumns(); } + bool useDefaultImplementationForReplicatedColumns() const final { return function->useDefaultImplementationForReplicatedColumns(); } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return function->getArgumentsThatAreAlwaysConstant(); } bool canBeExecutedOnDefaultArguments() const override { return function->canBeExecutedOnDefaultArguments(); } diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index a041de61cb250..5187fd27484d0 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +75,9 @@ class FunctionArrayElement : public IFunction String getName() const override; bool useDefaultImplementationForConstants() const override { return true; } + /// A lazily replicated array argument is consumed by gathering from the compact nested column. + /// When true it materializes the compacted representation to a full column. + bool useDefaultImplementationForReplicatedColumns() const override { return false; } /// `Nullable(QBit)` with an array of indices must produce `Array(Nullable(T))`, /// which cannot be represented by the default nullable wrapper around the result. bool useDefaultImplementationForNulls() const override { return false; } @@ -103,6 +107,19 @@ class FunctionArrayElement : public IFunction ArrayImpl::NullMapBuilder & builder, size_t input_rows_count) const; + /// Element access over a lazily replicated array (Replicated(Array)) without materializing it. + ColumnPtr executeReplicated( + const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const; + + template + static bool gatherReplicated( + const IColumn & index_column, + const ColumnIndex & replication_indexes, + const ColumnArray::Offsets & offsets, + const IColumn & data, + IColumn & result, + ArrayImpl::NullMapBuilder & builder); + template static ColumnPtr executeNumberConst( const ColumnsWithTypeAndName & arguments, @@ -3010,6 +3027,11 @@ template ColumnPtr FunctionArrayElement::executeImpl( const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { + /// A replicated column can arrive in either position. + if (typeid_cast(arguments[0].column.get()) + || typeid_cast(arguments[1].column.get())) + return executeReplicated(arguments, result_type, input_rows_count); + const bool is_qbit = checkAndGetDataType(removeNullable(arguments[0].type).get()); /// The default nullable implementation cannot preserve a NULL `QBit` source for an @@ -3133,6 +3155,165 @@ ColumnPtr FunctionArrayElement::executeImpl( return ColumnNullable::create(res, builder ? std::move(builder).getNullMapColumnPtr() : ColumnUInt8::create()); } +template +ColumnPtr FunctionArrayElement::executeReplicated( + const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const +{ + ColumnsWithTypeAndName args = arguments; + + const auto * replicated_array = typeid_cast(args[0].column.get()); + const auto * replicated_index = typeid_cast(args[1].column.get()); + + /// When the array and the index are replicated by the same indexes column (they came from the same expansion, + /// e.g. one ARRAY JOIN), row i reads nested_array[idx[i]][nested_index[idx[i]]], so the result is the nested + /// computation replicated by the same indexes. + if (replicated_array && replicated_index + && replicated_array->getIndexesColumn().get() == replicated_index->getIndexesColumn().get() + && replicated_array->getNestedColumn()->size() == replicated_index->getNestedColumn()->size()) + { + /// Compute on the compact nested columns and stay lazy.(Remove unused indexes) + auto compact = replicated_array->getIndexes().buildCompactIndexedColumns( + {replicated_array->getNestedColumn(), replicated_index->getNestedColumn()}); + + /// Now recurse on the internal rows + size_t nested_rows_count = compact.compact_indexed_columns[0]->size(); + ColumnsWithTypeAndName nested_args + = {{compact.compact_indexed_columns[0], args[0].type, args[0].name}, + {compact.compact_indexed_columns[1], args[1].type, args[1].name}}; + auto nested_res = executeImpl(nested_args, result_type, nested_rows_count); + + /// Wrap the result in a new ColumnReplicated with the compacted indexes column + return convertToFullColumnIfReplicationNotUseful( + ColumnReplicated::create(std::move(nested_res), compact.compact_indexes)); + } + + /// The index argument is a per-row number and when it is replicated independently of the array, + /// its repetition has no structure, so materialize it. + args[1].column = args[1].column->convertToFullColumnIfReplicated(); + + if (!replicated_array) + return executeImpl(args, result_type, input_rows_count); + + const auto * col_array = typeid_cast(replicated_array->getNestedColumn().get()); + + /// Fall back to materialization for the shapes the fast path does not cover: + /// Replicated over Map, and LowCardinality elements (their handling is layered above this function + bool fast_path_supported = col_array + && !typeid_cast(&col_array->getData()) + && !typeid_cast(&col_array->getData()); + if (!fast_path_supported) + { + args[0].column = args[0].column->convertToFullColumnIfReplicated(); + return executeImpl(args, result_type, input_rows_count); + } + + if (isColumnConst(*args[1].column)) + { + /// A constant index gives one value per nested row: execute on the compact nested array and replicate the result lazily. + /// Compact away nested rows the indexes never reference, so the nested work is proportional to the used rows. + auto compact = replicated_array->getIndexes().buildCompactIndexedColumns({replicated_array->getNestedColumn()}); + size_t nested_rows_count = compact.compact_indexed_columns[0]->size(); + ColumnsWithTypeAndName nested_args + = {{compact.compact_indexed_columns[0], args[0].type, args[0].name}, + {args[1].column->cloneResized(nested_rows_count), args[1].type, args[1].name}}; + + /// Recurse on the internal rows + auto nested_res = executeImpl(nested_args, result_type, nested_rows_count); + /// Wrap the result in a new ColumnReplicated with the compacted indexes column + return convertToFullColumnIfReplicationNotUseful( + ColumnReplicated::create(std::move(nested_res), compact.compact_indexes)); + } + + const auto & offsets = col_array->getOffsets(); + const IColumn * data = &col_array->getData(); + + ArrayImpl::NullMapBuilder builder; + bool is_array_of_nullable = isColumnNullable(*data); + if (is_array_of_nullable) + { + const auto & nullable_data = assert_cast(*data); + builder.initSource(nullable_data.getNullMapData().data()); + data = &nullable_data.getNestedColumn(); + } + + if (builder) + builder.initSink(input_rows_count); + + auto result = data->cloneEmpty(); + result->reserve(input_rows_count); + + const auto & replication_indexes = replicated_array->getIndexes(); + const auto & index_column = *args[1].column; + /// Core loop to build result by dispatching based on the index type + if (!(gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) + || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) + || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) + || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) + || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) + || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) + || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) + || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder))) + { + /// The index is not a plain numeric column (e.g. Nullable, or an array of indexes): + /// materialize the array and let the generic path handle it. + args[0].column = args[0].column->convertToFullColumnIfReplicated(); + return executeImpl(args, result_type, input_rows_count); + } + + if (is_array_of_nullable) + return ColumnNullable::create( + std::move(result), builder ? std::move(builder).getNullMapColumnPtr() : ColumnUInt8::create()); + + ColumnPtr immutable_result = std::move(result); + if (builder && immutable_result->canBeInsideNullable()) + return ColumnNullable::create(immutable_result, std::move(builder).getNullMapColumnPtr()); + + return immutable_result; +} + +template +template +bool FunctionArrayElement::gatherReplicated( + const IColumn & index_column, + const ColumnIndex & replication_indexes, + const ColumnArray::Offsets & offsets, + const IColumn & data, + IColumn & result, + ArrayImpl::NullMapBuilder & builder) +{ + const auto * index_vec = checkAndGetColumn>(&index_column); + if (!index_vec) + return false; + + const auto & indices = index_vec->getData(); + size_t rows = indices.size(); + /// Each output element is one insertFrom reading directly from the shared nested data + for (size_t i = 0; i < rows; ++i) + { + ssize_t nested_row = replication_indexes.getIndexAt(i); + /// `offsets[-1]` is a guaranteed zero (`PaddedPODArray` left padding), same as `ColumnArray::offsetAt`. + ColumnArray::Offset begin = offsets[nested_row - 1]; + ColumnArray::Offset end = offsets[nested_row]; + + IndexType index = indices[i]; + /// Positive index is 1-based from the beginning of the array, negative counts from the end. + /// Any invalid index (zero, out of range) lands outside [begin, end) and produces a default value. + size_t insert_position = index > 0 ? begin + index - 1 : end + index; + if (begin <= insert_position && insert_position < end) + { + result.insertFrom(data, insert_position); + builder.update(insert_position); + } + else + { + result.insertDefault(); + builder.update(); + } + } + + return true; +} + template ColumnPtr FunctionArrayElement::perform( const ColumnsWithTypeAndName & arguments, diff --git a/src/Functions/array/arraySlice.cpp b/src/Functions/array/arraySlice.cpp index f99099bd4af33..521b7cb78094e 100644 --- a/src/Functions/array/arraySlice.cpp +++ b/src/Functions/array/arraySlice.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -106,8 +107,10 @@ class FunctionArraySlice final : public IFunction return executeQBitSlice(arguments, input_rows_count); auto array_column = arguments[0].column; - const auto & offset_column = arguments[1].column; - const auto & length_column = arguments.size() > 2 ? arguments[2].column : nullptr; + /// The offset and length are per-row numbers + /// The array argument is the one worth keeping lazy: it is consumed by a ReplicatedSource + const auto offset_column = arguments[1].column->convertToFullColumnIfReplicated(); + const auto length_column = arguments.size() > 2 ? arguments[2].column->convertToFullColumnIfReplicated() : nullptr; std::unique_ptr source; @@ -120,7 +123,9 @@ class FunctionArraySlice final : public IFunction array_column = const_array_column->getDataColumnPtr(); } - if (const auto * argument_column_array = typeid_cast(array_column.get())) + if (const auto * replicated_column = typeid_cast(array_column.get())) + source = GatherUtils::createArraySourceFromReplicated(*replicated_column); + else if (const auto * argument_column_array = typeid_cast(array_column.get())) source = GatherUtils::createArraySource(*argument_column_array, is_const, size); else throw Exception(ErrorCodes::LOGICAL_ERROR, "First arguments for function {} must be array.", getName()); @@ -176,6 +181,8 @@ class FunctionArraySlice final : public IFunction bool useDefaultImplementationForConstants() const override { return true; } bool useDefaultImplementationForNulls() const override { return false; } + /// When set to true, materializes the columns whenever the offset or length is a full column. + bool useDefaultImplementationForReplicatedColumns() const override { return false; } private: struct QBitSliceBounds diff --git a/tests/performance/lazy_replicated_array_functions.xml b/tests/performance/lazy_replicated_array_functions.xml new file mode 100644 index 0000000000000..189daf39ec41d --- /dev/null +++ b/tests/performance/lazy_replicated_array_functions.xml @@ -0,0 +1,15 @@ + + + + lazy_replication + + 0 + 1 + + + + + + SELECT sum(length(arraySlice(big, x, 2))) FROM (SELECT materialize(range(10000)) AS big, range(100) AS r FROM numbers(100)) ARRAY JOIN r AS x SETTINGS enable_lazy_columns_replication = {lazy_replication} FORMAT Null + SELECT sum(big[x * 50]) FROM (SELECT materialize(range(10000)) AS big, range(100) AS r FROM numbers(100)) ARRAY JOIN r AS x SETTINGS enable_lazy_columns_replication = {lazy_replication} FORMAT Null + diff --git a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference new file mode 100644 index 0000000000000..20fd0846e32d9 --- /dev/null +++ b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference @@ -0,0 +1,171 @@ +-- numeric array, dynamic offset +1 0 9 [0,1] 10 +2 1 8 [1,2] 10 +5 4 5 [4,5] 10 +100 0 0 [] 10 +1 0 10 [0,1] 11 +2 1 9 [1,2] 11 +5 4 6 [4,5] 11 +100 0 0 [] 11 +1 0 9 [0,1] 10 +2 1 8 [1,2] 10 +5 4 5 [4,5] 10 +100 0 0 [] 10 +1 0 10 [0,1] 11 +2 1 9 [1,2] 11 +5 4 6 [4,5] 11 +100 0 0 [] 11 +-- string array, constant and dynamic slices +1 ['s1','s2','s3'] ['s3','s4'] ['s0','s1','s2','s3','s4'] ['s0'] s1 +-1 ['s1','s2','s3'] ['s3','s4'] ['s4'] [] s1 +0 ['s1','s2','s3'] ['s3','s4'] [] [] s1 +3 ['s1','s2','s3'] ['s3','s4'] ['s2','s3','s4'] ['s2','s3','s4'] s1 +1 ['s2','s3','s4'] ['s4','s5'] ['s1','s2','s3','s4','s5'] ['s1'] s2 +-1 ['s2','s3','s4'] ['s4','s5'] ['s5'] [] s2 +0 ['s2','s3','s4'] ['s4','s5'] [] [] s2 +3 ['s2','s3','s4'] ['s4','s5'] ['s3','s4','s5'] ['s3','s4','s5'] s2 +1 ['s1','s2','s3'] ['s3','s4'] ['s0','s1','s2','s3','s4'] ['s0'] s1 +-1 ['s1','s2','s3'] ['s3','s4'] ['s4'] [] s1 +0 ['s1','s2','s3'] ['s3','s4'] [] [] s1 +3 ['s1','s2','s3'] ['s3','s4'] ['s2','s3','s4'] ['s2','s3','s4'] s1 +1 ['s2','s3','s4'] ['s4','s5'] ['s1','s2','s3','s4','s5'] ['s1'] s2 +-1 ['s2','s3','s4'] ['s4','s5'] ['s5'] [] s2 +0 ['s2','s3','s4'] ['s4','s5'] [] [] s2 +3 ['s2','s3','s4'] ['s4','s5'] ['s3','s4','s5'] ['s3','s4','s5'] s2 +-- Array(LowCardinality(String)) elements and slices +1 Array(LowCardinality(String)) lc_0 lc_1 ['lc_0','lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +2 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1','lc_2'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +5 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +-1 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +1 Array(LowCardinality(String)) lc_1 lc_2 ['lc_1','lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +2 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2','lc_0'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +5 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +-1 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +1 Array(LowCardinality(String)) lc_0 lc_1 ['lc_0','lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +2 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1','lc_2'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +5 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +-1 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +1 Array(LowCardinality(String)) lc_1 lc_2 ['lc_1','lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +2 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2','lc_0'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +5 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +-1 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +-- nullable elements +1 \N \N [NULL,1] +2 1 1 [1,NULL] +7 \N \N [] +-1 5 5 [5] +1 \N \N [NULL,2] +2 2 2 [2,NULL] +7 \N \N [] +-1 6 6 [6] +1 \N \N [NULL,1] +2 1 1 [1,NULL] +7 \N \N [] +-1 5 5 [5] +1 \N \N [NULL,2] +2 2 2 [2,NULL] +7 \N \N [] +-1 6 6 [6] +-- negative and nullable offsets, negative length +1 [5,6] [1,2,3,4,5,6] [0,1] +2 [5,6] [1,2,3,4,5,6] [0,1] +3 [5,6] [1,2,3,4,5,6] [2,3] +-2 [5,6] [1,2,3,4,5,6] [0,1] +1 [6,7] [1,2,3,4,5,6,7] [0,1] +2 [6,7] [1,2,3,4,5,6,7] [0,1] +3 [6,7] [1,2,3,4,5,6,7] [2,3] +-2 [6,7] [1,2,3,4,5,6,7] [0,1] +1 [7,8] [1,2,3,4,5,6,7,8] [0,1] +2 [7,8] [1,2,3,4,5,6,7,8] [0,1] +3 [7,8] [1,2,3,4,5,6,7,8] [2,3] +-2 [7,8] [1,2,3,4,5,6,7,8] [0,1] +1 [5,6] [1,2,3,4,5,6] [0,1] +2 [5,6] [1,2,3,4,5,6] [0,1] +3 [5,6] [1,2,3,4,5,6] [2,3] +-2 [5,6] [1,2,3,4,5,6] [0,1] +1 [6,7] [1,2,3,4,5,6,7] [0,1] +2 [6,7] [1,2,3,4,5,6,7] [0,1] +3 [6,7] [1,2,3,4,5,6,7] [2,3] +-2 [6,7] [1,2,3,4,5,6,7] [0,1] +1 [7,8] [1,2,3,4,5,6,7,8] [0,1] +2 [7,8] [1,2,3,4,5,6,7,8] [0,1] +3 [7,8] [1,2,3,4,5,6,7,8] [2,3] +-2 [7,8] [1,2,3,4,5,6,7,8] [0,1] +-- tuple elements and replicated empty arrays +1 (0,'t0') [(0,'t0')] 0 [] +4 (3,'t3') [(0,'t0'),(1,'t1'),(2,'t2'),(3,'t3')] 0 [] +5 (0,'') [(0,'t0'),(1,'t1'),(2,'t2'),(3,'t3')] 0 [] +1 (1,'t0') [(1,'t0')] 0 [0] +4 (4,'t3') [(1,'t0'),(2,'t1'),(3,'t2'),(4,'t3')] 0 [0] +5 (0,'') [(1,'t0'),(2,'t1'),(3,'t2'),(4,'t3')] 0 [0] +1 (0,'t0') [(0,'t0')] 0 [] +4 (3,'t3') [(0,'t0'),(1,'t1'),(2,'t2'),(3,'t3')] 0 [] +5 (0,'') [(0,'t0'),(1,'t1'),(2,'t2'),(3,'t3')] 0 [] +1 (1,'t0') [(1,'t0')] 0 [0] +4 (4,'t3') [(1,'t0'),(2,'t1'),(3,'t2'),(4,'t3')] 0 [0] +5 (0,'') [(1,'t0'),(2,'t1'),(3,'t2'),(4,'t3')] 0 [0] +-- array produced by lazy JOIN replication +1 s0 ['s0','s1'] +2 s1 ['s1','s2'] +3 s2 ['s2','s3'] +4 s3 ['s3','s4'] +1 s0 ['s0','s1'] +2 s1 ['s1','s2'] +3 s2 ['s2','s3'] +4 s3 ['s3','s4'] +-- replication indexes of type UInt16 (more than 255 rows in a block) +90000 1800 90000 +90000 1800 90000 +-- arrayElement with a non-constant Nullable index +1 s0 s0 +2 \N \N +5 s4 s4 +-1 s4 s4 +1 s1 s1 +2 \N \N +5 s5 s5 +-1 s5 s5 +1 s0 s0 +2 \N \N +5 s4 s4 +-1 s4 s4 +1 s1 s1 +2 \N \N +5 s5 s5 +-1 s5 s5 +-- arrayElement with a lazily replicated Nullable index (AST fuzzer) +45150 0 0 +45150 0 0 +-- array and index replicated by the same expansion +1 1 [1,2] 1 +2 1 [1,2] 1 +5 1 [1,2] 1 +1 2 [2,3] 2 +2 2 [2,3] 2 +5 2 [2,3] 2 +1 1 [1,2] 1 +2 1 [1,2] 1 +5 1 [1,2] 1 +1 2 [2,3] 2 +2 2 [2,3] 2 +5 2 [2,3] 2 +-- unused nested rows are compacted away before the recursion on nested columns +1 0 1 [0,1] +2 0 1 [0,1] +1 1 1 [1,2] +2 1 1 [1,2] +1 0 1 [0,1] +2 0 1 [0,1] +1 1 1 [1,2] +2 1 1 [1,2] +-- out-of-order replication indexes (JOIN probe order) with unused nested rows +4 0 1 3 [0,1] +4 0 1 3 [0,1] +7 0 1 6 [0,1] +9 2 1 8 [2,3] +4 0 1 3 [0,1] +4 0 1 3 [0,1] +7 0 1 6 [0,1] +9 2 1 8 [2,3] +-- large array is not materialized per joined row +1 diff --git a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 new file mode 100644 index 0000000000000..abb626896ce1e --- /dev/null +++ b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 @@ -0,0 +1,126 @@ +-- arraySlice and arrayElement consume a lazily replicated array produced by ARRAY_JOIN or JOINs +-- Results must be identical with lazy replication disabled and enabled. + +SELECT '-- numeric array, dynamic offset'; +{% for i in range(2) -%} +SELECT x, big[x], big[-x], arraySlice(big, x, 2), length(big) +FROM (SELECT range(10 + number) AS big, [1, 2, 5, 100] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- string array, constant and dynamic slices'; +{% for i in range(2) -%} +SELECT x, arraySlice(big, 2, 3), arraySlice(big, -2), arraySlice(big, x), arraySlice(big, x, x), big[2] +FROM (SELECT arrayMap(i -> 's' || toString(i + number), range(5)) AS big, [1, -1, 0, 3] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- Array(LowCardinality(String)) elements and slices'; +{% for i in range(2) -%} +SELECT x, toTypeName(big), big[x], big[2], arraySlice(big, x, 2), arraySlice(big, 2, 3), arraySlice(big, -2) +FROM (SELECT arrayMap(i -> 'lc_' || toString((i + number) % 3), range(5))::Array(LowCardinality(String)) AS big, [1, 2, 5, -1] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- nullable elements'; +{% for i in range(2) -%} +SELECT x, big[x], arrayElementOrNull(big, x), arraySlice(big, x, 2) +FROM (SELECT arrayMap(i -> if(i % 2 = 0, NULL, i + number), range(6))::Array(Nullable(UInt64)) AS big, [1, 2, 7, -1] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- negative and nullable offsets, negative length'; +{% for i in range(2) -%} +SELECT x, arraySlice(big, -3, 2), arraySlice(big, 2, -1), arraySlice(big, if(x % 2 = 0, NULL, x)::Nullable(Int64), 2) +FROM (SELECT range(8 + number % 3) AS big, [1, 2, 3, -2] AS r FROM numbers(3)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- tuple elements and replicated empty arrays'; +{% for i in range(2) -%} +SELECT x, big[x], arraySlice(big, 1, x), small_arr[x], arraySlice(small_arr, 1, 2) +FROM (SELECT arrayMap(i -> (i + number, 't' || toString(i)), range(4)) AS big, range(number % 2) AS small_arr, [1, 4, 5] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- array produced by lazy JOIN replication'; +{% for i in range(2) -%} +SELECT x, big[x], arraySlice(big, x, 2) +FROM (SELECT 42 AS k, number + 1 AS x FROM numbers(4)) AS l +INNER JOIN (SELECT 42 AS k, materialize(arrayMap(i -> 's' || toString(i), range(5))) AS big) AS r ON l.k = r.k +ORDER BY x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- replication indexes of type UInt16 (more than 255 rows in a block)'; +{% for i in range(2) -%} +SELECT sum(big[x + 1]), sum(length(arraySlice(big, x + 1, 3))), sum(arraySlice(big, x + 1, 3)[1]) +FROM (SELECT range(number, number + 10) AS big, range(2) AS r FROM numbers(300)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- arrayElement with a non-constant Nullable index'; +{% for i in range(2) -%} +SELECT x, big[if(x % 2 = 0, NULL, x)::Nullable(Int64)], arrayElementOrNull(big, if(x % 2 = 0, NULL, x)::Nullable(Int64)) +FROM (SELECT arrayMap(i -> 's' || toString(i + number), range(5)) AS big, [1, 2, 5, -1] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- arrayElement with a lazily replicated Nullable index (AST fuzzer)'; +{% for i in range(2) -%} +SELECT DISTINCT sum(big[2 + x]), quantileTDigestDistinct(notEmpty(arraySlice(big, positiveModuloOrNull(x, 65535), 3))), sum(arraySlice(big, NULL + x, 3)[materialize(toNullable(100))]) +FROM (SELECT DISTINCT range(number, number + 2) AS big, range(2) AS r FROM numbers(300)) LEFT ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1; +{% endfor %} + +SELECT '-- array and index replicated by the same expansion'; +-- `big` and `idx` are both non-joined columns, so lazy ARRAY JOIN wraps them in ColumnReplicated +-- sharing one indexes column: `big[idx]` takes the compute-on-nested fast path and stays lazy. +-- `range(x + 3)[idx]` mixes a full array (built from the expanded column) with a replicated index, +-- which pins the materialize fallback for an index replicated independently of the array. +{% for i in range(2) -%} +SELECT x, big[idx], arraySlice(big, idx, 2), range(x + 3)[idx] +FROM (SELECT range(10 + number) AS big, number + 2 AS idx, [1, 2, 5] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- unused nested rows are compacted away before the recursion on nested columns'; +-- Only 2 of the 20 source rows have a non-empty `r`, so the shared replication indexes reference +-- only those nested rows: the compute-on-nested path must compact the unused ones away +-- and adjust the indexes, for both a replicated (`big[idx]`) and a constant (`big[2]`) index. +{% for i in range(2) -%} +SELECT x, big[idx], big[2], arraySlice(big, idx, 2) +FROM (SELECT range(10 + number) AS big, intDiv(number, 10) + 1 AS idx, if(number % 10 = 0, [1, 2], []) AS r FROM numbers(20)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- out-of-order replication indexes (JOIN probe order) with unused nested rows'; +-- A hash JOIN references build-side rows in probe order: left keys [9, 4, 7, 4] give a +-- non-monotonic index sequence with a repeat (e.g. [8, 3, 6, 3]) over 10 nested rows, +-- 7 of which are never referenced. The used rows are not a prefix, so the compaction must +-- remap the surviving indexes to new positions ([2, 0, 1, 0]). This covers index remapping +-- for the shared-indexes (`big[idx]`) and constant-index (`big[2]`) paths, and the +-- random-access gather (`big[l.k]`: full index over a replicated array). +{% for i in range(2) -%} +SELECT l.k, big[idx], big[2], big[l.k], arraySlice(big, idx, 2) +FROM (SELECT arrayJoin([9, 4, 7, 4]) AS k) AS l +INNER JOIN (SELECT number + 1 AS k, range(number + 3) AS big, number % 3 + 1 AS idx FROM numbers(10)) AS r ON l.k = r.k +ORDER BY ALL +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, join_algorithm = 'hash', query_plan_join_swap_table = 0; +{% endfor %} + +SELECT '-- large array is not materialized per joined row'; +{% for i in range(2) -%} +SELECT sum(length(arraySlice(big, x, 2))) + sum(length(big[x])) +FROM (SELECT arrayMap(i -> repeat('a', 100) || toString(i + number), range(1000)) AS big, range(50) AS r FROM numbers(100)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, max_block_size = 65536, log_queries = 1, log_comment = '04628_replicated_{{ ['eager', 'lazy'][i] }}' +FORMAT Null; +{% endfor %} + +SYSTEM FLUSH LOGS query_log; + +-- With lazy replication the big array stays compact (one copy per source row) and only the +-- requested slices/elements are copied, so the peak memory usage must be much lower. +SELECT + (SELECT max(memory_usage) FROM system.query_log WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '04628_replicated_lazy') + < 0.5 * (SELECT max(memory_usage) FROM system.query_log WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '04628_replicated_eager'); From 5fda4035bae18793328fd5a8918eb61bfed1015c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Oct 2026 21:50:56 +0000 Subject: [PATCH 233/296] Backport #124249 to 26.8: Bump `cctz` from 2026d to 2026e --- contrib/cctz | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/cctz b/contrib/cctz index bfd13df99e265..fa5fe114a87a3 160000 --- a/contrib/cctz +++ b/contrib/cctz @@ -1 +1 @@ -Subproject commit bfd13df99e26583c1a00edc5032964b01bab551b +Subproject commit fa5fe114a87a34c055853adf54dde305979acea0 From 3e1d7df7e4db4e175953351acbb7b3035e6bc9d7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Oct 2026 00:31:31 +0000 Subject: [PATCH 234/296] Backport #114346 to 26.8: Add test coverage for arrayElement compute-on-nested branches and simplify its type dispatch --- src/Functions/array/arrayElement.cpp | 149 ++++++------------ ...y_replicated_array_slice_element.reference | 14 ++ ...lazy_replicated_array_slice_element.sql.j2 | 33 ++++ 3 files changed, 98 insertions(+), 98 deletions(-) diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index 5187fd27484d0..441ed5c1d76da 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -61,6 +62,27 @@ template class NullMapBuilder; } +/// Array element types with a dedicated fast path. The dispatch ladders below try them in this +/// order, so the most common types come first; the generic fallback handles everything else. +using ElementTypesWithFastPath = TypeList< + UInt8, UInt16, UInt32, UInt64, + Int8, Int16, Int32, Int64, + Float32, Float64, + Decimal32, Decimal64, Decimal128, Decimal256, + DateTime64, + Int128, UInt128, Int256, UInt256>; + +/// Types the index (second) argument may have, in dispatch order. +using IndexTypes = TypeList; + +/// Calls try_one for each T in the list until one call succeeds. The fold expands to the same +/// short-circuit `||` chain as spelling out one call per type. +template +bool tryEachType(TypeList, F && try_one) +{ + return (try_one.template operator()() || ...); +} + /** arrayElement(arr, i) - get the array element by index. If index is not constant and out of range - return default value of data type. * The index begins with 1. Also, the index can be negative - then it is counted from the end of the array. @@ -1531,48 +1553,18 @@ ColumnPtr FunctionArrayElement::executeArgument( builder.initSink(index_data.size()); ColumnPtr res; - if (!((res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeNumber(arguments, result_type, index_data, builder)) - || (res = executeConst(arguments, result_type, index_data, builder, input_rows_count)) - || (res = executeString(arguments, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayNumber(arguments, result_type, index_data, builder)) - || (res = executeArrayString(arguments, index_data, builder)) - || (res = executeGeneric(arguments, index_data, builder)))) + bool matched = tryEachType( + ElementTypesWithFastPath{}, + [&]() { return (res = executeNumber(arguments, result_type, index_data, builder)) != nullptr; }) + || (res = executeConst(arguments, result_type, index_data, builder, input_rows_count)) + || (res = executeString(arguments, index_data, builder)) + || tryEachType( + ElementTypesWithFastPath{}, + [&]() { return (res = executeArrayNumber(arguments, result_type, index_data, builder)) != nullptr; }) + || (res = executeArrayString(arguments, index_data, builder)) + || (res = executeGeneric(arguments, index_data, builder)); + + if (!matched) throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName()); @@ -3245,14 +3237,9 @@ ColumnPtr FunctionArrayElement::executeReplicated( const auto & replication_indexes = replicated_array->getIndexes(); const auto & index_column = *args[1].column; /// Core loop to build result by dispatching based on the index type - if (!(gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) - || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) - || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) - || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) - || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) - || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) - || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder) - || gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder))) + if (!tryEachType( + IndexTypes{}, + [&]() { return gatherReplicated(index_column, replication_indexes, offsets, *data, *result, builder); })) { /// The index is not a plain numeric column (e.g. Nullable, or an array of indexes): /// materialize the array and let the generic path handle it. @@ -3328,14 +3315,9 @@ ColumnPtr FunctionArrayElement::perform( return res; if (!isColumnConst(*arguments[1].column)) { - if (!((res = executeArgument(arguments, result_type, builder, input_rows_count)) - || (res = executeArgument(arguments, result_type, builder, input_rows_count)) - || (res = executeArgument(arguments, result_type, builder, input_rows_count)) - || (res = executeArgument(arguments, result_type, builder, input_rows_count)) - || (res = executeArgument(arguments, result_type, builder, input_rows_count)) - || (res = executeArgument(arguments, result_type, builder, input_rows_count)) - || (res = executeArgument(arguments, result_type, builder, input_rows_count)) - || (res = executeArgument(arguments, result_type, builder, input_rows_count)))) + if (!tryEachType( + IndexTypes{}, + [&]() { return (res = executeArgument(arguments, result_type, builder, input_rows_count)) != nullptr; })) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Second argument for function {} must have UInt or Int type", getName()); } else @@ -3359,46 +3341,17 @@ ColumnPtr FunctionArrayElement::perform( } } - if (!((res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeNumberConst(arguments, result_type, index, builder)) - || (res = executeStringConst(arguments, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayNumberConst(arguments, result_type, index, builder)) - || (res = executeArrayStringConst(arguments, index, builder)) || (res = executeGenericConst(arguments, index, builder)))) + bool matched = tryEachType( + ElementTypesWithFastPath{}, + [&]() { return (res = executeNumberConst(arguments, result_type, index, builder)) != nullptr; }) + || (res = executeStringConst(arguments, index, builder)) + || tryEachType( + ElementTypesWithFastPath{}, + [&]() { return (res = executeArrayNumberConst(arguments, result_type, index, builder)) != nullptr; }) + || (res = executeArrayStringConst(arguments, index, builder)) + || (res = executeGenericConst(arguments, index, builder)); + + if (!matched) throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", diff --git a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference index 20fd0846e32d9..66efb62385cac 100644 --- a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference +++ b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference @@ -167,5 +167,19 @@ 4 0 1 3 [0,1] 7 0 1 6 [0,1] 9 2 1 8 [2,3] +-- constant index over a replicated array that survives a LIMIT (const-index compute-on-nested branch) +1 10 +1 10 +1 10 +1 10 +1 10 +1 10 +-- array and index lazily indexed by a JOIN under LIMIT (shared-indexes compute-on-nested branch) +0 1 +0 1 +2 1 +0 1 +0 1 +2 1 -- large array is not materialized per joined row 1 diff --git a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 index abb626896ce1e..8167e5a1ad992 100644 --- a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 +++ b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 @@ -109,6 +109,39 @@ ORDER BY ALL SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, join_algorithm = 'hash', query_plan_join_swap_table = 0; {% endfor %} +SELECT '-- constant index over a replicated array that survives a LIMIT (const-index compute-on-nested branch)'; +-- Source-only expressions are normally computed below the expansion, so `big[2]` never sees a +-- replicated array there. The LIMIT barrier keeps the projection above the ARRAY JOIN, and `big` +-- crosses it as `ColumnReplicated`. `compile_expressions = 0` keeps the call out of the JIT +-- (the harness may randomize `min_count_to_compile_expression` to 0). +{% for i in range(2) -%} +SELECT big[2], length(big) +FROM (SELECT big FROM (SELECT range(10 + number) AS big, if(number % 10 = 0, [1, 2, 3], []) AS r FROM numbers(20)) ARRAY JOIN r AS x LIMIT 3) +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, enable_parallel_replicas = 0, compile_expressions = 0; +{% endfor %} + +SELECT '-- array and index lazily indexed by a JOIN under LIMIT (shared-indexes compute-on-nested branch)'; +-- A LIMIT (<= query_plan_max_limit_for_join_lazy_indexing) above a hash JOIN with >= 3 probe-side +-- columns makes the join wrap all probe columns in `ColumnReplicated` sharing one indexes column +-- (`optimizeJoinLazyIndexing`), including the fixed-size `idx` that the type-based usefulness +-- heuristic would otherwise materialize. Only 3 of the 20 probe rows are matched, so `big[idx]` +-- (shared-indexes branch) and `big[2]` (constant-index branch) must both compact 17 unused nested +-- rows before the recursion on the nested columns. +DROP TABLE IF EXISTS t04628_left; +DROP TABLE IF EXISTS t04628_right; +CREATE TABLE t04628_left (k UInt64, big Array(UInt64), idx UInt64) ENGINE = Memory; +CREATE TABLE t04628_right (k UInt64) ENGINE = Memory; +INSERT INTO t04628_left SELECT number + 1, range(number + 3), number % 3 + 1 FROM numbers(20); +INSERT INTO t04628_right VALUES (9), (4), (7); +{% for i in range(2) -%} +SELECT big[idx], big[2] +FROM t04628_left AS l INNER JOIN t04628_right AS r ON l.k = r.k +LIMIT 100 +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, join_algorithm = 'hash', query_plan_join_swap_table = 0, enable_parallel_replicas = 0, compile_expressions = 0, query_plan_optimize_join_order_limit = 10, query_plan_optimize_join_order_randomize = 0, enable_join_runtime_filters = 0; +{% endfor %} +DROP TABLE t04628_left; +DROP TABLE t04628_right; + SELECT '-- large array is not materialized per joined row'; {% for i in range(2) -%} SELECT sum(length(arraySlice(big, x, 2))) + sum(length(big[x])) From 3403f538fa7feb7e07a531d1b62db2eba050b968 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Oct 2026 03:25:53 +0000 Subject: [PATCH 235/296] Backport #120224 to 26.8: Fix Bad cast from ColumnReplicated to ColumnQBit in arraySlice --- src/Functions/array/arraySlice.cpp | 3 ++- ...y_replicated_array_slice_element.reference | 15 ++++++++++++ ...lazy_replicated_array_slice_element.sql.j2 | 24 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/Functions/array/arraySlice.cpp b/src/Functions/array/arraySlice.cpp index 521b7cb78094e..1d6f0a7d88bf4 100644 --- a/src/Functions/array/arraySlice.cpp +++ b/src/Functions/array/arraySlice.cpp @@ -323,7 +323,8 @@ class FunctionArraySlice final : public IFunction if (bounds.start == 0 && bounds.length == dimension) return arguments[0].column; - ColumnPtr column = arguments[0].column; + /// The bit-plane streams are read directly and QBit has no replicated source, so a replicated argument is materialized. + ColumnPtr column = arguments[0].column->convertToFullColumnIfReplicated(); bool is_const = false; if (const auto * const_column = checkAndGetColumn(column.get())) diff --git a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference index 66efb62385cac..f40b28dde3758 100644 --- a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference +++ b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference @@ -181,5 +181,20 @@ 0 1 0 1 2 1 +-- QBit sliced above a lazy ARRAY JOIN expansion +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +1 [9,10,11,12,13,14,15,16] [2,3,4,5,6,7,8,9,10,11] +1 [9,10,11,12,13,14,15,16] [2,3,4,5,6,7,8,9,10,11] +2 \N \N +2 \N \N +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +1 [9,10,11,12,13,14,15,16] [2,3,4,5,6,7,8,9,10,11] +1 [9,10,11,12,13,14,15,16] [2,3,4,5,6,7,8,9,10,11] +2 \N \N +2 \N \N -- large array is not materialized per joined row 1 diff --git a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 index 8167e5a1ad992..51b799aaa7e5f 100644 --- a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 +++ b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 @@ -142,6 +142,30 @@ SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, join_algori DROP TABLE t04628_left; DROP TABLE t04628_right; +SELECT '-- QBit sliced above a lazy ARRAY JOIN expansion'; +-- `arraySlice` opts out of the framework's replicated materialization and dispatches on the QBit type +-- before any wrapper is peeled, so its QBit branch receives the `ColumnReplicated` the expansion built: +-- `Bad cast from type DB::ColumnReplicated to DB::ColumnQBit` in `executeQBitSlice`. +-- A QBit slice needs constant offset and length, so only the LIMIT barrier keeps it above the expansion. +DROP TABLE IF EXISTS t04628_qbit; +DROP TABLE IF EXISTS t04628_qbit_nullable; +CREATE TABLE t04628_qbit (id UInt32, vec QBit(Float32, 32, 8)) ENGINE = Memory; +CREATE TABLE t04628_qbit_nullable (id UInt32, vec Nullable(QBit(Float32, 32, 8))) ENGINE = Memory; +INSERT INTO t04628_qbit SELECT 1, arrayMap(i -> toFloat32(i + 1), range(32))::QBit(Float32, 32, 8); +INSERT INTO t04628_qbit_nullable SELECT 1, arrayMap(i -> toFloat32(i + 1), range(32))::QBit(Float32, 32, 8); +INSERT INTO t04628_qbit_nullable SELECT 2, NULL; +{% for i in range(2) -%} +SELECT arraySlice(vec, 9, 8) AS aligned, toTypeName(aligned), arraySlice(vec, 2, 10) AS repacked, toTypeName(repacked), arraySlice(vec, 1) AS whole, vec[3] +FROM (SELECT vec FROM t04628_qbit ARRAY JOIN [1, 2, 3] AS z LIMIT 3) +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, enable_parallel_replicas = 0, compile_expressions = 0; +SELECT id, arraySlice(vec, 9, 8) AS aligned, arraySlice(vec, 2, 10) AS repacked +FROM (SELECT id, vec FROM t04628_qbit_nullable ARRAY JOIN [1, 2] AS z LIMIT 4) +ORDER BY id +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, enable_parallel_replicas = 0, compile_expressions = 0; +{% endfor %} +DROP TABLE t04628_qbit; +DROP TABLE t04628_qbit_nullable; + SELECT '-- large array is not materialized per joined row'; {% for i in range(2) -%} SELECT sum(length(arraySlice(big, x, 2))) + sum(length(big[x])) From 866273b034d84720226e99b1d53e3d9d13893129 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Oct 2026 06:41:05 +0000 Subject: [PATCH 236/296] Backport #119184 to 26.8: Fix DeltaLake stats pruning with Date cast --- src/Parsers/ASTHelpers.h | 10 +- .../DataLakes/DeltaLake/EnginePredicate.cpp | 253 +++++++++++- .../DataLakes/DeltaLake/EnginePredicate.h | 4 + .../DataLakes/DeltaLake/TableSnapshot.cpp | 9 +- tests/integration/test_storage_delta/test.py | 6 +- ...elta_lake_cast_predicate_pruning.reference | 234 ++++++++++++ ...05153_delta_lake_cast_predicate_pruning.sh | 360 ++++++++++++++++++ 7 files changed, 849 insertions(+), 27 deletions(-) create mode 100644 tests/queries/0_stateless/05153_delta_lake_cast_predicate_pruning.reference create mode 100755 tests/queries/0_stateless/05153_delta_lake_cast_predicate_pruning.sh diff --git a/src/Parsers/ASTHelpers.h b/src/Parsers/ASTHelpers.h index 0b3db8e02d5c0..ce44fe6beaf27 100644 --- a/src/Parsers/ASTHelpers.h +++ b/src/Parsers/ASTHelpers.h @@ -1,16 +1,20 @@ #pragma once #include +#include namespace DB { +static inline bool isFunctionCast(std::string_view name) /// NOLINT +{ + return name == "CAST" || name == "_CAST"; +} + static inline bool isFunctionCast(const ASTFunction * function) /// NOLINT { - if (function) - return function->name == "CAST" || function->name == "_CAST"; - return false; + return function && isFunctionCast(function->name); } diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/EnginePredicate.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLake/EnginePredicate.cpp index 8b84af8161cf0..3089083dffc54 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/EnginePredicate.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/EnginePredicate.cpp @@ -7,18 +7,26 @@ #include #include +#include #include #include #include +#include #include #include #include #include +#include #include #include +#include #include +#include +#include +#include + namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; @@ -69,6 +77,108 @@ namespace const auto * nullable = assert_cast(node->result_type.get()); return nullable->getNestedType(); } + + /// Kernel visitor for `left OP right` over two expression IDs: + /// `visit_predicate_eq`, `_ne`, `_lt`, `_le`, `_gt`, `_ge`. + using ComparisonVisitor = uintptr_t (*)(ffi::KernelExpressionVisitorState *, uintptr_t, uintptr_t); + + struct ComparisonVisitors + { + ComparisonVisitor direct; + /// Reads the comparison with its operands swapped (`literal OP column`). + ComparisonVisitor mirrored; + + ComparisonVisitor forOperands(bool swapped) const + { + return swapped ? mirrored : direct; + } + }; + + std::optional getComparisonVisitors(const String & name) + { + if (name == DB::NameEquals::name) + return ComparisonVisitors{&ffi::visit_predicate_eq, &ffi::visit_predicate_eq}; + if (name == DB::NameNotEquals::name) + return ComparisonVisitors{&ffi::visit_predicate_ne, &ffi::visit_predicate_ne}; + if (name == DB::NameLess::name) + return ComparisonVisitors{&ffi::visit_predicate_lt, &ffi::visit_predicate_gt}; + if (name == DB::NameLessOrEquals::name) + return ComparisonVisitors{&ffi::visit_predicate_le, &ffi::visit_predicate_ge}; + if (name == DB::NameGreater::name) + return ComparisonVisitors{&ffi::visit_predicate_gt, &ffi::visit_predicate_lt}; + if (name == DB::NameGreaterOrEquals::name) + return ComparisonVisitors{&ffi::visit_predicate_ge, &ffi::visit_predicate_le}; + return std::nullopt; + } + + struct DateComparison + { + const DB::ActionsDAG::Node * column; + Int32 day; + bool narrows_to_date; + ComparisonVisitor visitor; + }; + + /// Matches `CAST`/`toDate`/`toDate32` on a `Date32` column against a literal, in either operand order. + /// Converts the literal and selects its visitor; returns nullopt when translation is unsafe. + std::optional matchDateComparison( + const DB::ActionsDAG::Node * node, + const ComparisonVisitors & visitors, + const DB::FormatSettings & format_settings) + { + const auto * conversion = node->children[0]; + const auto * literal = node->children[1]; + bool swapped = false; + if (isConstNode(conversion)) + { + std::swap(conversion, literal); + swapped = true; + } + if (!isFunctionNode(conversion) || !isConstNode(literal)) + return std::nullopt; + + const auto & name = conversion->function_base->getName(); + const auto & children = conversion->children; + const bool is_cast = DB::isFunctionCast(name) && children.size() == 2 && isConstNode(children[1]); + /// The optional constant timezone of `toDate` / `toDate32` is inert for a `Date32` source. + const bool is_to_date = (name == "toDate" || name == "toDate32") + && (children.size() == 1 || (children.size() == 2 && isConstNode(children[1]))); + if (!is_cast && !is_to_date) + return std::nullopt; + + const auto * column = children[0]; + const auto result_type = getTypeOrNestedType(conversion); + if (!isColumnNode(column) || getTypeIndex(column) != DB::TypeIndex::Date32 + || !DB::isDateOrDate32(result_type->getTypeId())) + return std::nullopt; + + /// Removing nullability can throw; pruning must not hide those rows. + if (column->result_type->isNullable() && !conversion->result_type->isNullable()) + return std::nullopt; + + /// Mixed temporal comparisons use a common type; a `Date`/`Date32` literal converts exactly or not at all (rejected below), anything else stays untranslated. + const auto literal_type = getTypeOrNestedType(literal); + if (!literal_type->equals(*result_type) && !isString(literal_type) + && !DB::isDateOrDate32(literal_type->getTypeId())) + return std::nullopt; + + /// Convert the literal exactly as the real comparison does: under the query's format settings + /// (notably `date_time_overflow_behavior`), not the default `ignore`. `FunctionComparison`'s + /// `executeWithConstString` converts the constant string with `params.format_settings`, so a + /// bare `tryConvertFieldToType` here would translate e.g. + /// `toDate(d) = '1969-12-31' SETTINGS date_time_overflow_behavior='throw'` as if the literal + /// had been clamped, and could prune away the exception the comparison must raise. Under + /// `throw` an out-of-range literal makes this return null, so the file stays unpruned. + const auto value = DB::tryConvertFieldToType(literal->column->getField(), *result_type, literal_type.get(), format_settings); + if (value.isNull()) + return std::nullopt; + + return DateComparison{ + column, + static_cast(value.safeGet()), + result_type->getTypeId() == DB::TypeIndex::Date, + visitors.forOperands(swapped)}; + } } std::shared_ptr getEnginePredicate( @@ -215,6 +325,10 @@ class EngineIterator : public ffi::EngineIterator } static uintptr_t getNextImpl(EngineIteratorData & iterator_data, const DB::ActionsDAG::Node * node); + + static uintptr_t visitComparisonOverDateConversion( + EngineIteratorData & iterator_data, + const DateComparison & comparison); }; uintptr_t EnginePredicate::visitPredicate(void * data, ffi::KernelExpressionVisitorState * state) @@ -314,6 +428,120 @@ static uintptr_t visitLiteralValue( } } +enum class Junction { And, Or }; + +/// Builds an AND / OR over two predicate IDs. +static uintptr_t visitJunction( + ffi::KernelExpressionVisitorState * state, + Junction junction, + std::array ids) +{ + /// The kernel consumes the IDs synchronously and does not retain the iterator. + struct Cursor + { + const std::array & ids; + size_t next = 0; + }; + Cursor cursor{ids}; + + ffi::EngineIterator iterator + { + .data = &cursor, + .get_next = [](void * data) -> const void * + { + auto & current = *static_cast(data); + if (current.next == current.ids.size()) + return nullptr; + return reinterpret_cast(current.ids[current.next++]); + }, + }; + return junction == Junction::And + ? ffi::visit_predicate_and(state, &iterator) + : ffi::visit_predicate_or(state, &iterator); +} + +/// Builds predicates for one column and validates every kernel ID. +struct ColumnPredicateBuilder +{ + ffi::KernelExpressionVisitorState * visitor_state; + const DB::ActionsDAG::Node * column_node; + DB::DataTypePtr column_type; + + uintptr_t compare(ComparisonVisitor visitor, Int32 day) const + { + /// Comparisons consume expression IDs, so register a fresh column and literal each time. + auto column_id = assertValid( + KernelUtils::unwrapResult( + ffi::visit_expression_column( + visitor_state, + KernelUtils::toDeltaString(column_node->result_name), + &KernelUtils::allocateError), + "visit_expression_column"), + "column"); + auto literal_id = assertValid( + visitLiteralValue( + DB::Field(Int64(day)), column_type->getTypeId(), column_type, visitor_state), + "date literal"); + return assertValid(visitor(visitor_state, column_id, literal_id), "date comparison"); + } + + uintptr_t andOf(uintptr_t left, uintptr_t right) const + { + return junction(Junction::And, left, right, "AND"); + } + + uintptr_t orOf(uintptr_t left, uintptr_t right) const + { + return junction(Junction::Or, left, right, "OR"); + } + +private: + uintptr_t junction( + Junction kind, uintptr_t left, uintptr_t right, std::string_view description) const + { + /// A zero child would end the iterator early and silently shrink the junction. + assertValid(left, description); + assertValid(right, description); + return assertValid( + visitJunction(visitor_state, kind, {left, right}), description); + } + + uintptr_t assertValid(uintptr_t id, std::string_view description) const + { + if (!id || id == EngineIterator::VISITOR_FAILED_OR_UNSUPPORTED) + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, + "delta-kernel rejected {} predicate on column `{}`", + description, column_node->result_name); + return id; + } +}; + +uintptr_t EngineIterator::visitComparisonOverDateConversion( + EngineIteratorData & iterator_data, + const DateComparison & comparison) +{ + ColumnPredicateBuilder predicates{ + iterator_data.state, comparison.column, getTypeOrNestedType(comparison.column)}; + + auto compared = predicates.compare(comparison.visitor, comparison.day); + if (!comparison.narrows_to_date) + return compared; + + /// `Date32` -> `Date` can wrap, saturate, or throw outside the Date domain. + /// Preserve outside-domain rows: (inside AND comparison) OR (outside AND Unknown). + auto is_date_range = predicates.andOf( + predicates.compare(ffi::visit_predicate_ge, 0), + predicates.compare(ffi::visit_predicate_le, DATE_LUT_MAX_DAY_NUM)); + auto outside_date_range = predicates.orOf( + predicates.compare(ffi::visit_predicate_lt, 0), + predicates.compare(ffi::visit_predicate_gt, DATE_LUT_MAX_DAY_NUM)); + auto unknown_outside_range = + predicates.andOf(outside_date_range, visitUntranslated(iterator_data)); + return predicates.orOf( + predicates.andOf(is_date_range, compared), unknown_outside_range); +} + uintptr_t EngineIterator::getNextImpl(EngineIteratorData & iterator_data, const DB::ActionsDAG::Node * node) { if (iterator_data.hasException()) @@ -372,12 +600,7 @@ uintptr_t EngineIterator::getNextImpl(EngineIteratorData & iterator_data, const return ffi::visit_predicate_not(iterator_data.state, column); } } - else if (func_name == DB::NameEquals::name - || func_name == DB::NameNotEquals::name - || func_name == DB::NameGreater::name - || func_name == DB::NameGreaterOrEquals::name - || func_name == DB::NameLess::name - || func_name == DB::NameLessOrEquals::name) + else if (auto visitors = getComparisonVisitors(func_name)) { if (node->children.size() != 2) { @@ -468,19 +691,13 @@ uintptr_t EngineIterator::getNextImpl(EngineIteratorData & iterator_data, const return VISITOR_FAILED_OR_UNSUPPORTED; } - if (func_name == DB::NameEquals::name) - return ffi::visit_predicate_eq(iterator_data.state, column, constant); - if (func_name == DB::NameNotEquals::name) - return ffi::visit_predicate_ne(iterator_data.state, column, constant); - if (func_name == DB::NameGreater::name) - return ffi::visit_predicate_gt(iterator_data.state, column, constant); - if (func_name == DB::NameGreaterOrEquals::name) - return ffi::visit_predicate_ge(iterator_data.state, column, constant); - if (func_name == DB::NameLess::name) - return ffi::visit_predicate_lt(iterator_data.state, column, constant); - if (func_name == DB::NameLessOrEquals::name) - return ffi::visit_predicate_le(iterator_data.state, column, constant); + auto visitor = visitors->forOperands(isConstNode(node->children[0])); + return visitor(iterator_data.state, column, constant); } + + if (auto comparison = matchDateComparison( + node, *visitors, iterator_data.predicate.getFormatSettings())) + return visitComparisonOverDateConversion(iterator_data, *comparison); } break; diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/EnginePredicate.h b/src/Storages/ObjectStorage/DataLakes/DeltaLake/EnginePredicate.h index 4ff17685a2f45..126e295cf6507 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/EnginePredicate.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/EnginePredicate.h @@ -5,6 +5,7 @@ #if USE_DELTA_KERNEL_RS #include #include +#include #include @@ -32,6 +33,7 @@ class EnginePredicate : public ffi::EnginePredicate : filter(filter_) , exception(exception_) , context(context_) + , format_settings(context_ ? DB::getFormatSettings(context_) : DB::FormatSettings{}) { predicate = this; visitor = &visitPredicate; @@ -48,6 +50,7 @@ class EnginePredicate : public ffi::EnginePredicate const DB::ActionsDAG & getFilterDAG() const { return filter; } DB::ContextPtr getContext() const { return context; } + const DB::FormatSettings & getFormatSettings() const { return format_settings; } private: const LoggerPtr log = getLogger("EnginePredicate"); @@ -60,6 +63,7 @@ class EnginePredicate : public ffi::EnginePredicate std::exception_ptr & exception; /// Context for accessing settings DB::ContextPtr context; + const DB::FormatSettings format_settings; static uintptr_t visitPredicate(void * data, ffi::KernelExpressionVisitorState * state); }; diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp index ccaec596d9849..5c14fcf47aec9 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp @@ -66,7 +66,7 @@ namespace DB::FailPoints namespace DeltaLake { -class TableSnapshot::Iterator final : public DB::IObjectIterator +class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::WithContext { private: /// Struct to hold ObjectInfo along with FFI handles for lazy parsing @@ -103,9 +103,11 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator bool enable_expression_visitor_logging_, bool throw_on_engine_predicate_error_, bool enable_engine_predicate_, + DB::ContextPtr query_context_, UpdateStatsFunc update_stats_func_, LoggerPtr log_) - : kernel_snapshot_state(kernel_snapshot_state_) + : DB::WithContext(query_context_) + , kernel_snapshot_state(kernel_snapshot_state_) , captured_credentials_fingerprint(helper_->getCredentialsFingerprint()) , helper(helper_) , read_schema(read_schema_) @@ -181,7 +183,7 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator { if (filter.has_value() && enable_engine_predicate) { - auto predicate = getEnginePredicate(filter.value(), engine_predicate_exception, nullptr); + auto predicate = getEnginePredicate(filter.value(), engine_predicate_exception, getContext()); scan = KernelUtils::unwrapResult( ffi::scan( kernel_snapshot_state->snapshot.get(), @@ -999,6 +1001,7 @@ DB::ObjectIterator TableSnapshot::iterate( settings[DB::Setting::delta_lake_enable_expression_visitor_logging], settings[DB::Setting::delta_lake_throw_on_engine_predicate_error], settings[DB::Setting::delta_lake_enable_engine_predicate], + context, std::move(update_stats_func), log); } diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index efc207fa0d186..e04c74cbeedea 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -3280,14 +3280,14 @@ def test_delta_kernel_internal_pruning(started_cluster): ) ) - assert result == 1 + assert result == 3 instance.query("SYSTEM FLUSH LOGS") - assert 1 == int( + assert 3 == int( instance.query( f"SELECT count() FROM system.text_log WHERE query_id = '{query_id}' and message ILIKE '%Scanned file%'" ) ) - assert 1 == int( + assert 3 == int( instance.query( f"SELECT count() FROM system.text_log WHERE query_id = '{query_id}' and message ILIKE '%Scanned file: {TABLE_NAME}/b=test2%'" ) diff --git a/tests/queries/0_stateless/05153_delta_lake_cast_predicate_pruning.reference b/tests/queries/0_stateless/05153_delta_lake_cast_predicate_pruning.reference new file mode 100644 index 0000000000000..76ad04e005400 --- /dev/null +++ b/tests/queries/0_stateless/05153_delta_lake_cast_predicate_pruning.reference @@ -0,0 +1,234 @@ +t: d = '2026-01-01' +['2026-01-01'] +1 +t: CAST(d AS Date) = '2026-01-01' +['2026-01-01','2205-06-08'] +2 +t: d::Date = '2026-01-01' +['2026-01-01','2205-06-08'] +2 +t: toDate(d) = '2026-01-01' +['2026-01-01','2205-06-08'] +2 +t: d::Date32 = '2026-01-01' +['2026-01-01'] +1 +t: toDate32(d) = toDate32('2026-01-01') +['2026-01-01'] +1 +t: d::Date != '2026-01-01' +['2026-01-02'] +2 +t: toDate('2026-01-01') = toDate(d) +['2026-01-01','2205-06-08'] +2 +t: toDate('2026-01-01') != toDate(d) +['2026-01-02'] +2 +t: toDate(d) = '2026-01-03' +[] +1 +t: toDate32(d) = '2026-01-03' +[] +0 +lower: toDate(d) = '1970-01-01' SETTINGS date_time_overflow_behavior='saturate' +['1969-12-31','1970-01-01'] +2 +lower: toDate(d) != '1970-01-01' SETTINGS date_time_overflow_behavior='saturate' +['1970-01-02'] +2 +upper: CAST(d AS Date) = '2149-06-06' SETTINGS date_time_overflow_behavior='saturate' +['2149-06-06','2149-06-07'] +2 +upper: CAST(d AS Date) != '2149-06-06' SETTINGS date_time_overflow_behavior='saturate' +['2149-06-05'] +2 +negation: NOT (toDate(d) = '2026-01-01' AND d > toDate32('1970-01-01')) +['2026-01-02','2205-06-09'] +2 +aliases: toDate(d) = '2026-01-01' +['1846-07-28','2384-11-12'] +2 +aliases: toDate(d) != '2026-01-01' +['2026-01-02'] +3 +valid: toDate(d) != toDateTime('2026-01-01 00:00:01', 'UTC') +['1970-01-02','2026-01-01'] +2 +valid: CAST(d AS Date) = '2026-01-01' SETTINGS date_time_overflow_behavior='throw' +['2026-01-01'] +1 +valid: toDate(d) != '2026-01-01' SETTINGS date_time_overflow_behavior='throw' +['1970-01-02'] +1 +nullable: toDate(d) = '2026-01-01' +['2026-01-01','2205-06-08'] +2 +nullable: CAST(d AS Nullable(Date)) = toNullable(toDate('2026-01-01')) +['2026-01-01','2205-06-08'] +2 +nullable: CAST(d AS Nullable(Date)) != '2026-01-01' +['2026-01-02'] +2 +no_stats: toDate(d) = '2026-01-01' +['2026-01-01'] +2 +nullable: cast rejects NULL (engine predicate 0) +1 +nullable: cast rejects NULL (engine predicate 1) +1 +nullable: 1 +['2026-01-01','2026-01-02','2205-06-08','NULL'] +4 +mixed: toDate(d) = '1970-01-01' SETTINGS date_time_overflow_behavior='ignore' +['1970-01-01','2149-06-07'] +2 +mixed: toDate(d) != '1970-01-01' SETTINGS date_time_overflow_behavior='ignore' +['1969-12-31','2026-01-01','2026-01-02','2149-06-06'] +4 +mixed: toDate(d) = '2149-06-06' SETTINGS date_time_overflow_behavior='ignore' +['1969-12-31','2149-06-06'] +2 +mixed: toDate(d) != '2149-06-06' SETTINGS date_time_overflow_behavior='ignore' +['1970-01-01','2026-01-01','2026-01-02','2149-06-07'] +4 +mixed: toDate(d) = '1970-01-01' SETTINGS date_time_overflow_behavior='saturate' +['1969-12-31','1970-01-01'] +2 +mixed: toDate(d) != '1970-01-01' SETTINGS date_time_overflow_behavior='saturate' +['2026-01-01','2026-01-02','2149-06-06','2149-06-07'] +4 +mixed: toDate(d) = '2149-06-06' SETTINGS date_time_overflow_behavior='saturate' +['2149-06-06','2149-06-07'] +2 +mixed: toDate(d) != '2149-06-06' SETTINGS date_time_overflow_behavior='saturate' +['1969-12-31','1970-01-01','2026-01-01','2026-01-02'] +4 +spanning: toDate(d) = '1970-01-02' +['1970-01-02'] +1 +spanning: NOT (toDate(d) != '1970-01-02') +['1970-01-02'] +1 +no_min: toDate(d) = '2026-01-01' +['2026-01-01'] +2 +no_min: toDate(d) != '2026-01-01' +['2026-01-02'] +2 +no_max: toDate(d) = '2026-01-01' +['2026-01-01'] +2 +no_max: toDate(d) != '2026-01-01' +['2026-01-02'] +2 +no_min: toDate32(d) = '2026-01-02' +['2026-01-02'] +1 +no_max: toDate32(d) = '2026-01-01' +['2026-01-01'] +1 +lower: CAST(d AS Date) = '2026-01-01' (engine predicate 0, throw) +1 +lower: CAST(d AS Date) = '2026-01-01' (engine predicate 1, throw) +1 +lower: toDate(d) != '2026-01-01' (engine predicate 0, throw) +1 +lower: toDate(d) != '2026-01-01' (engine predicate 1, throw) +1 +lower: NOT (toDate(d) = '2026-01-01' AND d > toDate32('1900-01-01')) (engine predicate 0, throw) +1 +lower: NOT (toDate(d) = '2026-01-01' AND d > toDate32('1900-01-01')) (engine predicate 1, throw) +1 +upper: CAST(d AS Date) = '2026-01-01' (engine predicate 0, throw) +1 +upper: CAST(d AS Date) = '2026-01-01' (engine predicate 1, throw) +1 +upper: toDate(d) != '2026-01-01' (engine predicate 0, throw) +1 +upper: toDate(d) != '2026-01-01' (engine predicate 1, throw) +1 +upper: NOT (toDate(d) = '2026-01-01' AND d > toDate32('1900-01-01')) (engine predicate 0, throw) +1 +upper: NOT (toDate(d) = '2026-01-01' AND d > toDate32('1900-01-01')) (engine predicate 1, throw) +1 +t: toDate(d, 'UTC') = '2026-01-01' +['2026-01-01','2205-06-08'] +2 +t: toDate32(d, 'UTC') = '2026-01-01' +['2026-01-01'] +1 +t: toDate32(d, 'UTC') != '2026-01-01' +['2026-01-02','2205-06-08'] +2 +t: toDate32(d) < '2026-01-02' +['2026-01-01'] +1 +t: toDate32(d) >= '2026-01-02' +['2026-01-02','2205-06-08'] +2 +t: '2026-01-02' > toDate32(d) +['2026-01-01'] +1 +t: d::Date32 <= '2026-01-01' +['2026-01-01'] +1 +t: toDate(d) < '2026-01-02' +['2026-01-01','2205-06-08'] +2 +t: toDate(d) >= '2026-01-02' +['2026-01-02'] +2 +t: '2026-01-02' > toDate(d) +['2026-01-01','2205-06-08'] +2 +t: NOT (toDate(d) < '2026-01-02') +['2026-01-02'] +2 +t: NOT (toDate(d) > '2026-01-01') +['2026-01-01','2205-06-08'] +2 +aliases: toDate(d) <= '2026-01-01' +['1846-07-28','2384-11-12'] +2 +lower: toDate(d) < '1970-01-02' SETTINGS date_time_overflow_behavior='saturate' +['1969-12-31','1970-01-01'] +2 +upper: toDate(d) > '2149-06-05' SETTINGS date_time_overflow_behavior='saturate' +['2149-06-06','2149-06-07'] +2 +no_min: toDate32(d) > '2026-01-01' +['2026-01-02'] +1 +no_max: toDate32(d) < '2026-01-02' +['2026-01-01'] +1 +upper: toDate(d) < '2026-01-01' (engine predicate 0, throw) +1 +upper: toDate(d) < '2026-01-01' (engine predicate 1, throw) +1 +t: toDate32(d) = toDate('2026-01-01') +['2026-01-01'] +1 +t: toDate(d) = toDate32('2026-01-01') +['2026-01-01','2205-06-08'] +2 +t: toDate(d) != toDate32('2026-01-01') +['2026-01-02'] +2 +t: toDate32(d) < toDate('2026-01-02') +['2026-01-01'] +1 +t: toDate(d) = toDate32('2205-06-08') +[] +3 +t: toDate(d) != toDate32('2205-06-08') +['2026-01-01','2026-01-02','2205-06-08'] +3 +t: toDate(d) < toDate32('2205-06-08') +['2026-01-01','2026-01-02','2205-06-08'] +3 +overflow_rhs: toDate(d) = '1969-12-31' (engine predicate 0, throw) +1 +overflow_rhs: toDate(d) = '1969-12-31' (engine predicate 1, throw) +1 diff --git a/tests/queries/0_stateless/05153_delta_lake_cast_predicate_pruning.sh b/tests/queries/0_stateless/05153_delta_lake_cast_predicate_pruning.sh new file mode 100755 index 0000000000000..20df67b780905 --- /dev/null +++ b/tests/queries/0_stateless/05153_delta_lake_cast_predicate_pruning.sh @@ -0,0 +1,360 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: delta-kernel pulls in extra dependencies. +# Tag no-msan: delta-kernel-rs (Rust) is not built under MSan, so DeltaLakeLocal is absent. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +set -e + +ROOT="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_delta_cast" +trap 'rm -rf "${ROOT}" 2>/dev/null' EXIT +rm -rf "${ROOT}" + +# A clickhouse-local launch costs seconds on slow builds, and one launch per query used to blow +# past the 180s test limit. The test therefore records everything first - `make_table`, `check` +# and `expect_error` only declare work - and `run_all` at the bottom executes it in five batched +# launches: one for the data files, then one per engine-predicate mode for the checks and one +# per mode for the expected errors. Per-query numbers are recovered from consecutive reads of +# the cumulative counters in `system.events` and `system.errors`, which is exact because the +# statements of one launch run sequentially in a single process. + +TABLES=() +declare -A TABLE_TYPE TABLE_STATS TABLE_DATES +FIXTURE_QUERIES="" + +# make_table ... +# Each argument is one file: sorted, comma-separated ISO dates, or a single NULL. +# `no_min` / `no_max` omit only that bound; `none` emits only `numRecords`. +make_table() { + local table=$1 type=$2 stats_mode=$3 + shift 3 + mkdir -p "${ROOT}/${table}/_delta_log" + TABLES+=("${table}") + TABLE_TYPE[$table]="${type}" + TABLE_STATS[$table]="${stats_mode}" + TABLE_DATES[$table]="$*" + local date value + for date in "$@"; do + value="'${date//,/\',\'}'" + [[ "${date}" == NULL ]] && value=NULL + FIXTURE_QUERIES+=" + INSERT INTO FUNCTION file('${ROOT}/${table}/${date}.parquet', Parquet, 'd ${type}') + SELECT CAST(arrayJoin([${value}]), '${type}');" + done +} + +# The second phase of `make_table`: protocol, metaData, then one add action per file. The log is +# hand-written because ClickHouse's Delta writer emits no per-column stats, and it can only be +# written after the fixture batch ran - it embeds the data files' byte sizes. `schemaString` and +# `stats` are JSON documents embedded as JSON strings, hence the escaped quotes. +write_log() { + local table=$1 + local type="${TABLE_TYPE[$table]}" stats_mode="${TABLE_STATS[$table]}" + local nullable=false + [[ "${type}" == Nullable* ]] && nullable=true + local log="${ROOT}/${table}/_delta_log/00000000000000000000.json" + cat > "${log}" <> "${log}" < [settings]: rows with the engine predicate on must match the rows with +# it off, the off run must scan every file, and the on rows and scan count go to the reference. +check() { + ITEMS+=("check ${#CHECK_TABLES[@]}") + CHECK_TABLES+=("$1") + CHECK_WHERES+=("$2") + CHECK_SETTINGS+=("${3:-}") +} + +# expect_error
[settings]: the query must fail with +# the given error identity; scan counts are not pinned after cancellation. +expect_error() { + ITEMS+=("error ${#ERR_HEADERS[@]}") + ERR_HEADERS+=("$1") + ERR_TABLES+=("$2") + ERR_WHERES+=("$3") + ERR_ENABLED+=("$4") + ERR_CODES+=("$5") + ERR_SETTINGS+=("${6:-}") +} + +# The data query of one recorded case, with its per-case settings attached. +data_query() { + printf "SELECT arraySort(groupArray(ifNull(toString(d), 'NULL'))) FROM deltaLakeLocal('%s') WHERE %s%s;" \ + "${ROOT}/$1" "$2" "${3:+ SETTINGS $3}" +} + +# Disable every applicable Parquet pruning path, so only the engine predicate can skip files. +run_batch() { + local enabled=$1 + shift + ${CLICKHOUSE_LOCAL} --date_time_overflow_behavior=ignore --convert_query_to_cnf=0 \ + --short_circuit_function_evaluation=disable \ + --input_format_parquet_filter_push_down=0 --input_format_parquet_page_filter_push_down=0 \ + --input_format_parquet_bloom_filter_push_down=0 --input_format_parquet_dictionary_filter_push_down=0 \ + --delta_lake_enable_engine_predicate="${enabled}" --delta_lake_throw_on_engine_predicate_error=1 "$@" +} + +# Runs every recorded check in one launch. Each data query is followed by a read of the +# cumulative `DeltaLakeScannedFiles` counter; consecutive differences give per-query scan counts. +run_checks() { + local enabled=$1 + local -n rows_out=$2 deltas_out=$3 + local i batch="" out + for i in "${!CHECK_TABLES[@]}"; do + batch+="$(data_query "${CHECK_TABLES[$i]}" "${CHECK_WHERES[$i]}" "${CHECK_SETTINGS[$i]}") + SELECT sumIf(value, event = 'DeltaLakeScannedFiles') FROM system.events;" + done + out=$(run_batch "${enabled}" --query "${batch}") + local -a lines + mapfile -t lines <<< "${out}" + if [[ "${#lines[@]}" != "$(( 2 * ${#CHECK_TABLES[@]} ))" ]]; then + printf 'Expected %s lines from the check batch (engine predicate %s), got %s:\n%s\n' \ + "$(( 2 * ${#CHECK_TABLES[@]} ))" "${enabled}" "${#lines[@]}" "${out}" >&2 + return 1 + fi + local previous=0 cumulative + for i in "${!CHECK_TABLES[@]}"; do + rows_out+=("${lines[2 * i]}") + cumulative="${lines[2 * i + 1]}" + deltas_out+=("$(( cumulative - previous ))") + previous="${cumulative}" + done +} + +# Runs each error case with a `-- { serverError }` hint that consumes the expected exception (no stderr), while its following `system.errors` counter must grow to prove the query failed. +run_errors() { + local enabled=$1 + local i batch="" out + local -a case_ids=() + for i in "${!ERR_HEADERS[@]}"; do + [[ "${ERR_ENABLED[$i]}" == "${enabled}" ]] || continue + case_ids+=("$i") + batch+="$(data_query "${ERR_TABLES[$i]}" "${ERR_WHERES[$i]}" "${ERR_SETTINGS[$i]}") -- { serverError ${ERR_CODES[$i]} } + SELECT sumIf(value, name = '${ERR_CODES[$i]}') FROM system.errors;" + done + out=$(run_batch "${enabled}" --query "${batch}") + local -a lines + mapfile -t lines <<< "${out}" + if [[ "${#lines[@]}" != "${#case_ids[@]}" ]]; then + printf 'Expected %s failing queries (engine predicate %s), got output:\n%s\n' \ + "${#case_ids[@]}" "${enabled}" "${out}" >&2 + return 1 + fi + local j code + declare -A previous=() + for j in "${!case_ids[@]}"; do + i="${case_ids[$j]}" + code="${ERR_CODES[$i]}" + if [[ ! "${lines[$j]}" =~ ^[0-9]+$ ]] || [[ "${lines[$j]}" -le "${previous[$code]:-0}" ]]; then + printf 'Expected %s (engine predicate %s) from: %s\n' "${code}" "${enabled}" "${ERR_WHERES[$i]}" >&2 + return 1 + fi + previous[$code]="${lines[$j]}" + done +} + +# Prints the results in declaration order; the reference interleaves checks and expected errors. +emit_results() { + local item kind idx + for item in "${ITEMS[@]}"; do + read -r kind idx <<< "${item}" + if [[ "${kind}" == error ]]; then + printf '%s\n' "${ERR_HEADERS[$idx]}" + printf '1\n' + continue + fi + local table="${CHECK_TABLES[$idx]}" where="${CHECK_WHERES[$idx]}" settings="${CHECK_SETTINGS[$idx]}" + local files=("${ROOT}/${table}/"*.parquet) + printf '%s: %s%s\n' "${table}" "${where}" "${settings:+ SETTINGS ${settings}}" + # Compare rows, not scan counts. The reference independently checks rows and selected files. + diff -u <(printf '%s\n' "${OFF_ROWS[$idx]}") <(printf '%s\n' "${ON_ROWS[$idx]}") + if [[ "${OFF_DELTAS[$idx]}" != "${#files[@]}" ]]; then + printf 'Unexpected unpruned file count %s for: %s\n' "${OFF_DELTAS[$idx]}" "${where}" >&2 + return 1 + fi + printf '%s\n' "${ON_ROWS[$idx]}" + printf '%s\n' "${ON_DELTAS[$idx]}" + done +} + +run_all() { + ${CLICKHOUSE_LOCAL} --query "${FIXTURE_QUERIES}" + local table + for table in "${TABLES[@]}"; do + write_log "${table}" + done + run_checks 0 OFF_ROWS OFF_DELTAS + run_checks 1 ON_ROWS ON_DELTAS + run_errors 0 + run_errors 1 + emit_results +} + +# `Date` wraps modulo 65536 days: the alias matches equality but stays scanned even for `!=`. +make_table t Date32 full 2026-01-01 2026-01-02 2205-06-08 +check t "d = '2026-01-01'" +check t "CAST(d AS Date) = '2026-01-01'" +check t "d::Date = '2026-01-01'" +check t "toDate(d) = '2026-01-01'" +check t "d::Date32 = '2026-01-01'" +check t "toDate32(d) = toDate32('2026-01-01')" +check t "d::Date != '2026-01-01'" +check t "toDate('2026-01-01') = toDate(d)" +check t "toDate('2026-01-01') != toDate(d)" +check t "toDate(d) = '2026-01-03'" +check t "toDate32(d) = '2026-01-03'" + +make_table lower Date32 full 1969-12-31 1970-01-01 1970-01-02 +check lower "toDate(d) = '1970-01-01'" "date_time_overflow_behavior='saturate'" +check lower "toDate(d) != '1970-01-01'" "date_time_overflow_behavior='saturate'" +make_table upper Date32 full 2149-06-05 2149-06-06 2149-06-07 +check upper "CAST(d AS Date) = '2149-06-06'" "date_time_overflow_behavior='saturate'" +check upper "CAST(d AS Date) != '2149-06-06'" "date_time_overflow_behavior='saturate'" + +make_table negation Date32 full 2026-01-01 2026-01-02 2205-06-09 +check negation "NOT (toDate(d) = '2026-01-01' AND d > toDate32('1970-01-01'))" + +# Aliases at -65536 and +131072 days also lie outside the DateLUT's 1900..2299 fast range. +make_table aliases Date32 full 1846-07-28 2026-01-02 2384-11-12 +check aliases "toDate(d) = '2026-01-01'" +check aliases "toDate(d) != '2026-01-01'" + +make_table valid Date32 full 1970-01-02 2026-01-01 +check valid "toDate(d) != toDateTime('2026-01-01 00:00:01', 'UTC')" +check valid "CAST(d AS Date) = '2026-01-01'" "date_time_overflow_behavior='throw'" +check valid "toDate(d) != '2026-01-01'" "date_time_overflow_behavior='throw'" + +make_table nullable 'Nullable(Date32)' full NULL 2026-01-01 2026-01-02 2205-06-08 +check nullable "toDate(d) = '2026-01-01'" +check nullable "CAST(d AS Nullable(Date)) = toNullable(toDate('2026-01-01'))" +check nullable "CAST(d AS Nullable(Date)) != '2026-01-01'" + +make_table no_stats Date32 none 2026-01-01 2026-01-02 +check no_stats "toDate(d) = '2026-01-01'" + +# A non-nullable cast must still reject the stats-bearing NULL file. +for enabled in 0 1; do + expect_error "nullable: cast rejects NULL (engine predicate ${enabled})" \ + nullable "CAST(d AS Date) = '2026-01-01'" "${enabled}" \ + CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN "cast_keep_nullable=0" +done + +# `groupArray` must not hide the all-null file in the unfiltered fixture check. +check nullable "1" + +# Two files cross the Date boundaries; two safe singletons must still be prunable. +make_table mixed Date32 full 1969-12-31,1970-01-01 2149-06-06,2149-06-07 2026-01-01 2026-01-02 +for mode in ignore saturate; do + for boundary in 1970-01-01 2149-06-06; do + check mixed "toDate(d) = '${boundary}'" "date_time_overflow_behavior='${mode}'" + check mixed "toDate(d) != '${boundary}'" "date_time_overflow_behavior='${mode}'" + done +done + +# Both endpoints wrap to day zero, but the file also contains a matching interior value. +make_table spanning Date32 full 1970-01-01,1970-01-02,2149-06-07 2026-01-01 +check spanning "toDate(d) = '1970-01-02'" +check spanning "NOT (toDate(d) != '1970-01-02')" + +# One missing bound cannot prove that a Date cast is safe. +for mode in no_min no_max; do + make_table "$mode" Date32 "$mode" 2026-01-01 2026-01-02 + check "$mode" "toDate(d) = '2026-01-01'" + check "$mode" "toDate(d) != '2026-01-01'" +done +# Identity conversions can still prune from the remaining bound. +check no_min "toDate32(d) = '2026-01-02'" +check no_max "toDate32(d) = '2026-01-01'" + +# Actual overflows at both bounds must survive equality, inequality, and nested negation. +for table in lower upper; do + for where in "CAST(d AS Date) = '2026-01-01'" "toDate(d) != '2026-01-01'" \ + "NOT (toDate(d) = '2026-01-01' AND d > toDate32('1900-01-01'))"; do + for enabled in 0 1; do + expect_error "${table}: ${where} (engine predicate ${enabled}, throw)" \ + "${table}" "${where}" "${enabled}" VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE \ + "date_time_overflow_behavior='throw'" + done + done +done + +# The optional timezone argument of `toDate` / `toDate32` is inert for a `Date32` source. +check t "toDate(d, 'UTC') = '2026-01-01'" +check t "toDate32(d, 'UTC') = '2026-01-01'" +check t "toDate32(d, 'UTC') != '2026-01-01'" + +# Ordering: identity conversions map exactly; `Date`-narrowing ones are fenced so out-of-domain files stay scanned in either polarity. +check t "toDate32(d) < '2026-01-02'" +check t "toDate32(d) >= '2026-01-02'" +check t "'2026-01-02' > toDate32(d)" +check t "d::Date32 <= '2026-01-01'" +check t "toDate(d) < '2026-01-02'" +check t "toDate(d) >= '2026-01-02'" +check t "'2026-01-02' > toDate(d)" +check t "NOT (toDate(d) < '2026-01-02')" +check t "NOT (toDate(d) > '2026-01-01')" +check aliases "toDate(d) <= '2026-01-01'" +check lower "toDate(d) < '1970-01-02'" "date_time_overflow_behavior='saturate'" +check upper "toDate(d) > '2149-06-05'" "date_time_overflow_behavior='saturate'" +# Identity ordering can still prune from a single remaining bound. +check no_min "toDate32(d) > '2026-01-01'" +check no_max "toDate32(d) < '2026-01-02'" +# An actual overflow must survive an ordering comparison whose in-domain answer is false. +for enabled in 0 1; do + expect_error "upper: toDate(d) < '2026-01-01' (engine predicate ${enabled}, throw)" \ + upper "toDate(d) < '2026-01-01'" "${enabled}" VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE \ + "date_time_overflow_behavior='throw'" +done + +# Mixed `Date` / `Date32` literals convert exactly; one beyond the `Date` domain stays untranslated and scans everything. +check t "toDate32(d) = toDate('2026-01-01')" +check t "toDate(d) = toDate32('2026-01-01')" +check t "toDate(d) != toDate32('2026-01-01')" +check t "toDate32(d) < toDate('2026-01-02')" +check t "toDate(d) = toDate32('2205-06-08')" +check t "toDate(d) != toDate32('2205-06-08')" +check t "toDate(d) < toDate32('2205-06-08')" + +# The snapshot scan must build the engine predicate with the query's overflow settings. A nullable +# source hides the out-of-range RHS parse from zero-row header analysis, so if pruning coerced the +# literal with the default `ignore` it would clamp `'1969-12-31'` to a real day and drop both files, +# suppressing the exception. Both rows stay inside the `Date` domain, so the only overflow is the RHS. +make_table overflow_rhs 'Nullable(Date32)' full 1970-01-02 2026-01-01 +for enabled in 0 1; do + expect_error "overflow_rhs: toDate(d) = '1969-12-31' (engine predicate ${enabled}, throw)" \ + overflow_rhs "toDate(d) = '1969-12-31'" "${enabled}" VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE \ + "date_time_overflow_behavior='throw'" +done + +run_all From 2ccbaa65ef918598af714a4f8d50e618e16dcbe7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Oct 2026 07:32:34 +0000 Subject: [PATCH 237/296] Backport #119054 to 26.8: Fix unbound read in Azure::readBigAt --- .../IO/ReadBufferFromAzureBlobStorage.cpp | 26 +++--- src/Disks/IO/ReadBufferFromAzureBlobStorage.h | 7 +- src/Disks/tests/gtest_azure_read_buffer.cpp | 80 ++++++++++++++++++- 3 files changed, 99 insertions(+), 14 deletions(-) diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index e8bd82ffe9621..5724a16deef8b 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -273,9 +273,6 @@ void ReadBufferFromAzureBlobStorage::initialize(size_t attempt) Azure::Core::Context azure_context = Azure::Core::Context().WithValue(PocoAzureHTTPClient::getSDKContextKeyForBufferRetry(), attempt); - if (!blob_client) - blob_client = std::make_unique(blob_container_client->GetBlobClient(path)); - size_t sleep_time_with_backoff_milliseconds = 100; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromAzureInitMicroseconds); @@ -290,7 +287,7 @@ void ReadBufferFromAzureBlobStorage::initialize(size_t attempt) if (blob_container_client->IsClientForDisk()) ProfileEvents::increment(ProfileEvents::DiskAzureGetObject); - auto download_response = blob_client->Download(download_options, azure_context); + auto download_response = getBlobClient().Download(download_options, azure_context); checkReturnedGeneration(download_response.Value.Details); setMetadataFromResponse(download_response.Value.Details, download_response.Value.BlobSize); @@ -427,11 +424,8 @@ void ReadBufferFromAzureBlobStorage::rethrowIfGenerationChanged(const Azure::Cor std::optional ReadBufferFromAzureBlobStorage::tryGetFileSize() { - if (!blob_client) - blob_client = std::make_unique(blob_container_client->GetBlobClient(path)); - if (!file_size) - file_size = blob_client->GetProperties().Value.BlobSize; + file_size = getBlobClient().GetProperties().Value.BlobSize; return file_size; } @@ -447,6 +441,15 @@ std::optional ReadBufferFromAzureBlobStorage::getRemoteFileM .last_modification_time = static_cast(last_modification_time)}; } +const AzureBlobStorage::BlobClient & ReadBufferFromAzureBlobStorage::getBlobClient() const +{ + std::call_once(blob_client_created, [this] + { + blob_client = std::make_unique(blob_container_client->GetBlobClient(path)); + }); + return *blob_client; +} + size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t range_begin, const std::function & /*progress_callback*/) const { size_t initial_n = n; @@ -470,7 +473,7 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran setAccessConditions(download_options); Azure::Core::Context azure_context = Azure::Core::Context().WithValue(PocoAzureHTTPClient::getSDKContextKeyForBufferRetry(), size_t{0}); - auto download_response = blob_client->Download(download_options, azure_context); + auto download_response = getBlobClient().Download(download_options, azure_context); checkReturnedGeneration(download_response.Value.Details); if (blob_storage_log) @@ -486,7 +489,8 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran setMetadataFromResponse(download_response.Value.Details, download_response.Value.BlobSize); std::unique_ptr body_stream = std::move(download_response.Value.BodyStream); - bytes_copied = body_stream->ReadToCount(reinterpret_cast(to), body_stream->Length(), azure_context); + bytes_copied = body_stream->ReadToCount(reinterpret_cast(to), n, azure_context); + chassert(bytes_copied <= n); LOG_TEST(log, "AzureBlobStorage readBigAt read bytes {}", bytes_copied); @@ -550,7 +554,7 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran n -= bytes_copied; } - return initial_n; + return initial_n - n; } ObjectMetadata ReadBufferFromAzureBlobStorage::getObjectMetadataFromTheLastRequest() const diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.h b/src/Disks/IO/ReadBufferFromAzureBlobStorage.h index a7da6b4f5712a..0079579440c37 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "config.h" #if USE_AZURE_BLOB_STORAGE @@ -83,6 +84,9 @@ class ReadBufferFromAzureBlobStorage : public ReadBufferFromFileBase void initialize(size_t attempt); void setMetadataFromResponse(const Azure::Storage::Blobs::Models::DownloadBlobDetails & details, size_t blob_size) const; + /// Creates the client on first use. Thread-safe. + const AzureBlobStorage::BlobClient & getBlobClient() const; + /// The offset just past the last byte that the current download is allowed to deliver. /// `reported_length` is the length of the response body as reported by the remote endpoint, /// and is not trusted: it is bounded by `read_until_position_`, which is set locally. @@ -103,7 +107,8 @@ class ReadBufferFromAzureBlobStorage : public ReadBufferFromFileBase std::unique_ptr data_stream; ContainerClientPtr blob_container_client; - BlobClientPtr blob_client; + mutable BlobClientPtr blob_client; + mutable std::once_flag blob_client_created; const String path; size_t max_single_read_retries; diff --git a/src/Disks/tests/gtest_azure_read_buffer.cpp b/src/Disks/tests/gtest_azure_read_buffer.cpp index 141924dc2f025..0c562df51c732 100644 --- a/src/Disks/tests/gtest_azure_read_buffer.cpp +++ b/src/Disks/tests/gtest_azure_read_buffer.cpp @@ -108,6 +108,23 @@ class RangeResponseTransport : public Azure::Core::Http::HttpTransport size_t served_size; }; +std::unique_ptr makeBuffer(size_t claimed_size, size_t served_size) +{ + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = std::make_shared(claimed_size, served_size); + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + return std::make_unique( + container_client, + "blob", + DB::ReadSettings{}, + /* max_single_read_retries */ 1, + /* max_single_download_retries */ 1); +} + /// Reads a blob sequentially from an endpoint that answers every ranged request with `response_size` /// bytes counting up from zero, with the right bound set to `read_until_position` and a /// `buffer_size`-byte reading buffer. @@ -855,8 +872,7 @@ TEST(AzureReadBigAt, RejectsReplacedBlobThroughIfMatch) listed_etag, /* file_size */ 1000); - /// In this version `readBigAt` uses the client that `tryGetFileSize` creates, so it is called first, - /// as the callers of `readBigAt` do. The size is known, so it issues no request of its own. + /// The size is already known, so this does not issue a properties request. ASSERT_EQ(buffer.getFileSize(), static_cast(1000)); std::array out{}; @@ -866,4 +882,64 @@ TEST(AzureReadBigAt, RejectsReplacedBlobThroughIfMatch) ASSERT_EQ(transport->getDownloadCount(), static_cast(1)); } +TEST(AzureReadBigAt, DoesNotTrustResponseLength) +{ + constexpr size_t requested = 16; + + /// The response advertises and serves 32 bytes while only 16 are requested. + auto buffer = makeBuffer(/* claimed_size */ 32, /* served_size */ 32); + + /// The documented handshake: supportsReadAt is the required setup call before readBigAt. + ASSERT_TRUE(buffer->supportsReadAt()); + + struct Storage + { + std::array payload{}; + std::array canary{}; + } storage; + + std::fill(storage.canary.begin(), storage.canary.end(), '\xCD'); + + const size_t bytes_read = buffer->readBigAt(storage.payload.data(), requested, /* range_begin */ 0, {}); + + ASSERT_EQ(bytes_read, requested); + for (size_t i = 0; i < requested; ++i) + ASSERT_EQ(static_cast(storage.payload[i]), static_cast(i)); + for (char byte : storage.canary) + ASSERT_EQ(byte, '\xCD'); +} + +TEST(AzureReadBigAt, ReturnsAccumulatedCountOnTruncatedResponse) +{ + constexpr size_t requested = 16; + constexpr size_t served = 8; + + /// The response advertises `requested` bytes but its body ends after `served`. + auto buffer = makeBuffer(/* claimed_size */ requested, /* served_size */ served); + + ASSERT_TRUE(buffer->supportsReadAt()); + + struct Storage + { + std::array payload{}; + std::array canary{}; + } storage; + + std::fill(storage.payload.begin(), storage.payload.end(), '\xCD'); + std::fill(storage.canary.begin(), storage.canary.end(), '\xCD'); + + /// max_single_download_retries == 1, so the short read is not retried: readBigAt must return + /// the accumulated byte count, not the requested size. + const size_t bytes_read = buffer->readBigAt(storage.payload.data(), requested, /* range_begin */ 0, {}); + + ASSERT_EQ(bytes_read, served); + for (size_t i = 0; i < served; ++i) + ASSERT_EQ(static_cast(storage.payload[i]), static_cast(i)); + /// The unread tail must be left untouched instead of silently trusted as read. + for (size_t i = served; i < requested; ++i) + ASSERT_EQ(storage.payload[i], '\xCD'); + for (char byte : storage.canary) + ASSERT_EQ(byte, '\xCD'); +} + #endif From b125bbbc5abf07388c9c2c8b28de759b15459058 Mon Sep 17 00:00:00 2001 From: Vladimir Cherkasov Date: Fri, 9 Oct 2026 12:36:19 +0200 Subject: [PATCH 238/296] fix --- src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp | 9 +++++---- .../05243_enum_unknown_literal_key_condition.sql | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index 356091fd75fc3..19d5a3516a5b7 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -1120,11 +1120,12 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( return false; } - /// With validation off, `equals` against a name the enum lacks is constant false for every row, - /// so the conversion below must not throw; `KeyCondition` folds the same way. - if (function_name == "equals" && !validate_enum_literals_in_operators && isUnknownEnumElement(*actual_type, value_field)) + /// With validation off, `equals`/`notEquals` against a name the enum lacks fold to a constant + /// for every row, so the conversion below must not throw; `KeyCondition` folds the same way. + if ((function_name == "equals" || function_name == "notEquals") + && !validate_enum_literals_in_operators && isUnknownEnumElement(*actual_type, value_field)) { - out.function = RPNElement::ALWAYS_FALSE; + out.function = function_name == "equals" ? RPNElement::ALWAYS_FALSE : RPNElement::ALWAYS_TRUE; return true; } diff --git a/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.sql b/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.sql index c0ef817ac1d7a..4fb0ded6408c8 100644 --- a/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.sql +++ b/tests/queries/0_stateless/05243_enum_unknown_literal_key_condition.sql @@ -1,3 +1,7 @@ +-- This branch predates #117803: the query condition cache key carries no `validate_enum_literals_in_operators` +-- salt, so a cached "no rows match" entry for `e = 'c'` would hide the error expected with validation on. +SET use_query_condition_cache = 0; + DROP TABLE IF EXISTS t_enum_key; DROP TABLE IF EXISTS t_enum_nokey; From cfae1ebf0bde5d4528cdcf08ccefc099cbb7d87c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Oct 2026 12:12:29 +0000 Subject: [PATCH 239/296] Backport #123898 to 26.8: Fix stale bytes in JSON shared data after a failed insert of a path value --- src/Formats/JSONExtractTree.cpp | 7 +++++-- ...son_shared_data_restore_after_rejected_insert.reference | 2 ++ ...5258_json_shared_data_restore_after_rejected_insert.sql | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05258_json_shared_data_restore_after_rejected_insert.reference create mode 100644 tests/queries/0_stateless/05258_json_shared_data_restore_after_rejected_insert.sql diff --git a/src/Formats/JSONExtractTree.cpp b/src/Formats/JSONExtractTree.cpp index c00dac73ec2b0..bc8dde8e72454 100644 --- a/src/Formats/JSONExtractTree.cpp +++ b/src/Formats/JSONExtractTree.cpp @@ -2006,13 +2006,16 @@ class ObjectJSONNode : public JSONExtractTreeNode { /// Serialize value directly into shared data chars. WriteBufferFromVector value_buf(shared_data_values_chars, AppendModeTag(), format_settings.json_max_string_column_growth_step); - if (!insertIntoSharedData(value_buf, value, insert_settings, format_settings, error, tmp_dynamic_column)) + bool inserted = insertIntoSharedData(value_buf, value, insert_settings, format_settings, error, tmp_dynamic_column); + /// Finalize the buffer before `restoreColumnObject`: finalizing resizes the chars to the + /// buffer position, so doing it after the restore would bring back the popped values. + value_buf.finalize(); + if (!inserted) { error += fmt::format(" (while reading path {})", path); SerializationObject::restoreColumnObject(column_object, prev_size); return false; } - value_buf.finalize(); shared_data_values_offsets.push_back(shared_data_values_chars.size()); shared_data_paths->insertData(path.data(), path.size()); } diff --git a/tests/queries/0_stateless/05258_json_shared_data_restore_after_rejected_insert.reference b/tests/queries/0_stateless/05258_json_shared_data_restore_after_rejected_insert.reference new file mode 100644 index 0000000000000..96b90800d8c29 --- /dev/null +++ b/tests/queries/0_stateless/05258_json_shared_data_restore_after_rejected_insert.reference @@ -0,0 +1,2 @@ +[\'{}\'] +[\'{"a":1,"b":0,"c":[1,1]}\'] diff --git a/tests/queries/0_stateless/05258_json_shared_data_restore_after_rejected_insert.sql b/tests/queries/0_stateless/05258_json_shared_data_restore_after_rejected_insert.sql new file mode 100644 index 0000000000000..787c337a99ed4 --- /dev/null +++ b/tests/queries/0_stateless/05258_json_shared_data_restore_after_rejected_insert.sql @@ -0,0 +1,4 @@ +-- A failed attempt to insert a nested JSON object must not leave stale bytes in its shared data. +-- The cast to `Dynamic(max_types=0)` serializes the nested object in binary form. +SELECT toString(CAST(p.entries, 'Dynamic(max_types=0)')) +FROM format(JSONAsObject, 'p JSON(max_dynamic_paths=4)', '{"entries":[{}]}\n{"entries":[{"a":1,"b":0,"c":[true,1]}]}'); From fbf597e13546baf58b0101c6b3e1b0cf3d104377 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Oct 2026 15:10:21 +0000 Subject: [PATCH 240/296] Update autogenerated version to 26.8.22.13 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 457d01977a754..51af7ba58e03f 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54534) +SET(VERSION_REVISION 54535) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 22) -SET(VERSION_GITHASH e9b8ef33ed7e4d28376d0c75ea01ae7ccc8859eb) -SET(VERSION_DESCRIBE v26.8.22.1-lts) -SET(VERSION_STRING 26.8.22.1) +SET(VERSION_PATCH 23) +SET(VERSION_GITHASH cc72af4d4e916a278d8d750b2b0cab13aaac53a3) +SET(VERSION_DESCRIBE v26.8.23.1-lts) +SET(VERSION_STRING 26.8.23.1) # end of autochange From aebf795cb5916cc3eb902f10cad573ff39942dff Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Oct 2026 16:12:25 +0000 Subject: [PATCH 241/296] Backport #124441 to 26.8: Use `castStorage` for the `Join` row policy check in the legacy analyzer --- src/Interpreters/JoinedTables.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index f3d671cf38c15..a533a3d908a79 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -349,6 +349,20 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se StoragePtr storage = DatabaseCatalog::instance().tryGetTable(joined_table_id, context); /// A special storage replaces the right-side plan, and with it the `FilterStep` carrying the + /// table's row policy, so such a table has to be joined as an ordinary stream. A `Join` table + /// is a prebuilt hash table read as is, so it cannot be filtered at all. + if (storage) + { + auto row_policy_filter = context->getRowPolicyFilter( + joined_table_id.getDatabaseName(), joined_table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + { + if (castStorage(storage, DeferredTable::Load)) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot join table {} with the Join engine because a row policy is applied on it", joined_table_id.getNameForLogs()); + storage = nullptr; + } + } /// table's row policy, so such a table has to be joined as an ordinary stream. auto joined_table_row_policy = context->getRowPolicyFilter( joined_table_id.getDatabaseName(), joined_table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); From 4b81114f91bf3893ca6d86b1074af3dc4f5c25e1 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 9 Oct 2026 16:28:30 +0000 Subject: [PATCH 242/296] Fix the conflict resolution of the #124441 backport in `JoinedTables` The resolved cherry-pick kept both sides of the conflict: the `Join` table check from https://github.com/ClickHouse/ClickHouse/pull/124441 and the trimmed guard backported earlier from https://github.com/ClickHouse/ClickHouse/pull/121398, with the old comment split between them. Keep the check only and declare `ErrorCodes::ACCESS_DENIED`, which the file never had. Co-Authored-By: Claude Fable 5.1 --- src/Interpreters/JoinedTables.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index a533a3d908a79..fda8e3e8f45e5 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -43,6 +43,7 @@ namespace Setting namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int ALIAS_REQUIRED; extern const int AMBIGUOUS_COLUMN_NAME; extern const int LOGICAL_ERROR; @@ -363,11 +364,6 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se storage = nullptr; } } - /// table's row policy, so such a table has to be joined as an ordinary stream. - auto joined_table_row_policy = context->getRowPolicyFilter( - joined_table_id.getDatabaseName(), joined_table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); - if (joined_table_row_policy && !joined_table_row_policy->isAlwaysTrue()) - storage = nullptr; if (storage) { From b1ca43c9fa9171899dbe1cf700cd79181e76a479 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 9 Oct 2026 16:28:31 +0000 Subject: [PATCH 243/296] Backport the rest of #121398: `Join` tables under a row policy The earlier backport of https://github.com/ClickHouse/ClickHouse/pull/121398 kept only the legacy analyzer guard. Add the remaining parts: `joinGet` and `joinGetOrNull` fail with `ACCESS_DENIED` while a row policy applies to the `Join` table, the analyzer does the same in `buildQueryPlanForJoinNode` instead of joining the table as a stream, the documentation says so, and test 05234 covers it. Test 05233, which runs the legacy analyzer here, now expects `ACCESS_DENIED` for the `Join` table as well. The statement documentation does not live in C++ on this branch, so the row policy page is edited directly. Co-Authored-By: Claude Fable 5.1 --- .../statements/create/row-policy.mdx | 4 +++ src/Functions/FunctionJoinGet.cpp | 12 ++++++- src/Planner/PlannerJoinTree.cpp | 20 +++++++++--- src/Storages/StorageJoin.cpp | 2 ++ .../05233_direct_join_row_policy.reference | 4 --- .../05233_direct_join_row_policy.sql | 2 +- .../05234_row_policy_storage_join.reference | 12 +++++++ .../05234_row_policy_storage_join.sql | 31 +++++++++++++++++++ 8 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/05234_row_policy_storage_join.reference create mode 100644 tests/queries/0_stateless/05234_row_policy_storage_join.sql diff --git a/docs/reference/statements/create/row-policy.mdx b/docs/reference/statements/create/row-policy.mdx index e036318fe9f50..47fcdef01d405 100644 --- a/docs/reference/statements/create/row-policy.mdx +++ b/docs/reference/statements/create/row-policy.mdx @@ -160,6 +160,10 @@ CREATE ROW POLICY filter ON mydb.local_table USING a < 1000 TO john; This works while the query is shipped as text, which is the default. With [`serialize_query_plan = 1`](/reference/settings/session-settings/serialize#serialize_query_plan) the initiator ships an already-built read plan instead, and a remote server executing such a plan does not apply its own row policies, so a read of a `Distributed` table over `local_table` returns unfiltered rows. Keep `serialize_query_plan = 0` for users whose row policies must be enforced. See [issue #112891](https://github.com/ClickHouse/ClickHouse/issues/112891). +## Join tables {#join-tables} + +A [Join](/reference/engines/table-engines/special/join) table is a prepared hash table that a `JOIN` or `joinGet` reads as is, so its rows cannot be filtered there. A policy on such a table, including a database-wide `ON db.*` policy, filters a plain `SELECT` from the table, but while it applies, `JOIN` and `joinGet` queries against the table fail with `ACCESS_DENIED`. + ## ON CLUSTER Clause {#on-cluster-clause} Allows creating row policies on a cluster, see [Distributed DDL](/reference/statements/distributed-ddl). This is also the convenient way to create the policy on the local tables of every server of the cluster. diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 613b1b1aaf6be..b32fbbb085e7d 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include namespace DB { @@ -22,6 +24,7 @@ namespace Setting namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -147,7 +150,14 @@ ExecutableFunctionPtr FunctionJoinGet::prepare(const ColumnsWithTypeAndName &) c Names column_names = storage_join->getKeyNames(); column_names.push_back(attr_name); - context->checkAccess(AccessType::SELECT, storage_join->getStorageID(), column_names); + const auto storage_id = storage_join->getStorageID(); + context->checkAccess(AccessType::SELECT, storage_id, column_names); + + /// The hash table is read as is, so a row policy on the table cannot be applied here any more than in a JOIN. + auto row_policy_filter = context->getRowPolicyFilter(storage_id.getDatabaseName(), storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot use {} because a row policy is applied on table {} with the Join engine", function_name, storage_id.getNameForLogs()); return std::make_unique(function_name, context, table_lock, storage_join, result_columns); } diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 52e09cafb0b39..ab57c16ce1038 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -184,6 +185,7 @@ namespace ErrorCodes extern const int PARAMETER_OUT_OF_BOUND; extern const int TOO_MANY_COLUMNS; extern const int UNSUPPORTED_METHOD; + extern const int ACCESS_DENIED; } namespace @@ -3138,12 +3140,22 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode( join_node, planner_context); - PreparedJoinStorage prepared_join; - bool allow_storage_join = right_join_tree_query_plan.used_row_policies.empty() + /// A prepared storage replaces the right-side plan, and with it the `FilterStep` applying the + /// table's row policy, so such a table is joined as a stream. A `Join` table is a prebuilt + /// hash table read as is, so it cannot be filtered at all. + bool right_table_has_row_policy = !right_join_tree_query_plan.used_row_policies.empty(); + + PreparedJoinStorage prepared_join = tryGetStorageInTableJoin(join_node.getRightTableExpressionNode(), planner_context); + if (prepared_join.storage_join && right_table_has_row_policy) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot join table {} with the Join engine because a row policy is applied on it", + prepared_join.storage_join->getStorageID().getNameForLogs()); + + bool allow_storage_join = !right_table_has_row_policy && right_join_tree_query_plan.stage == QueryProcessingStage::FetchColumns && right_join_tree_query_plan.useful_sets.empty(); - if (allow_storage_join) - prepared_join = tryGetStorageInTableJoin(join_node.getRightTableExpressionNode(), planner_context); + if (!allow_storage_join) + prepared_join = {}; if (prepared_join) { bool use_nulls = settings[Setting::join_use_nulls] && isLeftOrFull(join_node.getKind()); diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index cbbb62afde5a5..cdc7a5c14c2f6 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -641,6 +641,8 @@ Default value: `1`. The `Join`-engine tables can't be used in `GLOBAL JOIN` operations. +A [row policy](/reference/statements/create/row-policy) on a `Join`-engine table filters a plain `SELECT` from it, but a `JOIN` or `joinGet` reads the prepared hash table as is and cannot filter its rows, so while a policy applies to the table such queries fail with `ACCESS_DENIED`. + The `Join`-engine allows to specify [join_use_nulls](/reference/settings/session-settings/join#join_use_nulls) setting in the `CREATE TABLE` statement. [SELECT](/reference/statements/select/index) query should have the same `join_use_nulls` value. ## Usage examples {#example} diff --git a/tests/queries/0_stateless/05233_direct_join_row_policy.reference b/tests/queries/0_stateless/05233_direct_join_row_policy.reference index f196664e9f2c7..eb1cc8f5e9e0f 100644 --- a/tests/queries/0_stateless/05233_direct_join_row_policy.reference +++ b/tests/queries/0_stateless/05233_direct_join_row_policy.reference @@ -21,10 +21,6 @@ 2 5 -- join table -1 public-1 -2 -3 public-3 -5 -- without a policy 1 public public-1 2 hidden hidden-2 diff --git a/tests/queries/0_stateless/05233_direct_join_row_policy.sql b/tests/queries/0_stateless/05233_direct_join_row_policy.sql index 38db3f227d2d7..993f530a6e2bf 100644 --- a/tests/queries/0_stateless/05233_direct_join_row_policy.sql +++ b/tests/queries/0_stateless/05233_direct_join_row_policy.sql @@ -39,7 +39,7 @@ SELECT '-- left anti'; SELECT p.key FROM probe_rls AS p LEFT ANTI JOIN kv_rls AS kv ON kv.key = p.key ORDER BY p.key; SELECT '-- join table'; -SELECT p.key, j.secret FROM probe_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; +SELECT p.key, j.secret FROM probe_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; -- { serverError ACCESS_DENIED } DROP ROW POLICY kv_rls_public ON kv_rls; DROP ROW POLICY join_rls_public ON join_rls; diff --git a/tests/queries/0_stateless/05234_row_policy_storage_join.reference b/tests/queries/0_stateless/05234_row_policy_storage_join.reference new file mode 100644 index 0000000000000..2ac892e8f5052 --- /dev/null +++ b/tests/queries/0_stateless/05234_row_policy_storage_join.reference @@ -0,0 +1,12 @@ +-- without a policy +1 a +2 b +3 +b \N +-- with a policy +1 a +-- after dropping the policy +1 a +2 b +3 +b \N diff --git a/tests/queries/0_stateless/05234_row_policy_storage_join.sql b/tests/queries/0_stateless/05234_row_policy_storage_join.sql new file mode 100644 index 0000000000000..91d6f948686d2 --- /dev/null +++ b/tests/queries/0_stateless/05234_row_policy_storage_join.sql @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS join_rls; +DROP TABLE IF EXISTS probe_join_rls; +DROP ROW POLICY IF EXISTS join_rls_policy ON join_rls; + +CREATE TABLE join_rls (key UInt64, value String) ENGINE = Join(ANY, LEFT, key); +INSERT INTO join_rls VALUES (1, 'a'), (2, 'b'); + +CREATE TABLE probe_join_rls (key UInt64) ENGINE = TinyLog; +INSERT INTO probe_join_rls VALUES (1), (2), (3); + +SELECT '-- without a policy'; +SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; +SELECT joinGet(join_rls, 'value', toUInt64(2)), joinGetOrNull(join_rls, 'value', toUInt64(3)); + +CREATE ROW POLICY join_rls_policy ON join_rls FOR SELECT USING value = 'a' TO CURRENT_USER; + +SELECT '-- with a policy'; +SELECT key, value FROM join_rls ORDER BY key; +SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; -- { serverError ACCESS_DENIED } +SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key SETTINGS join_algorithm = 'hash'; -- { serverError ACCESS_DENIED } +SELECT joinGet(join_rls, 'value', toUInt64(2)); -- { serverError ACCESS_DENIED } +SELECT joinGetOrNull(join_rls, 'value', toUInt64(2)); -- { serverError ACCESS_DENIED } + +DROP ROW POLICY join_rls_policy ON join_rls; + +SELECT '-- after dropping the policy'; +SELECT p.key, j.value FROM probe_join_rls AS p LEFT ANY JOIN join_rls AS j ON j.key = p.key ORDER BY p.key; +SELECT joinGet(join_rls, 'value', toUInt64(2)), joinGetOrNull(join_rls, 'value', toUInt64(3)); + +DROP TABLE join_rls; +DROP TABLE probe_join_rls; From 84c6cc3075d7665ab9e02df3f00de229d8ead256 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Oct 2026 21:40:35 +0000 Subject: [PATCH 244/296] Backport #123808 to 26.8: Speed up in-order LIMIT BY, DISTINCT, window functions, merge joins and FINAL on multi-column keys with long leading-key runs --- src/Columns/findEqualRangeEndAssumeSorted.h | 68 +++ ...test_get_equal_range_end_assume_sorted.cpp | 492 ++++++++++++++++++ src/Core/SortCursor.h | 50 +- src/Interpreters/MergeJoin.cpp | 5 +- .../DistinctSortedStreamTransform.cpp | 5 +- .../DistinctSortedStreamTransform.h | 2 + .../Transforms/LimitByTransform.cpp | 3 +- .../Transforms/MergeJoinTransform.cpp | 35 +- .../Transforms/MergeJoinTransform.h | 2 + .../Transforms/NegativeLimitByTransform.cpp | 3 +- tests/performance/full_sorting_merge_join.xml | 7 + tests/performance/limit_by_in_pk_order.xml | 23 + .../negative_limit_by_in_order.xml | 7 + tests/performance/sorted_key_prefix_runs.xml | 11 + 14 files changed, 685 insertions(+), 28 deletions(-) create mode 100644 tests/performance/sorted_key_prefix_runs.xml diff --git a/src/Columns/findEqualRangeEndAssumeSorted.h b/src/Columns/findEqualRangeEndAssumeSorted.h index 1ffe807d1031b..81f30be352e5e 100644 --- a/src/Columns/findEqualRangeEndAssumeSorted.h +++ b/src/Columns/findEqualRangeEndAssumeSorted.h @@ -5,6 +5,7 @@ #include #include +#include namespace DB { @@ -147,4 +148,71 @@ ALWAYS_INLINE Int64 compareTrackAtImpl( return 0; } +/** Returns the end of the run of rows whose multi-column key equals the key at `begin`, within `[begin, end)` sorted by + * that key. `search(i, from, bound)` must return the end of the run of column `i` at `from` within `[from, bound)`; + * column `i` is sorted there because `bound` is the end of the run of columns `0..i-1`. + */ +template +ALWAYS_INLINE size_t findKeyRangeEndAssumeSorted(size_t key_size, size_t begin, size_t end, Search && search) +{ + size_t run_end = end; + for (size_t i = 0; i < key_size; ++i) + { + run_end = search(i, begin, run_end); + if (run_end <= begin + 1) + break; + } + return run_end; +} + +/** Same result as `findKeyRangeEndAssumeSorted`, for callers that search the runs of one range one after another. + * Remembers the last run found for each key prefix: a search from a row inside it ends at the same row, so a column + * is searched again only when the search leaves its prefix's run. Call `reset` when the columns change, or when `end` + * changes and the next search may start before the previous `end`. + */ +class SortedKeyRuns +{ +public: + SortedKeyRuns() = default; + explicit SortedKeyRuns(size_t key_size) : runs(key_size) {} + + void reset(size_t key_size) { runs.assign(key_size, Run{}); } + size_t keySize() const { return runs.size(); } + + template + ALWAYS_INLINE size_t findRunEnd(size_t begin, size_t end, Search && search) + { + const size_t key_size = runs.size(); + if (key_size < 2) + return findKeyRangeEndAssumeSorted(key_size, begin, end, search); + + size_t level = 0; + while (level < key_size && runs[level].begin <= begin && begin < runs[level].end) + ++level; + + size_t run_end = level == 0 ? end : runs[level - 1].end; + for (; level < key_size; ++level) + { + run_end = search(level, begin, run_end); + if (run_end <= begin + 1) + break; + runs[level] = {begin, run_end}; + } + + chassert( + run_end == findKeyRangeEndAssumeSorted(key_size, begin, end, search), + "Equal values are not contiguous within the range assumed to be sorted"); + return run_end; + } + +private: + struct Run + { + size_t begin = 0; + size_t end = 0; + }; + + VectorWithMemoryTracking runs; +}; + } diff --git a/src/Columns/tests/gtest_get_equal_range_end_assume_sorted.cpp b/src/Columns/tests/gtest_get_equal_range_end_assume_sorted.cpp index 65995debe2039..4604ba90e8734 100644 --- a/src/Columns/tests/gtest_get_equal_range_end_assume_sorted.cpp +++ b/src/Columns/tests/gtest_get_equal_range_end_assume_sorted.cpp @@ -1,6 +1,9 @@ +#include #include #include #include +#include +#include #include #include @@ -14,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -263,6 +267,222 @@ void runVectorTests(const char * name) } } +/// Values of a sorted multi-column key: column 0 has runs of `leading_run` rows, and column k + 1 restarts at 0 +/// in every run of column k and has runs of `sub_runs[k]` rows there (0 means unique values). +std::vector> makeSortedKeyValues(size_t rows, size_t leading_run, const std::vector & sub_runs) +{ + std::vector> values(sub_runs.size() + 1, std::vector(rows)); + for (size_t row = 0; row < rows; ++row) + { + values[0][row] = row / leading_run; + size_t offset = row % leading_run; + for (size_t k = 0; k < sub_runs.size(); ++k) + { + if (sub_runs[k] == 0) + { + values[k + 1][row] = row; + offset = 0; + } + else + { + values[k + 1][row] = offset / sub_runs[k]; + offset %= sub_runs[k]; + } + } + } + return values; +} + +enum class KeyType : uint8_t +{ + UInt32, + UInt64, + String, + /// 0 is NULL (with arbitrary nested values and non-zero null map bytes), so it sorts first for hint -1. + NullableString, + /// 0 is NaN and 1 is -0.0 or +0.0 by row parity, so it sorts as the values for hint -1. + Float64WithNaNAndZeros, + /// Dictionary indices are in the reverse order of the values. + LowCardinalityString, +}; + +String sortableString(UInt64 value) +{ + char buf[32]; + (void)snprintf(buf, sizeof(buf), "%08llu", static_cast(value)); + return buf; +} + +ColumnPtr makeKeyColumn(KeyType type, const std::vector & values) +{ + switch (type) + { + case KeyType::UInt32: + { + auto col = ColumnUInt32::create(); + for (UInt64 v : values) + col->getData().push_back(static_cast(v)); + return col; + } + case KeyType::UInt64: + { + auto col = ColumnUInt64::create(); + for (UInt64 v : values) + col->getData().push_back(v); + return col; + } + case KeyType::String: + { + auto col = ColumnString::create(); + for (UInt64 v : values) + col->insert(Field(sortableString(v))); + return col; + } + case KeyType::NullableString: + { + auto nested = ColumnString::create(); + auto null_map = ColumnUInt8::create(); + for (size_t row = 0; row < values.size(); ++row) + { + const bool is_null = values[row] == 0; + nested->insert(Field(sortableString(is_null ? (row * 7919) % 1000 : values[row]))); + null_map->getData().push_back(is_null ? static_cast(1 + row % 255) : static_cast(0)); + } + return ColumnNullable::create(std::move(nested), std::move(null_map)); + } + case KeyType::Float64WithNaNAndZeros: + { + auto col = ColumnFloat64::create(); + for (size_t row = 0; row < values.size(); ++row) + { + Float64 x = static_cast(values[row]); + if (values[row] == 0) + x = std::numeric_limits::quiet_NaN(); + else if (values[row] == 1) + x = row % 2 ? -0.0 : 0.0; + col->getData().push_back(x); + } + return col; + } + case KeyType::LowCardinalityString: + { + auto type_lc = std::make_shared(std::make_shared()); + auto builder = type_lc->createColumn(); + const UInt64 max_value = values.empty() ? 0 : *std::max_element(values.begin(), values.end()); + for (UInt64 v = max_value + 1; v > 0; --v) + builder->insert(Field(sortableString(v - 1))); + for (UInt64 v : values) + builder->insert(Field(sortableString(v))); + return builder->cut(max_value + 1, values.size()); + } + } + UNREACHABLE(); +} + +bool keyEquals(const ColumnRawPtrs & key, size_t lhs, size_t rhs, int hint) +{ + return std::all_of(key.begin(), key.end(), [&](const IColumn * col) { return col->compareAt(lhs, rhs, *col, hint) == 0; }); +} + +size_t oracleKeyRangeEnd(const ColumnRawPtrs & key, size_t begin, size_t end, int hint) +{ + if (begin >= end) + return begin; + size_t r = begin + 1; + while (r < end && keyEquals(key, r, begin, hint)) + ++r; + return r; +} + +/// Checks the three multi-column overloads against `oracleKeyRangeEnd` for every `begin` and two ends, and adds +/// the lengths of the runs it sees to `run_lengths`. +void checkKeyAgainstOracle(const Columns & key_columns, int hint, const std::string & label, std::set & run_lengths) +{ + const size_t n = key_columns.front()->size(); + ColumnRawPtrs key; + for (const auto & col : key_columns) + key.push_back(col.get()); + + for (size_t row = 0; row + 1 < n; ++row) + { + int cmp = 0; + for (size_t i = 0; i < key.size() && cmp == 0; ++i) + cmp = key[i]->compareAt(row, row + 1, *key[i], hint); + ASSERT_LE(cmp, 0) << label << ": the key is not sorted at row " << row; + } + + /// The key columns in reverse order after an unsorted column, selected back by `positions`. + auto unsorted = ColumnUInt64::create(); + for (size_t row = 0; row < n; ++row) + unsorted->getData().push_back(row % 2); + ColumnRawPtrs with_other_columns{unsorted.get()}; + std::vector positions; + for (size_t i = key.size(); i > 0; --i) + { + with_other_columns.push_back(key[i - 1]); + positions.push_back(i); + } + + SortDescription descr; + for (size_t i = 0; i < key.size(); ++i) + descr.emplace_back("k" + std::to_string(i), 1, hint); + + for (size_t end : {n / 2, n}) + { + for (size_t begin = 0; begin <= end; ++begin) + { + const size_t want = oracleKeyRangeEnd(key, begin, end, hint); + if (end == n && begin < end) + run_lengths.insert(want - begin); + + ASSERT_EQ(getEqualRangeEndAssumeSorted(key, begin, end, hint), want) << label << ": columns begin=" << begin << " end=" << end; + ASSERT_EQ(getEqualRangeEndAssumeSorted(with_other_columns, positions, begin, end, hint), want) + << label << ": positions begin=" << begin << " end=" << end; + ASSERT_EQ(getEqualRangeEndAssumeSorted(key, descr, begin, end), want) << label << ": descr begin=" << begin << " end=" << end; + } + } +} + +/// Calls `check(key_columns, label)` for sorted keys of `types` with the leading and trailing run lengths below. +template +void forEachKeyFixture(const std::vector & types, const std::vector & last_sub_runs, Check && check) +{ + const std::vector leading_runs{1, 7, 8, 9, 600}; + const std::vector sub_runs{1, 2, 7, 8, 9, 300, 0}; + + for (size_t leading_run : leading_runs) + { + for (size_t sub_run : sub_runs) + { + for (size_t last_sub_run : last_sub_runs) + { + std::vector key_sub_runs{sub_run}; + if (types.size() > 2) + key_sub_runs.push_back(last_sub_run); + + const size_t rows = std::max(3 * leading_run, 40); + const auto values = makeSortedKeyValues(rows, leading_run, key_sub_runs); + Columns key_columns; + for (size_t i = 0; i < types.size(); ++i) + key_columns.push_back(makeKeyColumn(types[i], values[i])); + + const std::string label = "types=" + std::to_string(static_cast(types[0])) + "," + std::to_string(static_cast(types[1])) + + " leading_run=" + std::to_string(leading_run) + " sub_runs=" + std::to_string(sub_run) + "," + std::to_string(last_sub_run); + check(key_columns, label); + if (::testing::Test::HasFatalFailure()) + return; + } + } + } +} + +void runKeyTests(const std::vector & types, int hint, const std::vector & last_sub_runs, std::set & run_lengths) +{ + forEachKeyFixture( + types, last_sub_runs, [&](const Columns & key_columns, const std::string & label) + { checkKeyAgainstOracle(key_columns, hint, label, run_lengths); }); +} + } TEST(SortedEqualRuns, ColumnVectorIntegers) @@ -441,3 +661,275 @@ TEST(SortedEqualRuns, ColumnNullableDefaultPath) checkAgainstOracle(*col, 1, "ColumnNullable no-NULL"); } } + +namespace +{ + +/// Runs shorter than, equal to and longer than the linear probe of the String search must all occur. +void runKeyTestsWithCoverage(const std::vector & types, int hint, const std::vector & last_sub_runs) +{ + std::set run_lengths; + runKeyTests(types, hint, last_sub_runs, run_lengths); + for (size_t len : {1, 2, 7, 8, 9, 300}) + EXPECT_TRUE(run_lengths.contains(len)) << "no run of length " << len; +} + +struct KeyTypes +{ + std::vector types; + int hint; + std::vector last_sub_runs; +}; + +const std::vector & allKeyTypes() +{ + static const std::vector key_types{ + {{KeyType::UInt64, KeyType::String}, 1, {1}}, + {{KeyType::String, KeyType::String}, 1, {1}}, + {{KeyType::NullableString, KeyType::Float64WithNaNAndZeros}, -1, {1}}, + {{KeyType::LowCardinalityString, KeyType::UInt32}, 1, {1}}, + {{KeyType::UInt64, KeyType::String, KeyType::UInt32}, 1, {1, 3, 8, 300}}, + }; + return key_types; +} + +} + +TEST(SortedEqualRuns, MultiColumnKeyOracle) +{ + for (const auto & key_types : allKeyTypes()) + { + runKeyTestsWithCoverage(key_types.types, key_types.hint, key_types.last_sub_runs); + if (::testing::Test::HasFatalFailure()) + return; + } +} + +TEST(SortedEqualRuns, MultiColumnHelperSingleAndNoColumns) +{ + for (const auto & p : makePatterns()) + { + auto col = makeSortedString(p); + const ColumnRawPtrs key{col.get()}; + const std::vector positions{0}; + SortDescription descr; + descr.emplace_back("s", 1, 1); + + const size_t n = col->size(); + for (size_t end : {n / 2, n}) + { + for (size_t begin = 0; begin <= end; ++begin) + { + const size_t want = oracleRangeEnd(*col, begin, end, 1); + ASSERT_EQ(getEqualRangeEndAssumeSorted(key, begin, end, 1), want) << "columns begin=" << begin << " end=" << end; + ASSERT_EQ(getEqualRangeEndAssumeSorted(key, positions, begin, end, 1), want) << "positions begin=" << begin << " end=" << end; + ASSERT_EQ(getEqualRangeEndAssumeSorted(key, descr, begin, end), want) << "descr begin=" << begin << " end=" << end; + } + } + } + + /// Without key columns all rows have the same key. + const ColumnRawPtrs no_key; + EXPECT_EQ(getEqualRangeEndAssumeSorted(no_key, 3, 10, 1), 10u); +} + +namespace +{ + +ColumnRawPtrs rawPtrs(const Columns & columns) +{ + ColumnRawPtrs res; + for (const auto & col : columns) + res.push_back(col.get()); + return res; +} + +/// Finds the runs of `[0, end)` one after another, each search starting where the previous run ended. +template +void checkSortedKeyRunsWalk(SortedKeyRuns & runs, const ColumnRawPtrs & key, size_t end, int hint, Search && search, const std::string & label) +{ + std::vector cached_calls(key.size()); + std::vector stateless_calls(key.size()); + std::vector expected_calls(key.size()); + auto counted_cached = [&](size_t i, size_t from, size_t bound) + { + ++cached_calls[i]; + return search(i, from, bound); + }; + auto counted_stateless = [&](size_t i, size_t from, size_t bound) + { + ++stateless_calls[i]; + return search(i, from, bound); + }; + + for (size_t begin = 0; begin < end;) + { + const size_t run_end = runs.findRunEnd(begin, end, counted_cached); + ASSERT_GT(run_end, begin) << label << ": begin=" << begin << " end=" << end; + ASSERT_EQ(run_end, oracleKeyRangeEnd(key, begin, end, hint)) << label << ": begin=" << begin << " end=" << end; + (void)findKeyRangeEndAssumeSorted(key.size(), begin, end, counted_stateless); + + /// Searching forward, column `i` is searched only from the first row of a run of columns `0..i`, and the + /// search stops after the first column whose run of columns `0..i` from `begin` is one row long. + size_t first_new = 0; + if (begin > 0) + { + while (first_new < key.size() && key[first_new]->compareAt(begin - 1, begin, *key[first_new], hint) == 0) + ++first_new; + ASSERT_LT(first_new, key.size()) << label << ": begin=" << begin << " is not a run boundary"; + } + for (size_t i = first_new; i < key.size(); ++i) + { + ++expected_calls[i]; + const ColumnRawPtrs prefix(key.begin(), key.begin() + i + 1); + if (oracleKeyRangeEnd(prefix, begin, end, hint) <= begin + 1) + break; + } + begin = run_end; + } + +#ifdef DEBUG_OR_SANITIZER_BUILD + /// `findRunEnd` checks every result for two or more columns against the stateless search through the same callback. + if (key.size() >= 2) + for (size_t i = 0; i < key.size(); ++i) + cached_calls[i] -= stateless_calls[i]; +#endif + for (size_t i = 0; i < key.size(); ++i) + EXPECT_EQ(cached_calls[i], expected_calls[i]) << label << ": column " << i << " end=" << end; + EXPECT_LE( + std::accumulate(cached_calls.begin(), cached_calls.end(), size_t{0}), + std::accumulate(stateless_calls.begin(), stateless_calls.end(), size_t{0})) + << label << ": end=" << end; +} + +} + +TEST(SortedEqualRuns, SortedKeyRunsWalk) +{ + for (const auto & key_types : allKeyTypes()) + { + forEachKeyFixture( + key_types.types, + key_types.last_sub_runs, + [&](const Columns & key_columns, const std::string & label) + { + const ColumnRawPtrs key = rawPtrs(key_columns); + SortDescription descr; + for (size_t i = 0; i < key.size(); ++i) + descr.emplace_back("k" + std::to_string(i), 1, key_types.hint); + + const size_t n = key.front()->size(); + for (size_t end : {n / 2, n}) + { + SortedKeyRuns runs(key.size()); + checkSortedKeyRunsWalk( + runs, key, end, key_types.hint, + [&](size_t i, size_t from, size_t bound) { return key[i]->getEqualRangeEndAssumeSorted(from, bound, key_types.hint); }, + label + " columns"); + if (::testing::Test::HasFatalFailure()) + return; + + SortedKeyRuns descr_runs(key.size()); + checkSortedKeyRunsWalk( + descr_runs, key, end, key_types.hint, + [&](size_t i, size_t from, size_t bound) + { return key[i]->getEqualRangeEndAssumeSorted(from, bound, descr[i].nulls_direction); }, + label + " descr"); + if (::testing::Test::HasFatalFailure()) + return; + } + }); + if (::testing::Test::HasFatalFailure()) + return; + } +} + +/// Searches from every row, also from rows inside a run and before the previous search. +TEST(SortedEqualRuns, SortedKeyRunsAnyOrder) +{ + for (const auto & key_types : allKeyTypes()) + { + forEachKeyFixture( + key_types.types, + key_types.last_sub_runs, + [&](const Columns & key_columns, const std::string & label) + { + const ColumnRawPtrs key = rawPtrs(key_columns); + auto search = [&](size_t i, size_t from, size_t bound) { return key[i]->getEqualRangeEndAssumeSorted(from, bound, key_types.hint); }; + + const size_t n = key.front()->size(); + for (size_t end : {n / 2, n}) + { + SortedKeyRuns runs(key.size()); + for (size_t begin = end + 1; begin > 0; --begin) + ASSERT_EQ(runs.findRunEnd(begin - 1, end, search), oracleKeyRangeEnd(key, begin - 1, end, key_types.hint)) + << label << ": descending begin=" << begin - 1 << " end=" << end; + for (size_t begin = 0; begin <= end; ++begin) + ASSERT_EQ(runs.findRunEnd(begin, end, search), oracleKeyRangeEnd(key, begin, end, key_types.hint)) + << label << ": ascending begin=" << begin << " end=" << end; + } + }); + if (::testing::Test::HasFatalFailure()) + return; + } +} + +TEST(SortedEqualRuns, SortedKeyRunsReset) +{ + const size_t rows = 1800; + Columns first; + Columns second; + for (const auto & values : makeSortedKeyValues(rows, 600, {300})) + first.push_back(makeKeyColumn(KeyType::UInt64, values)); + for (const auto & values : makeSortedKeyValues(rows, 7, {2})) + second.push_back(makeKeyColumn(KeyType::UInt64, values)); + const ColumnRawPtrs first_key = rawPtrs(first); + const ColumnRawPtrs second_key = rawPtrs(second); + + SortedKeyRuns runs(2); + + /// Leaves the runs of the first key at row 0 remembered. + auto first_search = [&](size_t i, size_t from, size_t bound) { return first_key[i]->getEqualRangeEndAssumeSorted(from, bound, 1); }; + for (size_t begin = rows; begin > 0; --begin) + ASSERT_EQ(runs.findRunEnd(begin - 1, rows, first_search), oracleKeyRangeEnd(first_key, begin - 1, rows, 1)) << "begin=" << begin - 1; + + runs.reset(2); + checkSortedKeyRunsWalk( + runs, second_key, rows, 1, + [&](size_t i, size_t from, size_t bound) { return second_key[i]->getEqualRangeEndAssumeSorted(from, bound, 1); }, + "after reset"); +} + +TEST(SortedEqualRuns, SortedKeyRunsSingleAndNoColumns) +{ + for (const auto & p : makePatterns()) + { + auto col = makeSortedString(p); + auto search = [&](size_t i, size_t from, size_t bound) + { + EXPECT_EQ(i, 0u); + return col->getEqualRangeEndAssumeSorted(from, bound, 1); + }; + + const size_t n = col->size(); + for (size_t end : {n / 2, n}) + { + SortedKeyRuns runs(1); + for (size_t begin = end + 1; begin > 0; --begin) + ASSERT_EQ(runs.findRunEnd(begin - 1, end, search), col->getEqualRangeEndAssumeSorted(begin - 1, end, 1)) + << "begin=" << begin - 1 << " end=" << end; + } + } + + /// Without key columns all rows have the same key. + auto no_search = [](size_t, size_t, size_t) -> size_t + { + ADD_FAILURE() << "no column to search"; + return 0; + }; + SortedKeyRuns no_key; + EXPECT_EQ(no_key.findRunEnd(3, 10, no_search), 10u); + SortedKeyRuns no_key_after_reset(2); + no_key_after_reset.reset(0); + EXPECT_EQ(no_key_after_reset.findRunEnd(3, 10, no_search), 10u); +} diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index 34ab10bcb5ba4..56c51693c8a24 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -770,14 +771,8 @@ namespace detail template size_t equalRangeEndAcrossColumns(size_t count, size_t begin, size_t end, GetColumn && column, GetHint && hint) { - size_t run_end = end; - for (size_t i = 0; i < count; ++i) - { - run_end = column(i)->getEqualRangeEndAssumeSorted(begin, run_end, hint(i)); - if (run_end <= begin + 1) - break; /// single-row run: cannot shrink further - } - return run_end; + return findKeyRangeEndAssumeSorted( + count, begin, end, [&](size_t i, size_t from, size_t bound) { return column(i)->getEqualRangeEndAssumeSorted(from, bound, hint(i)); }); } } @@ -813,4 +808,43 @@ size_t getEqualRangeEndAssumeSorted(const TColumns & columns, const TSortDescrip return detail::equalRangeEndAcrossColumns( descr.size(), begin, end, [&](size_t i) -> decltype(auto) { return columns[i]; }, [&](size_t i) { return descr[i].nulls_direction; }); } + +/** Same as the overloads above, for a caller that finds the runs of one range one after another (see `SortedKeyRuns`). + * `runs` must be sized to the number of key columns and reset as `SortedKeyRuns` requires. + */ +template +ALWAYS_INLINE size_t getEqualRangeEndAssumeSorted(SortedKeyRuns & runs, const TColumns & columns, size_t begin, size_t end, int nan_direction_hint) +{ + if (columns.size() < 2) + return getEqualRangeEndAssumeSorted(columns, begin, end, nan_direction_hint); + chassert(runs.keySize() == columns.size()); + return runs.findRunEnd( + begin, end, [&](size_t i, size_t from, size_t bound) { return columns[i]->getEqualRangeEndAssumeSorted(from, bound, nan_direction_hint); }); +} + +template +ALWAYS_INLINE size_t getEqualRangeEndAssumeSorted( + SortedKeyRuns & runs, const TColumns & columns, const std::vector & positions, size_t begin, size_t end, int nan_direction_hint) +{ + if (positions.size() < 2) + return getEqualRangeEndAssumeSorted(columns, positions, begin, end, nan_direction_hint); + chassert(runs.keySize() == positions.size()); + return runs.findRunEnd( + begin, + end, + [&](size_t i, size_t from, size_t bound) { return columns[positions[i]]->getEqualRangeEndAssumeSorted(from, bound, nan_direction_hint); }); +} + +template +requires requires (const TSortDescription & d) { d.size(); d[0].nulls_direction; } +ALWAYS_INLINE size_t getEqualRangeEndAssumeSorted(SortedKeyRuns & runs, const TColumns & columns, const TSortDescription & descr, size_t begin, size_t end) +{ + if (descr.size() < 2) + return getEqualRangeEndAssumeSorted(columns, descr, begin, end); + chassert(runs.keySize() == descr.size()); + return runs.findRunEnd( + begin, + end, + [&](size_t i, size_t from, size_t bound) { return columns[i]->getEqualRangeEndAssumeSorted(from, bound, descr[i].nulls_direction); }); +} } diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index edf08106a443c..e08cd9c9d16dc 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -357,6 +357,8 @@ class MergeJoinCursor /// We use zero position as 'is first range in block' detector if (position()) throw Exception(ErrorCodes::LOGICAL_ERROR, "MergeJoinCursor is expected to have initial position 0"); + + key_runs.reset(impl.sort_columns.size()); } size_t position() const { return impl.getPos(); } @@ -419,6 +421,7 @@ class MergeJoinCursor private: SortCursorImpl impl; Columns column_holder; + SortedKeyRuns key_runs; bool has_left_nullable = false; bool has_right_nullable = false; @@ -464,7 +467,7 @@ class MergeJoinCursor size_t getEqualLength() { const size_t base_pos = impl.getPos(); - return getEqualRangeEndAssumeSorted(impl.sort_columns, base_pos, impl.rows, 1) - base_pos; + return getEqualRangeEndAssumeSorted(key_runs, impl.sort_columns, base_pos, impl.rows, 1) - base_pos; } }; diff --git a/src/Processors/Transforms/DistinctSortedStreamTransform.cpp b/src/Processors/Transforms/DistinctSortedStreamTransform.cpp index 40479f2eb41e9..46d6d01d455cd 100644 --- a/src/Processors/Transforms/DistinctSortedStreamTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedStreamTransform.cpp @@ -144,7 +144,7 @@ std::pair DistinctSortedStreamTransform::continueWithPrevRange(c return {0, 0}; size_t output_rows = 0; - const size_t range_end = getEqualRangeEndAssumeSorted(sorted_columns, sorted_columns_descr, 0, chunk_rows); + const size_t range_end = getEqualRangeEndAssumeSorted(key_runs, sorted_columns, sorted_columns_descr, 0, chunk_rows); if (other_columns.empty()) std::fill(filter.begin(), filter.begin() + range_end, 0); /// skip rows already included in distinct on previous transform() else @@ -168,6 +168,7 @@ void DistinctSortedStreamTransform::transform(Chunk & chunk) Columns input_columns = chunk.detachColumns(); /// split input columns into sorted and other("non-sorted") columns initChunkProcessing(input_columns); + key_runs.reset(sorted_columns.size()); /// build filter: /// (1) find range with the same values in sorted columns -> [range_begin, range_end) @@ -181,7 +182,7 @@ void DistinctSortedStreamTransform::transform(Chunk & chunk) while (range_end != chunk_rows) { // find new range [range_begin, range_end) - range_end = getEqualRangeEndAssumeSorted(sorted_columns, sorted_columns_descr, range_begin, chunk_rows); + range_end = getEqualRangeEndAssumeSorted(key_runs, sorted_columns, sorted_columns_descr, range_begin, chunk_rows); // update filter for range if (other_columns.empty()) diff --git a/src/Processors/Transforms/DistinctSortedStreamTransform.h b/src/Processors/Transforms/DistinctSortedStreamTransform.h index 897b80319095e..93130f0bb497f 100644 --- a/src/Processors/Transforms/DistinctSortedStreamTransform.h +++ b/src/Processors/Transforms/DistinctSortedStreamTransform.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -64,6 +65,7 @@ class DistinctSortedStreamTransform final : public ISimpleTransform const SortDescription sorted_columns_descr; ColumnNumbers sorted_columns_pos; ColumnRawPtrs sorted_columns; // used during processing + SortedKeyRuns key_runs; // runs of the sorting prefix in the current chunk ColumnNumbers other_columns_pos; Sizes other_columns_sizes; diff --git a/src/Processors/Transforms/LimitByTransform.cpp b/src/Processors/Transforms/LimitByTransform.cpp index 32ac966c17bee..a1168bb4d1a2f 100644 --- a/src/Processors/Transforms/LimitByTransform.cpp +++ b/src/Processors/Transforms/LimitByTransform.cpp @@ -458,6 +458,7 @@ void LimitBySortedStreamTransform::transform(Chunk & chunk) /// Segment the sorted chunk into maximal runs of rows that share one grouping key. Each run is /// one group. UInt64 current_run_start_row = 0; + SortedKeyRuns key_runs(normalized_grouping_key_columns.size()); FailPointInjection::pauseFailPoint(FailPoints::limit_by_sorted_stream_transform_pause); @@ -474,7 +475,7 @@ void LimitBySortedStreamTransform::transform(Chunk & chunk) if (run_count == 5) FailPointInjection::pauseFailPoint(FailPoints::limit_by_sorted_stream_transform_mid_loop_pause); - const UInt64 run_end = getEqualRangeEndAssumeSorted(normalized_grouping_key_columns, current_run_start_row, row_count, 1); + const UInt64 run_end = getEqualRangeEndAssumeSorted(key_runs, normalized_grouping_key_columns, current_run_start_row, row_count, 1); processRun(current_run_start_row, run_end - current_run_start_row); /// A group boundary inside the chunk resets the per-group counter before the next run. diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 8170289ad8e72..2c01c042ac75b 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -223,26 +223,28 @@ size_t ALWAYS_INLINE nextDistinct(FullMergeJoinCursor & impl) { chassert(impl.isValid()); const size_t start_pos = impl.getRow(); - size_t run_end = impl.rows; /// Find the end of the run of rows that share the same (multi-column) key, starting at start_pos. - for (size_t i = 0; i < impl.sort_columns.size(); ++i) - { - const auto * nm = getNullMapData(impl.null_maps[i]); - const bool ref_is_null = nm && (*nm)[start_pos] != 0; + const size_t run_end = impl.key_runs.findRunEnd( + start_pos, + impl.rows, + [&](size_t i, size_t from, size_t bound) + { + const auto * nm = getNullMapData(impl.null_maps[i]); + const bool ref_is_null = nm && (*nm)[from] != 0; - if (nm) - run_end = findEqualRangeEndAssumeSorted(start_pos, run_end, 16, [&](size_t row) { return ((*nm)[row] != 0) == ref_is_null; }); + size_t column_run_end = bound; + if (nm) + column_run_end + = findEqualRangeEndAssumeSorted(from, column_run_end, 16, [&](size_t row) { return ((*nm)[row] != 0) == ref_is_null; }); - if (!ref_is_null) - run_end = impl.sort_columns[i]->getEqualRangeEndAssumeSorted(start_pos, run_end, 1); + if (!ref_is_null) + column_run_end = impl.sort_columns[i]->getEqualRangeEndAssumeSorted(from, column_run_end, 1); - if (run_end <= start_pos + 1) - break; - } - const size_t length = run_end - start_pos; - impl.pos = start_pos + length; - return length; + return column_run_end; + }); + impl.pos = run_end; + return run_end - start_pos; } /// Lookup the equal range of the cursor, without moving the cursor @@ -459,6 +461,7 @@ void FullMergeJoinCursor::setChunk(Chunk && chunk) rows = current_chunk.getNumRows(); sort_columns.clear(); null_maps.clear(); + key_runs.reset(0); asof_column = nullptr; if (!current_chunk) @@ -484,6 +487,8 @@ void FullMergeJoinCursor::setChunk(Chunk && chunk) asof_column = std::move(sort_columns.back()); sort_columns.pop_back(); } + + key_runs.reset(sort_columns.size()); } bool FullMergeJoinCursor::fullyCompleted() const diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 19e179438d59a..a7f4535729235 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -220,6 +221,7 @@ class FullMergeJoinCursor absl::InlinedVector sort_columns; absl::InlinedVector null_maps; + SortedKeyRuns key_runs; ColumnPtr asof_column = nullptr; size_t pos = 0; size_t rows = 0; diff --git a/src/Processors/Transforms/NegativeLimitByTransform.cpp b/src/Processors/Transforms/NegativeLimitByTransform.cpp index 072588ddace83..565500895bbd0 100644 --- a/src/Processors/Transforms/NegativeLimitByTransform.cpp +++ b/src/Processors/Transforms/NegativeLimitByTransform.cpp @@ -479,9 +479,10 @@ void NegativeLimitBySortedStreamTransform::consume(Chunk chunk) /// Segment the sorted chunk into maximal runs of rows that share the same key; each run is /// one group. + SortedKeyRuns key_runs(normalized_keys.size()); while (run_start < num_rows) { - const UInt64 run_end = getEqualRangeEndAssumeSorted(normalized_keys, run_start, num_rows, 1); + const UInt64 run_end = getEqualRangeEndAssumeSorted(key_runs, normalized_keys, run_start, num_rows, 1); append_run(run_start, run_end - run_start); diff --git a/tests/performance/full_sorting_merge_join.xml b/tests/performance/full_sorting_merge_join.xml index 874b4be213944..39aef42d749e6 100644 --- a/tests/performance/full_sorting_merge_join.xml +++ b/tests/performance/full_sorting_merge_join.xml @@ -67,6 +67,11 @@ OPTIMIZE TABLE fsm_nullable_left FINAL OPTIMIZE TABLE fsm_nullable_right FINAL + + CREATE TABLE fsm_u64_two_keys (k1 UInt64, k2 UInt64) ENGINE = MergeTree ORDER BY (k1, k2) + INSERT INTO fsm_u64_two_keys SELECT number % 10000, sipHash64(number) FROM numbers_mt(5000000) + OPTIMIZE TABLE fsm_u64_two_keys FINAL + SELECT count() FROM test_u64_left_{card} AS a JOIN test_u64_right_{card} AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' SELECT count() FROM test_str_left AS a JOIN test_str_right AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' SELECT count() FROM test_lc_left AS a JOIN test_lc_right AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' @@ -78,6 +83,7 @@ SELECT count() FROM fsm_u64_mult2 AS a JOIN fsm_u64_mult3 AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' SELECT count() FROM fsm_str_sparse_left AS a JOIN fsm_str_sparse_right AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' SELECT count() FROM fsm_nullable_left AS a JOIN fsm_nullable_right AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge', join_use_nulls = 1 + SELECT count() FROM fsm_u64_two_keys AS a JOIN fsm_u64_two_keys AS b ON a.k1 = b.k1 AND a.k2 = b.k2 FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' DROP TABLE IF EXISTS test_u64_left_{card} DROP TABLE IF EXISTS test_u64_right_{card} @@ -93,4 +99,5 @@ DROP TABLE IF EXISTS fsm_str_sparse_right DROP TABLE IF EXISTS fsm_nullable_left DROP TABLE IF EXISTS fsm_nullable_right + DROP TABLE IF EXISTS fsm_u64_two_keys diff --git a/tests/performance/limit_by_in_pk_order.xml b/tests/performance/limit_by_in_pk_order.xml index b7eed01dcb338..75e8f67ccd33d 100644 --- a/tests/performance/limit_by_in_pk_order.xml +++ b/tests/performance/limit_by_in_pk_order.xml @@ -201,6 +201,28 @@ SETTINGS allow_limit_by_partitions_independently = 1 + + + CREATE TABLE limit_by_in_pk_order_str + (a String, b String, z UInt64) + ENGINE = MergeTree + ORDER BY (a, b) + + + + INSERT INTO limit_by_in_pk_order_str + SELECT concat('k', toString(sipHash64(number % 10000))), hex(sipHash64(number)), number + FROM numbers(5000000) + + OPTIMIZE TABLE limit_by_in_pk_order_str FINAL + + + SELECT a, b, z FROM limit_by_in_pk_order_str + ORDER BY a, b + LIMIT 1 BY a, b + FORMAT NULL + + DROP TABLE IF EXISTS limit_by_in_pk_order_int DROP TABLE IF EXISTS limit_by_in_pk_order_dt DROP TABLE IF EXISTS limit_by_in_pk_order_low @@ -209,4 +231,5 @@ DROP TABLE IF EXISTS limit_by_in_pk_order_merge_1 DROP TABLE IF EXISTS limit_by_in_pk_order_merge_2 DROP TABLE IF EXISTS limit_by_in_pk_order_part_in_order + DROP TABLE IF EXISTS limit_by_in_pk_order_str diff --git a/tests/performance/negative_limit_by_in_order.xml b/tests/performance/negative_limit_by_in_order.xml index 70445dde56953..b8dfeb025a41d 100644 --- a/tests/performance/negative_limit_by_in_order.xml +++ b/tests/performance/negative_limit_by_in_order.xml @@ -31,15 +31,22 @@ INSERT INTO test_datetime SELECT toDateTime(1700000000 + number % 100) FROM numbers_mt(10000000) OPTIMIZE TABLE test_datetime FINAL + + CREATE TABLE test_str_two_keys (a String, b String) ENGINE = MergeTree ORDER BY (a, b) + INSERT INTO test_str_two_keys SELECT concat('k', toString(sipHash64(number % 10000))), hex(sipHash64(number)) FROM numbers_mt(5000000) + OPTIMIZE TABLE test_str_two_keys FINAL + SELECT k FROM test_u64_{card} ORDER BY k LIMIT -1 BY k FORMAT Null SELECT s FROM test_str ORDER BY s LIMIT -1 BY s FORMAT Null SELECT lc FROM test_lc ORDER BY lc LIMIT -1 BY lc FORMAT Null SELECT nk FROM test_nullable ORDER BY nk LIMIT -1 BY nk FORMAT Null SELECT dt FROM test_datetime ORDER BY dt LIMIT -1 BY dt FORMAT Null + SELECT a, b FROM test_str_two_keys ORDER BY a, b LIMIT -1 BY a, b FORMAT Null DROP TABLE IF EXISTS test_u64_{card} DROP TABLE IF EXISTS test_str DROP TABLE IF EXISTS test_lc DROP TABLE IF EXISTS test_nullable DROP TABLE IF EXISTS test_datetime + DROP TABLE IF EXISTS test_str_two_keys diff --git a/tests/performance/sorted_key_prefix_runs.xml b/tests/performance/sorted_key_prefix_runs.xml new file mode 100644 index 0000000000000..289a942b6f471 --- /dev/null +++ b/tests/performance/sorted_key_prefix_runs.xml @@ -0,0 +1,11 @@ + + + CREATE TABLE sorted_key_prefix_runs (a String, b String, x UInt64) ENGINE = MergeTree ORDER BY (a, b) + INSERT INTO sorted_key_prefix_runs SELECT concat('k', toString(sipHash64(intDiv(number, 2) % 1000))), hex(sipHash64(intDiv(number, 2))), number FROM numbers_mt(5000000) + OPTIMIZE TABLE sorted_key_prefix_runs FINAL + + SELECT DISTINCT a, b FROM sorted_key_prefix_runs FORMAT Null SETTINGS max_threads = 1 + SELECT count() FROM sorted_key_prefix_runs AS l INNER JOIN sorted_key_prefix_runs AS r USING (a, b) FORMAT Null SETTINGS join_algorithm = 'partial_merge' + + DROP TABLE IF EXISTS sorted_key_prefix_runs + From 1d6cbc23c681c2c682b01b952e79c662668bc80a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 10 Oct 2026 00:20:23 +0000 Subject: [PATCH 245/296] Backport #124612 to 26.8: Fix skip index analysis through casts and on missing map keys or JSON paths --- .../optimizeDirectReadFromTextIndex.cpp | 7 ++- .../MergeTree/MergeTreeIndexConditionText.cpp | 41 +++++++++------ .../MergeTreeIndexTextPreprocessor.cpp | 5 ++ .../MergeTreeIndexTextPreprocessor.h | 3 ++ src/Storages/MergeTree/RPNBuilder.cpp | 4 +- ...ssless_conversion_time_zone_bool.reference | 12 +++++ ...ter_lossless_conversion_time_zone_bool.sql | 42 ++++++++++++++++ ..._preprocessor_cast_drop_nullable.reference | 6 +++ ..._index_preprocessor_cast_drop_nullable.sql | 50 +++++++++++++++++++ ...ex_map_json_default_value_throws.reference | 10 ++++ ...xt_index_map_json_default_value_throws.sql | 44 ++++++++++++++++ 11 files changed, 205 insertions(+), 19 deletions(-) create mode 100644 tests/queries/0_stateless/05339_bloom_filter_lossless_conversion_time_zone_bool.reference create mode 100644 tests/queries/0_stateless/05339_bloom_filter_lossless_conversion_time_zone_bool.sql create mode 100644 tests/queries/0_stateless/05340_text_index_preprocessor_cast_drop_nullable.reference create mode 100644 tests/queries/0_stateless/05340_text_index_preprocessor_cast_drop_nullable.sql create mode 100644 tests/queries/0_stateless/05341_text_index_map_json_default_value_throws.reference create mode 100644 tests/queries/0_stateless/05341_text_index_map_json_default_value_throws.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp index 617fd50849313..7ad5d7b69be93 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp @@ -718,7 +718,8 @@ class TextIndexDAGReplacer /// Check that preprocessor contains current expression as its argument. if (hasSubexpression(preprocessor_output, haystack_name)) { - new_children[0] = haystack; + /// Keep a `CAST` that drops `Nullable` under the preprocessor, so that the predicate still throws on NULL. + new_children[0] = unwrapLosslessConversion(arg_haystack, /*allow_drop_nullable=*/ false); if (apply_postprocessor) { @@ -727,7 +728,9 @@ class TextIndexDAGReplacer else { ActionsDAG::NodeRawConstPtrs merged_outputs; - actions_dag.mergeNodes(preprocessor_dag.clone(), &merged_outputs); + actions_dag.mergeNodes( + preprocessor->getActionsDAGForColumn(new_children[0]->result_name, new_children[0]->result_type), + &merged_outputs); chassert(merged_outputs.size() == 1); new_children[0] = merged_outputs.front(); diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index 81f576d8a6e89..ffa2b23e1a006 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -1638,6 +1638,28 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( return false; } +/// Whether the predicate of the sub-DAG may be true on the default value of its single input column, +/// i.e. for a row with a missing map key or JSON path. Unknown if the predicate throws on the default value, +/// e.g. `toUInt64(m['key'])` on an empty string, as in `filterResultForNotMatchedRows` for JOIN. +static bool mayBeTrueOnDefaultValue(const ActionsDAG & subdag) +{ + const auto * input = subdag.getInputs().front(); + ActionsDAG::IntermediateExecutionResult default_input; + default_input.emplace(input, ColumnWithTypeAndName(input->result_type->createColumnConstWithDefaultValue(1), input->result_type, input->result_name)); + + ColumnsWithTypeAndName result; + try + { + result = ActionsDAG::evaluatePartialResult(default_input, subdag.getOutputs(), /*input_rows_count=*/ 1); + } + catch (const Exception &) + { + return true; + } + + return !result.front().column || result.front().column->getBool(0); +} + bool MergeTreeIndexConditionText::traverseMapElementKeyNode(const RPNBuilderFunctionTreeNode & function_node, RPNElement & out) const { /// Here we check whether we can use index defined for `mapKeys(m)` for functions like `func(arrayElement(m, 'const_key'), ...)`. @@ -1662,7 +1684,6 @@ bool MergeTreeIndexConditionText::traverseMapElementKeyNode(const RPNBuilderFunc return false; auto required_column = required_columns.front(); - auto output_column_name = outputs.front()->result_name; std::optional key_const_value; @@ -1745,13 +1766,8 @@ bool MergeTreeIndexConditionText::traverseMapElementKeyNode(const RPNBuilderFunc } /// Evaluate function on the empty map. Empty map will return default value for any key. - Block block{{required_column.type->createColumnConstWithDefaultValue(1), required_column.type, required_column.name}}; - ExpressionActions actions(std::move(subdag)); - actions.execute(block); - const auto & result_column = block.getByName(output_column_name).column; - /// If the function returns true for the empty map, we cannot use index. - if (result_column->getBool(0)) + if (mayBeTrueOnDefaultValue(subdag)) return false; auto tokens = stringToTokens(std::string_view(*key_const_value)); @@ -1815,8 +1831,7 @@ bool MergeTreeIndexConditionText::traverseJSONSubcolumnKeyNode( if (required_columns.size() != 1 || outputs.size() != 1) return false; - auto required_column = required_columns.front(); - auto output_column_name = outputs.front()->result_name; + const auto & required_column = required_columns.front(); /// Try to match the required column to a JSON subcolumn with JSONAllPaths index. auto json_info = tryMatchJSONSubcolumnToIndex(required_column.name, header, "JSONAllPaths"); @@ -1846,13 +1861,7 @@ bool MergeTreeIndexConditionText::traverseJSONSubcolumnKeyNode( /// Evaluate the function on a default column value. /// If the function returns true for the default value (what we'd get when the path is missing), /// we cannot safely skip the granule. - Block block{{required_column.type->createColumnConstWithDefaultValue(1), - required_column.type, required_column.name}}; - ExpressionActions actions(std::move(subdag)); - actions.execute(block); - const auto & result_column = block.getByName(output_column_name).column; - - if (result_column->getBool(0)) + if (mayBeTrueOnDefaultValue(subdag)) return false; auto tokens = stringToTokens(Field(json_info->path)); diff --git a/src/Storages/MergeTree/MergeTreeIndexTextPreprocessor.cpp b/src/Storages/MergeTree/MergeTreeIndexTextPreprocessor.cpp index 643552843bea8..36d75fdf4fa5a 100644 --- a/src/Storages/MergeTree/MergeTreeIndexTextPreprocessor.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexTextPreprocessor.cpp @@ -192,6 +192,11 @@ ASTPtr MergeTreeIndexTextPreprocessor::getExpressionAST(const String & col_name) return result; } +ActionsDAG MergeTreeIndexTextPreprocessor::getActionsDAGForColumn(const String & col_name, const DataTypePtr & col_type) const +{ + return buildActionsDAGFromAST(getExpressionAST(col_name), {{col_name, col_type}}); +} + String MergeTreeIndexTextPreprocessor::processConstant(const String & input) const { if (actions_for_constant.getActions().empty()) diff --git a/src/Storages/MergeTree/MergeTreeIndexTextPreprocessor.h b/src/Storages/MergeTree/MergeTreeIndexTextPreprocessor.h index 575b68f396537..2a5b68a765cea 100644 --- a/src/Storages/MergeTree/MergeTreeIndexTextPreprocessor.h +++ b/src/Storages/MergeTree/MergeTreeIndexTextPreprocessor.h @@ -29,6 +29,9 @@ class MergeTreeIndexTextPreprocessor /// The preprocessor expression as an AST reading from an identifier named `col_name`, to splice into a larger expression. ASTPtr getExpressionAST(const String & col_name) const; + /// The preprocessor expression over an input column `col_name` of type `col_type`, to merge into a larger DAG. + ActionsDAG getActionsDAGForColumn(const String & col_name, const DataTypePtr & col_type) const; + bool isLowerOrUpper() const { return is_lower_or_upper; } private: diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 7153007436ded..c57b19d4dd1b1 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -599,7 +599,9 @@ bool isLosslessConversion(const DataTypePtr & from, const DataTypePtr & to, bool from_type = removeNullable(from_type); to_type = removeNullable(to_type); - if (from_type->equals(*to_type)) + /// Not `equals`: it ignores the time zone of `DateTime`, which changes how a literal is parsed, + /// and the custom name of `Bool`, to which `CAST` converts a `UInt8` 2 as 1. + if (from_type->getName() == to_type->getName()) return true; const auto * from_array = typeid_cast(from_type.get()); diff --git a/tests/queries/0_stateless/05339_bloom_filter_lossless_conversion_time_zone_bool.reference b/tests/queries/0_stateless/05339_bloom_filter_lossless_conversion_time_zone_bool.reference new file mode 100644 index 0000000000000..e440007272bce --- /dev/null +++ b/tests/queries/0_stateless/05339_bloom_filter_lossless_conversion_time_zone_bool.reference @@ -0,0 +1,12 @@ +-- time zone +1 +1 +1 +3 +-- Bool +2 +3 +2 +3 +-- the same time zone is still looked through +2 diff --git a/tests/queries/0_stateless/05339_bloom_filter_lossless_conversion_time_zone_bool.sql b/tests/queries/0_stateless/05339_bloom_filter_lossless_conversion_time_zone_bool.sql new file mode 100644 index 0000000000000..62a72acabcafb --- /dev/null +++ b/tests/queries/0_stateless/05339_bloom_filter_lossless_conversion_time_zone_bool.sql @@ -0,0 +1,42 @@ +-- A skip index is not analyzed through a cast to another time zone or from `UInt8` to `Bool`. These types are +-- equal by `IDataType::equals`, but the time zone changes how a literal is parsed, and `Bool` converts 2 to 1. + +SET enable_analyzer = 1; +SET use_skip_indexes = 1; +SET use_skip_indexes_on_data_read = 0; +SET use_query_condition_cache = 0; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + id UInt32, + dt DateTime('Asia/Tokyo'), + dt_utc DateTime('UTC') ALIAS dt, + u UInt8, + b Bool ALIAS u, + INDEX idx_dt dt TYPE bloom_filter GRANULARITY 1, + INDEX idx_u u TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab (id, dt, u) VALUES + (1, toDateTime('2024-01-01 00:00:00', 'UTC'), 0), + (2, toDateTime('2024-01-01 00:00:00', 'Asia/Tokyo'), 1), + (3, toDateTime('2024-01-02 00:00:00', 'UTC'), 2); + +SELECT '-- time zone'; +SELECT id FROM tab WHERE dt_utc = '2024-01-01 00:00:00' ORDER BY id; +SELECT id FROM tab WHERE CAST(dt, 'DateTime(\'UTC\')') = '2024-01-01 00:00:00' ORDER BY id; +SELECT id FROM tab WHERE CAST(dt, 'DateTime(\'UTC\')') IN ('2024-01-01 00:00:00', '2024-01-02 00:00:00') ORDER BY id; + +SELECT '-- Bool'; +SELECT id FROM tab WHERE b = true ORDER BY id; +SELECT id FROM tab WHERE CAST(u, 'Bool') IN (true) ORDER BY id; + +SELECT '-- the same time zone is still looked through'; +SELECT id FROM tab WHERE CAST(dt, 'Nullable(DateTime(\'Asia/Tokyo\'))') = '2024-01-01 00:00:00' ORDER BY id SETTINGS force_data_skipping_indices = 'idx_dt'; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/05340_text_index_preprocessor_cast_drop_nullable.reference b/tests/queries/0_stateless/05340_text_index_preprocessor_cast_drop_nullable.reference new file mode 100644 index 0000000000000..6ba3cbec83523 --- /dev/null +++ b/tests/queries/0_stateless/05340_text_index_preprocessor_cast_drop_nullable.reference @@ -0,0 +1,6 @@ +-- preprocessor and postprocessor +5 +1 +5 +-- preprocessor that maps NULL to a string +5 diff --git a/tests/queries/0_stateless/05340_text_index_preprocessor_cast_drop_nullable.sql b/tests/queries/0_stateless/05340_text_index_preprocessor_cast_drop_nullable.sql new file mode 100644 index 0000000000000..396d8d65ea143 --- /dev/null +++ b/tests/queries/0_stateless/05340_text_index_preprocessor_cast_drop_nullable.sql @@ -0,0 +1,50 @@ +-- The text index is analyzed through a `CAST` that drops `Nullable`, and the preprocessor is applied to the +-- haystack of the row-level function. The `CAST` stays under the preprocessor, so NULL still throws. + +SET enable_analyzer = 1; +SET use_skip_indexes = 1; +SET use_query_condition_cache = 0; + +DROP TABLE IF EXISTS tab_pre_post; +DROP TABLE IF EXISTS tab_pre; + +CREATE TABLE tab_pre_post +( + id UInt32, + s Nullable(String), + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(s), postprocessor = lower(s)) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2; + +CREATE TABLE tab_pre +( + id UInt32, + s Nullable(String), + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(ifNull(s, ''))) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2; + +-- Four granules of two rows. The last granule holds NULL. +INSERT INTO tab_pre_post SELECT number, if(number < 6, concat('Text', toString(number), ' tail'), NULL) FROM numbers(8); +INSERT INTO tab_pre SELECT number, if(number < 6, concat('Text', toString(number), ' tail'), NULL) FROM numbers(8); + +SELECT '-- preprocessor and postprocessor'; +SELECT id FROM tab_pre_post WHERE NOT hasToken(CAST(s, 'String'), 'text5') ORDER BY id; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id FROM tab_pre_post WHERE NOT hasAnyTokens(CAST(s, 'String'), 'text5') ORDER BY id; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id FROM tab_pre_post WHERE NOT hasToken(s::String, 'text5') ORDER BY id SETTINGS query_plan_direct_read_from_text_index = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id FROM tab_pre_post WHERE NOT hasToken(CAST(s, 'String'), 'text5') ORDER BY id SETTINGS use_skip_indexes = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +-- The granule with NULL is skipped. +SELECT id FROM tab_pre_post WHERE hasToken(CAST(s, 'String'), 'TEXT5') ORDER BY id; +SELECT id FROM tab_pre_post WHERE hasAnyTokens(CAST(s, 'String'), 'TEXT1 TEXT5') ORDER BY id; + +SELECT '-- preprocessor that maps NULL to a string'; +SELECT id FROM tab_pre WHERE NOT hasToken(CAST(s, 'String'), 'text5') ORDER BY id; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id FROM tab_pre WHERE NOT hasToken(CAST(s, 'String'), 'text5') ORDER BY id SETTINGS query_plan_direct_read_from_text_index = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id FROM tab_pre WHERE hasToken(CAST(s, 'String'), 'TEXT5') ORDER BY id; + +DROP TABLE tab_pre_post; +DROP TABLE tab_pre; diff --git a/tests/queries/0_stateless/05341_text_index_map_json_default_value_throws.reference b/tests/queries/0_stateless/05341_text_index_map_json_default_value_throws.reference new file mode 100644 index 0000000000000..c87cf3366bf2d --- /dev/null +++ b/tests/queries/0_stateless/05341_text_index_map_json_default_value_throws.reference @@ -0,0 +1,10 @@ +-- map +2 +2 +2 +2 +-- JSON +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/05341_text_index_map_json_default_value_throws.sql b/tests/queries/0_stateless/05341_text_index_map_json_default_value_throws.sql new file mode 100644 index 0000000000000..51323ea68fb4c --- /dev/null +++ b/tests/queries/0_stateless/05341_text_index_map_json_default_value_throws.sql @@ -0,0 +1,44 @@ +-- A text index on `mapKeys` or `JSONAllPaths` evaluates the predicate on the value of a missing key or path. +-- If the predicate throws on it, the index cannot answer the predicate, and the query must not fail. + +SET enable_analyzer = 1; +SET use_query_condition_cache = 0; + +DROP TABLE IF EXISTS tab_map; +DROP TABLE IF EXISTS tab_json; + +CREATE TABLE tab_map +( + id UInt32, + m Map(String, String), + INDEX idx mapKeys(m) TYPE text(tokenizer = 'array') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id; + +CREATE TABLE tab_json +( + id UInt32, + d JSON(TypeName Nullable(String)), + INDEX idx JSONAllPaths(d) TYPE text(tokenizer = 'array') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO tab_map VALUES (1, {'status': '200'}), (2, {'status': '404'}); +INSERT INTO tab_json VALUES (1, '{"TypeName": "User"}'), (2, '{"TypeName": "Group"}'); + +SELECT '-- map'; +SELECT id FROM tab_map WHERE toUInt64(m['status']) = 404 ORDER BY id; +SELECT id FROM tab_map WHERE toUInt64(m['status']) = 404 ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT id FROM tab_map WHERE hasToken(toString(toUInt64(m['status'])), '404') ORDER BY id; +SELECT id FROM tab_map WHERE hasToken(toString(toUInt64(m['status'])), '404') ORDER BY id SETTINGS use_skip_indexes = 0; + +SELECT '-- JSON'; +SELECT id FROM tab_json WHERE d.TypeName::String = 'User' ORDER BY id; +SELECT id FROM tab_json WHERE d.TypeName::String = 'User' ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT id FROM tab_json WHERE hasToken(d.TypeName::String, 'User') ORDER BY id; +SELECT id FROM tab_json WHERE hasToken(d.TypeName::String, 'User') ORDER BY id SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_map; +DROP TABLE tab_json; From c211f18ec563e443d7f002f69e9b1b8ffb011daa Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 10 Oct 2026 11:13:13 +0000 Subject: [PATCH 246/296] Backport #124964 to 26.8: Fix stack overflow when skipping a deeply nested Avro field --- .../Formats/Impl/AvroRowInputFormat.cpp | 1 + ...45_avro_skip_deeply_nested_field.reference | 4 ++ .../05345_avro_skip_deeply_nested_field.sh | 64 +++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 tests/queries/0_stateless/05345_avro_skip_deeply_nested_field.reference create mode 100755 tests/queries/0_stateless/05345_avro_skip_deeply_nested_field.sh diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 61fdb4751525c..366af904b8d5f 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -1041,6 +1041,7 @@ AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(const avro::NodePtr & ro } return [&skip_fn = it->second](avro::Decoder & decoder) { + checkStackSize(); skip_fn(decoder); }; } diff --git a/tests/queries/0_stateless/05345_avro_skip_deeply_nested_field.reference b/tests/queries/0_stateless/05345_avro_skip_deeply_nested_field.reference new file mode 100644 index 0000000000000..dcefe6e28f3a8 --- /dev/null +++ b/tests/queries/0_stateless/05345_avro_skip_deeply_nested_field.reference @@ -0,0 +1,4 @@ +42 +42 +chain_50000: too deep +list_100000: too deep diff --git a/tests/queries/0_stateless/05345_avro_skip_deeply_nested_field.sh b/tests/queries/0_stateless/05345_avro_skip_deeply_nested_field.sh new file mode 100755 index 0000000000000..7ce7c50a070a9 --- /dev/null +++ b/tests/queries/0_stateless/05345_avro_skip_deeply_nested_field.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the Avro format is not available in the fast test build. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DIR="$CLICKHOUSE_TMP/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "$DIR" +mkdir -p "$DIR" + +# Both files hold `id` and a field that is not read, so it is skipped. `chain`: a chain of named records +# A_k { x: A_(k-1) }, each defined inside its own array field. `list`: a recursive record L { next: [null, L] }. +function gen() +{ + python3 -c " +import json, sys +dst, mode, n = sys.argv[1], sys.argv[2], int(sys.argv[3]) +def zz(v): + v = (v << 1) ^ (v >> 63) + v &= (1 << 64) - 1 + out = bytearray() + while True: + b = v & 0x7f + v >>= 7 + out.append(b | 0x80 if v else b) + if not v: + break + return bytes(out) +def ab(b): + return zz(len(b)) + b +if mode == 'chain': + fields = [{'name': 'id', 'type': 'long'}, + {'name': 'f1', 'type': {'type': 'array', 'items': {'type': 'record', 'name': 'A1', 'fields': [{'name': 'v', 'type': 'long'}]}}}] + for k in range(2, n + 1): + fields.append({'name': 'f%d' % k, 'type': {'type': 'array', 'items': {'type': 'record', 'name': 'A%d' % k, 'fields': [{'name': 'x', 'type': 'A%d' % (k - 1)}]}}}) + fields.append({'name': 'last', 'type': 'A%d' % n}) + payload = zz(42) + b'\x00' * n + zz(7) +else: + fields = [{'name': 'id', 'type': 'long'}, + {'name': 'list', 'type': {'type': 'record', 'name': 'L', 'fields': [{'name': 'next', 'type': ['null', 'L']}]}}] + payload = zz(42) + zz(1) * n + zz(0) +schema = json.dumps({'type': 'record', 'name': 'root', 'fields': fields}, separators=(',', ':')).encode() +meta = zz(2) + ab(b'avro.schema') + ab(schema) + ab(b'avro.codec') + ab(b'null') + zz(0) +sync = bytes(16) +open(dst, 'wb').write(b'Obj\x01' + meta + sync + zz(1) + zz(len(payload)) + payload + sync) +" "$@" +} + +gen "$DIR/chain_40.avro" chain 40 +gen "$DIR/chain_50000.avro" chain 50000 +gen "$DIR/list_10.avro" list 10 +gen "$DIR/list_100000.avro" list 100000 + +for f in chain_40 list_10; do + $CLICKHOUSE_LOCAL -q "SELECT id FROM file('$DIR/$f.avro', Avro, 'id Int64')" +done + +for f in chain_50000 list_100000; do + (ulimit -s 1024; $CLICKHOUSE_LOCAL -q "SELECT id FROM file('$DIR/$f.avro', Avro, 'id Int64')" 2>&1) | grep -q -F 'TOO_DEEP_RECURSION' && echo "$f: too deep" +done + +rm -rf "$DIR" From 0f00611fbd21783e33cfac57865ae1e72f238b36 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 10 Oct 2026 11:21:45 +0000 Subject: [PATCH 247/296] Backport #124724 to 26.8: Do not use parallel replicas for `clusterAllReplicas` --- src/Interpreters/Cluster.cpp | 2 ++ src/Interpreters/Cluster.h | 4 +++ .../ClusterProxy/executeQuery.cpp | 5 ++++ ...opagated_to_cluster_all_replicas.reference | 2 ++ ...not_propagated_to_cluster_all_replicas.sql | 30 +++++++++++++++++++ 5 files changed, 43 insertions(+) create mode 100644 tests/queries/0_stateless/05339_pr_not_propagated_to_cluster_all_replicas.reference create mode 100644 tests/queries/0_stateless/05339_pr_not_propagated_to_cluster_all_replicas.sql diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index f24818dbca0ce..31dfa0636417a 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -873,6 +873,7 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti secret = from.secret; name = from.name; + replicas_as_shards = true; initMisc(); } @@ -894,6 +895,7 @@ Cluster::Cluster(Cluster::SubclusterTag, const Cluster & from, const std::vector secret = from.secret; name = from.name; + replicas_as_shards = from.replicas_as_shards; initMisc(); } diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index 2b74c2b9e6c81..a875ccb933861 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -296,6 +296,9 @@ class Cluster const String & getName() const { return name; } + /// Every replica of the original cluster became a shard of its own, as `clusterAllReplicas` does. + bool replicasAsShards() const { return replicas_as_shards; } + private: SlotToShard slot_to_shard; @@ -342,6 +345,7 @@ class Cluster size_t local_shard_count = 0; String name; + bool replicas_as_shards = false; }; using ClusterPtr = std::shared_ptr; diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 7fa2d6fadecb9..6c74b448aacc6 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -458,6 +458,11 @@ static ContextMutablePtr updateSettingsAndClientInfoForCluster(const Cluster & c new_settings[Setting::allow_experimental_parallel_reading_from_replicas] = 0; } + /// `clusterAllReplicas` already reads every replica as a shard of its own, so there is nothing left + /// for parallel replicas to split + if (context->canUseTaskBasedParallelReplicas() && cluster.replicasAsShards()) + new_settings[Setting::allow_experimental_parallel_reading_from_replicas] = 0; + if (settings[Setting::max_execution_time_leaf].totalMicroseconds() > 0) { /// Replace 'max_execution_time' of this sub-query with 'max_execution_time_leaf' and 'timeout_overflow_mode' diff --git a/tests/queries/0_stateless/05339_pr_not_propagated_to_cluster_all_replicas.reference b/tests/queries/0_stateless/05339_pr_not_propagated_to_cluster_all_replicas.reference new file mode 100644 index 0000000000000..565b700fb61e8 --- /dev/null +++ b/tests/queries/0_stateless/05339_pr_not_propagated_to_cluster_all_replicas.reference @@ -0,0 +1,2 @@ +30 +3 0 diff --git a/tests/queries/0_stateless/05339_pr_not_propagated_to_cluster_all_replicas.sql b/tests/queries/0_stateless/05339_pr_not_propagated_to_cluster_all_replicas.sql new file mode 100644 index 0000000000000..2e531fdc2a291 --- /dev/null +++ b/tests/queries/0_stateless/05339_pr_not_propagated_to_cluster_all_replicas.sql @@ -0,0 +1,30 @@ +-- `clusterAllReplicas` reads every replica as a shard of its own, so there is nothing left for parallel +-- replicas to split, and the shard number it ships indexes that numbering rather than the cluster of the +-- same name. Parallel replicas must not be propagated into its sub-queries. + +DROP TABLE IF EXISTS t_cluster_all_replicas; +CREATE TABLE t_cluster_all_replicas (n UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_cluster_all_replicas SELECT * FROM numbers(10); + +SET automatic_parallel_replicas_mode = 0; +SET enable_parallel_replicas = 1, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree = 1; +SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; +-- Every replica gets a sub-query of its own. +SET prefer_localhost_replica = 0; + +SELECT count() FROM clusterAllReplicas(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), t_cluster_all_replicas) +SETTINGS log_comment = '05339_cluster_all_replicas'; + +SYSTEM FLUSH LOGS query_log; + +-- The setting is recorded only when it differs from the default, so a sub-query that ran without parallel +-- replicas has either no entry for it or `0`, when a profile enables them. +SELECT count(), countIf(Settings['allow_experimental_parallel_reading_from_replicas'] NOT IN ('', '0')) +FROM system.query_log +WHERE type = 'QueryFinish' AND NOT is_initial_query AND event_date >= yesterday() + AND initial_query_id IN ( + SELECT query_id FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND is_initial_query + AND event_date >= yesterday() AND log_comment = '05339_cluster_all_replicas'); + +DROP TABLE t_cluster_all_replicas; From bf9cf54524a35a3b8f9d1d86862f96c64c9999fa Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 10 Oct 2026 11:23:16 +0000 Subject: [PATCH 248/296] Backport #123683 to 26.8: Fix duplicated rows of `ANY RIGHT JOIN` with `any_join_distinct_right_table_keys` under parallel replicas --- src/Planner/PlannerJoinTree.cpp | 5 +- .../Optimizations/applyParallelReplicas.cpp | 5 ++ ...el_replicas_right_join_right_any.reference | 6 ++ ...parallel_replicas_right_join_right_any.sql | 84 +++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05320_parallel_replicas_right_join_right_any.reference create mode 100644 tests/queries/0_stateless/05320_parallel_replicas_right_join_right_any.sql diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index ab57c16ce1038..c5d95b83f25fa 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1567,7 +1567,10 @@ bool allowParallelReplicasForJoinTree(const QueryTreeNodePtr & join_tree_node, c && left_table_expr->getNodeType() != QueryTreeNodeType::CROSS_JOIN; } - if (join_kind == JoinKind::Right) + /// RightAny picks one right row per left row out of the whole right table, so its right side cannot be split. + /// In plan-based mode this gate only sizes the read, and `liftSplitAboveJoin` keeps such a join local. + if (join_kind == JoinKind::Right + && (join_strictness != JoinStrictness::RightAny || query_settings[Setting::parallel_replicas_plan_based])) { // parallel replicas is allowed only simple RIGHT JOINs i.e. t1 RIGHT JOIN t2 if (left_table_expr->getNodeType() != QueryTreeNodeType::TABLE diff --git a/src/Processors/QueryPlan/Optimizations/applyParallelReplicas.cpp b/src/Processors/QueryPlan/Optimizations/applyParallelReplicas.cpp index c934e92784923..2fd4b2eeb4011 100644 --- a/src/Processors/QueryPlan/Optimizations/applyParallelReplicas.cpp +++ b/src/Processors/QueryPlan/Optimizations/applyParallelReplicas.cpp @@ -199,6 +199,11 @@ class ApplyParallelReplicasVisitor : public QueryPlanVisitor(node->step.get())->getJoinOperator().strictness == JoinStrictness::RightAny) + return; + auto * coordinated_child = node->children[static_cast(coordinated_side)]; if (!typeid_cast(coordinated_child->step.get())) return; diff --git a/tests/queries/0_stateless/05320_parallel_replicas_right_join_right_any.reference b/tests/queries/0_stateless/05320_parallel_replicas_right_join_right_any.reference new file mode 100644 index 0000000000000..e5fab230e1b3a --- /dev/null +++ b/tests/queries/0_stateless/05320_parallel_replicas_right_join_right_any.reference @@ -0,0 +1,6 @@ +10 +0 +1 +Aggregating Join ReadFromMergeTree Union ReadFromMergeTree ReadFromParallelReplicas +Aggregating Join ReadFromMergeTree ReadFromMergeTree +Union Aggregating Join ReadFromMergeTree ReadFromMergeTree ReadFromParallelReplicas diff --git a/tests/queries/0_stateless/05320_parallel_replicas_right_join_right_any.sql b/tests/queries/0_stateless/05320_parallel_replicas_right_join_right_any.sql new file mode 100644 index 0000000000000..bf63535b2f71f --- /dev/null +++ b/tests/queries/0_stateless/05320_parallel_replicas_right_join_right_any.sql @@ -0,0 +1,84 @@ +-- With parallel replicas, a two-way ANY RIGHT JOIN under any_join_distinct_right_table_keys = 1 must return +-- the same rows as without them. Evaluated on each replica, every replica picked a right row for each left row +-- from its own share of the right table, so a left row was emitted once per replica. + +SET automatic_parallel_replicas_mode = 0; +SET any_join_distinct_right_table_keys = 1; +SET explain_query_plan_default = 'legacy'; +-- A randomized join order or a table swap would change the plan shape asserted below. +SET query_plan_optimize_join_order_randomize = 0; +SET query_plan_join_swap_table = 'false'; + +DROP TABLE IF EXISTS t_l; +DROP TABLE IF EXISTS t_r; + +CREATE TABLE t_l (k UInt64) ENGINE = MergeTree ORDER BY k; +CREATE TABLE t_r (k UInt64, v UInt64) ENGINE = MergeTree ORDER BY v SETTINGS index_granularity = 64; + +INSERT INTO t_l SELECT number FROM numbers(10); +INSERT INTO t_r SELECT number % 10, number FROM numbers(100000); + +-- Each left row takes one right row of its key, and every key is matched. +SELECT count() FROM t_l ANY RIGHT JOIN t_r ON t_l.k = t_r.k SETTINGS enable_parallel_replicas = 0; + +-- The join is not shipped to the replicas. +SELECT countIf(explain ILIKE '%ReadFromRemoteParallelReplicas%') > 0 +FROM ( + EXPLAIN SELECT count() FROM t_l ANY RIGHT JOIN t_r ON t_l.k = t_r.k + SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 0 +); + +-- Positive control: ANY RIGHT JOIN without the setting still reads with parallel replicas. +SELECT countIf(explain ILIKE '%ReadFromRemoteParallelReplicas%') > 0 +FROM ( + EXPLAIN SELECT count() FROM t_l ANY RIGHT JOIN t_r ON t_l.k = t_r.k + SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 0, any_join_distinct_right_table_keys = 0 +); + +-- Plan-based parallel replicas: the join stays above the distributed read of t_r. The local plan is pinned +-- because it adds the initiator's own read to the plan shape. +SELECT arrayStringConcat(groupArray(step), ' ') +FROM ( + SELECT trimLeft(explain) AS step + FROM ( + EXPLAIN actions = 0, pretty = 0, optimize = 1, description = 0, header = 0 + SELECT count() FROM t_l ANY RIGHT JOIN t_r ON t_l.k = t_r.k + SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 0, parallel_replicas_plan_based = 1, + parallel_replicas_local_plan = 1 + ) + WHERE step IN ('Aggregating', 'Union', 'Join', 'ReadFromMergeTree', 'ReadFromParallelReplicas') +); + +-- Plan-based parallel replicas still apply the row threshold to it. +SELECT arrayStringConcat(groupArray(step), ' ') +FROM ( + SELECT trimLeft(explain) AS step + FROM ( + EXPLAIN actions = 0, pretty = 0, optimize = 1, description = 0, header = 0 + SELECT count() FROM t_l ANY RIGHT JOIN t_r ON t_l.k = t_r.k + SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 1000000, parallel_replicas_plan_based = 1, + parallel_replicas_local_plan = 1 + ) + WHERE step IN ('Aggregating', 'Union', 'Join', 'ReadFromMergeTree', 'ReadFromParallelReplicas') +); + +-- Positive control: without the setting the whole join still ships. +SELECT arrayStringConcat(groupArray(step), ' ') +FROM ( + SELECT trimLeft(explain) AS step + FROM ( + EXPLAIN actions = 0, pretty = 0, optimize = 1, description = 0, header = 0 + SELECT count() FROM t_l ANY RIGHT JOIN t_r ON t_l.k = t_r.k + SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 0, parallel_replicas_plan_based = 1, + parallel_replicas_local_plan = 1, any_join_distinct_right_table_keys = 0 + ) + WHERE step IN ('Aggregating', 'Union', 'Join', 'ReadFromMergeTree', 'ReadFromParallelReplicas') +); + +DROP TABLE t_l; +DROP TABLE t_r; From 6e169f6fd02568e360efb2fb88e8029406ff1447 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 10 Oct 2026 16:19:46 +0000 Subject: [PATCH 249/296] Update autogenerated version to 26.8.23.6 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 51af7ba58e03f..5befdb9f4c631 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54535) +SET(VERSION_REVISION 54536) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 23) -SET(VERSION_GITHASH cc72af4d4e916a278d8d750b2b0cab13aaac53a3) -SET(VERSION_DESCRIBE v26.8.23.1-lts) -SET(VERSION_STRING 26.8.23.1) +SET(VERSION_PATCH 24) +SET(VERSION_GITHASH 4ba605fc98d5614aa74b4f7ab8a00850b59ef8ee) +SET(VERSION_DESCRIBE v26.8.24.1-lts) +SET(VERSION_STRING 26.8.24.1) # end of autochange From e60aca52a3a7a44112eade9698913dc08428ecc2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 10 Oct 2026 17:20:01 +0000 Subject: [PATCH 250/296] Backport #119105 to 26.8: Fix `TRUNCATE` of an `Alias` table over a `MergeTree` blocking on concurrent readers --- src/Interpreters/InterpreterDropQuery.cpp | 4 +- src/Storages/StorageAlias.cpp | 17 ++- .../05153_alias_truncate_lock.reference | 25 ++++ .../0_stateless/05153_alias_truncate_lock.sh | 114 ++++++++++++++++++ 4 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05153_alias_truncate_lock.reference create mode 100755 tests/queries/0_stateless/05153_alias_truncate_lock.sh diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 9e16ed6cc7224..a29ec43938dbb 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -331,7 +331,9 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, AS TableExclusiveLockHolder table_excl_lock; /// We don't need any lock for ReplicatedMergeTree and for simple MergeTree /// For the rest of tables types exclusive lock is needed - if (!castStorage(table, DeferredTable::Load)) + /// An `Alias` runs the truncate on its target, so the exemption follows the target: + /// `isMergeTree()` resolves it, and is false while it is missing or not loaded yet. + if (!castStorage(table, DeferredTable::Load) && !table->isMergeTree()) table_excl_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef()[Setting::lock_acquire_timeout]); auto metadata_snapshot = table->getInMemoryMetadataPtr(context_, false); diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index 9fe5812ded41d..eff18fef021cf 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -283,11 +283,24 @@ void StorageAlias::truncate( const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr local_context, - TableExclusiveLockHolder & table_lock_holder) + TableExclusiveLockHolder & /*table_lock_holder*/) { auto target_storage = getTargetTable(TargetAccess{local_context, AccessType::TRUNCATE}); + + /// The target is what executes the truncate, so it carries the lock its own engine needs, and the + /// caller's holder is the alias's, not the target's. A MergeTree removes its data under its own + /// locks; every other engine needs its readers excluded while its data goes away. + TableExclusiveLockHolder target_excl_lock; + TableLockHolder target_shared_lock; + if (target_storage->isMergeTree()) + target_shared_lock = target_storage->lockForShare( + local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); + else + target_excl_lock = target_storage->lockExclusively( + local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); + auto target_metadata = target_storage->getInMemoryMetadataPtr(local_context, false); - target_storage->truncate(query, target_metadata, local_context, table_lock_holder); + target_storage->truncate(query, target_metadata, local_context, target_excl_lock); } bool StorageAlias::optimize( diff --git a/tests/queries/0_stateless/05153_alias_truncate_lock.reference b/tests/queries/0_stateless/05153_alias_truncate_lock.reference new file mode 100644 index 0000000000000..bb31165c34ab2 --- /dev/null +++ b/tests/queries/0_stateless/05153_alias_truncate_lock.reference @@ -0,0 +1,25 @@ +-- fixture +al_a Alias +al_b Alias +al_c Alias +al_d_inner Alias +al_d_outer Alias +al_f Alias +al_g Alias +al_h Alias +mem_c Memory +mem_g Memory +mt_a MergeTree +mt_d MergeTree +mt_e MergeTree +mt_h MergeTree +rmt_b ReplicatedMergeTree +-- arms +A_alias_to_mergetree blocked=0 rows_after=0 +B_alias_to_replicated_mergetree blocked=0 rows_after=0 +C_alias_to_memory blocked=1 rows_after=300 +D_alias_chain_to_mergetree blocked=0 rows_after=0 +E_direct_mergetree blocked=0 rows_after=0 +G_direct_reader_of_memory_target blocked=1 rows_after=300 +H_direct_reader_of_mergetree_target blocked=0 rows_after=0 +F_dangling_alias err=UNKNOWN_TABLE diff --git a/tests/queries/0_stateless/05153_alias_truncate_lock.sh b/tests/queries/0_stateless/05153_alias_truncate_lock.sh new file mode 100755 index 0000000000000..c0a21d0f4db60 --- /dev/null +++ b/tests/queries/0_stateless/05153_alias_truncate_lock.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-replicated-database, no-shared-merge-tree, memory-engine +# no-replicated-database: there TRUNCATE is enqueued as a DDL entry instead of taking the lock these +# arms measure, so every arm would report blocked=0 regardless of the fix. +# no-shared-merge-tree, memory-engine: the fixture assertion pins each engine name, and the +# ReplicatedMergeTree -> SharedMergeTree and Memory -> MergeTree test-runner +# replacements rewrite two of them. +# +# TRUNCATE through an Alias must choose its lock on, and take it on, the storage that executes the +# truncate. Each arm starts a reader, then issues TRUNCATE with a 3s lock_acquire_timeout and reports +# whether that reader blocked it. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE mt_a (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE rmt_b (id UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt_b', 'r1') ORDER BY id; + CREATE TABLE mem_c (id UInt64) ENGINE = Memory; + CREATE TABLE mt_d (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE mt_e (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE mem_g (id UInt64) ENGINE = Memory; + CREATE TABLE mt_h (id UInt64) ENGINE = MergeTree ORDER BY id; + + INSERT INTO mt_a SELECT number FROM numbers(300); + INSERT INTO rmt_b SELECT number FROM numbers(300); + INSERT INTO mem_c SELECT number FROM numbers(300); + INSERT INTO mt_d SELECT number FROM numbers(300); + INSERT INTO mt_e SELECT number FROM numbers(300); + INSERT INTO mem_g SELECT number FROM numbers(300); + INSERT INTO mt_h SELECT number FROM numbers(300); +" + +# al_d_outer is created while al_d_inner is still absent: an Alias whose target already resolves to +# an Alias is rejected at creation, an Alias to a missing name is not. +$CLICKHOUSE_CLIENT -q " + CREATE TABLE al_a ENGINE = Alias(currentDatabase(), 'mt_a'); + CREATE TABLE al_b ENGINE = Alias(currentDatabase(), 'rmt_b'); + CREATE TABLE al_c ENGINE = Alias(currentDatabase(), 'mem_c'); + CREATE TABLE al_d_outer ENGINE = Alias(currentDatabase(), 'al_d_inner'); + CREATE TABLE al_d_inner ENGINE = Alias(currentDatabase(), 'mt_d'); + CREATE TABLE al_f ENGINE = Alias(currentDatabase(), 'no_such_table'); + CREATE TABLE al_g ENGINE = Alias(currentDatabase(), 'mem_g'); + CREATE TABLE al_h ENGINE = Alias(currentDatabase(), 'mt_h'); +" + +# Without this an arm could report the expected value while probing a plain table instead of an alias. +echo '-- fixture' +$CLICKHOUSE_CLIENT -q "SELECT name, engine FROM system.tables WHERE database = currentDatabase() ORDER BY name" + +# arm
+ cat < saw that probe prune. +# Reading a counter back through `log_comment` keeps a probe from counting itself. +report() { # report ... + local counter=$1 tags="" t + shift + for t in "$@"; do tags="${tags:+${tags}, }'${CLICKHOUSE_DATABASE}_${t}'"; done + ${CLICKHOUSE_CLIENT} --query " + SELECT max(ProfileEvents['${counter}']) + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment IN (${tags}) + GROUP BY log_comment + ORDER BY indexOf([${tags}], log_comment) + SETTINGS enable_parallel_replicas = 0" +} + +echo '--- A0 a filter on the column returns every row that matches it ---' +# A bound that comes back inverted prunes the data file that holds the row, so these lose rows. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM high; + SELECT count() FROM high WHERE d = toDecimal128('0.99', 38); + SELECT count() FROM high WHERE d = toDecimal128('-0.99', 38); + SELECT count() FROM high WHERE d > toDecimal128('0.5', 38); + SELECT count() FROM high WHERE d < toDecimal128('-0.5', 38);" + +echo '--- A1 a probe outside one file min/max-prunes it, and only it ---' +# `low` is the control: its shift stays inside `Int128` whatever the reader does with an overflow. +# Each `high` probe sits outside exactly one of the two shifted bounds, so one file is pruned and +# the other is read - the row counts say the surviving file is the one that could hold a match. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM low WHERE d = toDecimal128('0.95', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_low_pos'; + SELECT count() FROM low WHERE d = toDecimal128('-0.95', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_low_neg'; + SELECT count() FROM high WHERE d = toDecimal128('0.5', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_high_pos'; + SELECT count() FROM high WHERE d = toDecimal128('-0.5', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_high_neg'; + SYSTEM FLUSH LOGS query_log;" +report IcebergMinMaxIndexPrunedFiles low_pos low_neg high_pos high_neg + +echo '--- A2 a value inside both shifted bounds prunes nothing ---' +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM high WHERE d = toDecimal128('0.0', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_high_zero'; + SYSTEM FLUSH LOGS query_log;" +report IcebergMinMaxIndexPrunedFiles high_zero + +echo '--- B0 a bound the writer rounded past the precision of the type still prunes ---' +# Spark and Amazon Athena round the bound to the integral part, so a file whose extreme value is +# `0.99` carries `1.0` - `10^38` unscaled, one unit outside what `Decimal(38, 38)` holds. ClickHouse +# writes the exact unscaled value instead, so the rounded form is patched into the manifests here. +# The bound is an uncompressed 16-byte big-endian `fixed`, and both forms need all 16 bytes, so the +# substitution keeps every length in the Avro container intact. +python3 - "${ROOT}/high/metadata" <<'PATCH_MANIFESTS' +import pathlib +import sys + +def big_endian(value): + return (value % (1 << 128)).to_bytes(16, 'big') + +exact, rounded = 99 * 10 ** 36, 10 ** 38 +substitutions = [(big_endian(exact), big_endian(rounded)), (big_endian(-exact), big_endian(-rounded))] + +patched = 0 +for path in pathlib.Path(sys.argv[1]).rglob('*.avro'): + manifest = path.read_bytes() + replaced = manifest + for old, new in substitutions: + replaced = replaced.replace(old, new) + if replaced != manifest: + path.write_bytes(replaced) + patched += 1 + +if patched != 2: + raise SystemExit(f'patched {patched} manifests, expected one per data file') +PATCH_MANIFESTS + +# A table function reads the patched manifests afresh; the `high` table above still holds the ones +# it wrote. Every bound is now `±1.0`, so one data file declares `[0, 1)` and the other `(-1, 0]`. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM icebergLocal('${ROOT}/high/'); + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('0.99', 38); + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('-0.99', 38);" + +echo '--- B1 the saturated bounds prune the file that cannot hold the probe ---' +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('0.5', 38) + SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_rounded_pos'; + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('-0.5', 38) + SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_rounded_neg'; + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('0.0', 38) + SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_rounded_zero'; + SYSTEM FLUSH LOGS query_log;" +report IcebergMinMaxIndexPrunedFiles rounded_pos rounded_neg rounded_zero + +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS high SYNC; + DROP TABLE IF EXISTS low SYNC;" From c73c6dbf822644a7718a26784b5fa4ba8be265ae Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 11 Oct 2026 21:33:24 +0000 Subject: [PATCH 260/296] Drop the `FINAL` data query from the 26.8 backport of `05205_vector_search_lightweight_delete_limit` Reading `SELECT ... FINAL PREWHERE _row_exists` with `apply_prewhere_after_final = 1` fails with `THERE_IS_NO_COLUMN` in 26.8 regardless of the vector index (reproduced without any index). The bug is fixed on master by https://github.com/ClickHouse/ClickHouse/pull/114578, which is not in 26.8. Keep the `EXPLAIN` check that the index is not used in this case, which is what this backport changes. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=125108&sha=76bfddc8e3fe65c82d6272e7107fb6cb46e658ca&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20distributed%20plan%2C%20parallel%29 PR: https://github.com/ClickHouse/ClickHouse/pull/125108 Co-Authored-By: Claude Opus 5.5 --- .../05205_vector_search_lightweight_delete_limit.reference | 1 - .../05205_vector_search_lightweight_delete_limit.sql | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.reference b/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.reference index 32a2228b7a7f4..ea9603c180be9 100644 --- a/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.reference +++ b/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.reference @@ -11,7 +11,6 @@ apply_deleted_mask = 0 and _row_exists uses the index 0 apply_deleted_mask = 0 and _row_exists 3 apply_deleted_mask = 0 and _row_exists [1,1,1] FINAL, apply_deleted_mask = 0 and deferred PREWHERE _row_exists uses the index 0 -FINAL, apply_deleted_mask = 0 and deferred PREWHERE _row_exists 3 a deleted cluster 3 a pending delete 3 a delete in a patch part 3 diff --git a/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.sql b/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.sql index 5c9dbd8d07775..86d90930fb694 100644 --- a/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.sql +++ b/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.sql @@ -49,7 +49,8 @@ SELECT 'apply_deleted_mask = 0 and _row_exists', count() FROM (SELECT id FROM t_ SELECT 'apply_deleted_mask = 0 and _row_exists', arraySort(groupArray(d)) FROM (SELECT L2Distance(v, [5.0, 5.0]) AS d FROM t_05205 WHERE _row_exists ORDER BY d LIMIT 3 SETTINGS apply_deleted_mask = 0); -- The same with `FINAL`, where the `PREWHERE` on `_row_exists` is deferred until after the `FINAL` merge. SELECT 'FINAL, apply_deleted_mask = 0 and deferred PREWHERE _row_exists uses the index', count() FROM (EXPLAIN indexes = 1 SELECT id FROM t_05205 FINAL PREWHERE _row_exists ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS apply_deleted_mask = 0, apply_prewhere_after_final = 1, use_skip_indexes_on_data_read = 0) WHERE explain LIKE '% Granules: 3/300'; -SELECT 'FINAL, apply_deleted_mask = 0 and deferred PREWHERE _row_exists', count() FROM (SELECT id FROM t_05205 FINAL PREWHERE _row_exists ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS apply_deleted_mask = 0, apply_prewhere_after_final = 1); +-- Only the plan is checked here: reading with a bare `PREWHERE _row_exists` deferred after `FINAL` fails with +-- `THERE_IS_NO_COLUMN` in 26.8 regardless of the index, fixed on master by https://github.com/ClickHouse/ClickHouse/pull/114578. -- Deleting a whole cluster of near neighbours used to empty the result as well. DELETE FROM t_05205 WHERE L2Distance(v, [5.0, 5.0]) < 1.5; From 34bd0d0d2bc791e92681b1117ca855ebfbef1d10 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 11 Oct 2026 21:42:36 +0000 Subject: [PATCH 261/296] Backport #124233 to 26.8: Do not derive conjuncts in outer JOIN ON --- .../Passes/LogicalExpressionOptimizerPass.cpp | 28 +++++++++++++++- ..._and_compare_chain_outer_join_on.reference | 28 ++++++++++++++++ .../05331_and_compare_chain_outer_join_on.sql | 33 +++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05331_and_compare_chain_outer_join_on.reference create mode 100644 tests/queries/0_stateless/05331_and_compare_chain_outer_join_on.sql diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 520da50c21243..4610a6447318a 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -1831,6 +1831,31 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont return result; } + /// a conjunct over these can't leave ON, and full_sorting_merge rejects it there + TableExpressionNodes preserved_join_tables; + + bool needChildVisit(QueryTreeNodePtr & parent, QueryTreeNodePtr & child) + { + const auto * join_node = parent->as(); + if (!join_node || child != join_node->getJoinExpression()) + return true; + + TableExpressionNodes tables; + /// semi join keeps only matched rows + if (join_node->getStrictness() != JoinStrictness::Semi) + { + if (isLeftOrFull(join_node->getKind())) + tables = extractTableExpressions(join_node->getLeftTableExpressionNodeTyped(), /*add_array_join=*/ true); + if (isRightOrFull(join_node->getKind())) + tables.append_range(extractTableExpressions(join_node->getRightTableExpressionNodeTyped(), /*add_array_join=*/ true)); + } + + std::swap(preserved_join_tables, tables); + visit(child); + std::swap(preserved_join_tables, tables); + return false; + } + void enterImpl(QueryTreeNodePtr & node) { if (auto * join_node = node->as()) @@ -2497,7 +2522,8 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont existing_plain_conjuncts.emplace(and_node); function_node.getArguments().getNodes().push_back(and_node); } - else if (add_result != AddComparisonFilterResult::ALWAYS_TRUE) + else if (add_result != AddComparisonFilterResult::ALWAYS_TRUE + && !std::ranges::contains(preserved_join_tables, getExpressionSource(left.node).first)) { if (left.crosses_source) { diff --git a/tests/queries/0_stateless/05331_and_compare_chain_outer_join_on.reference b/tests/queries/0_stateless/05331_and_compare_chain_outer_join_on.reference new file mode 100644 index 0000000000000..7dd4445434fd0 --- /dev/null +++ b/tests/queries/0_stateless/05331_and_compare_chain_outer_join_on.reference @@ -0,0 +1,28 @@ +left +0 0 +1 1 +2 0 +right +0 0 +1 1 +0 2 +nested and +0 0 +1 1 +2 0 +preserved side is a join +0 0 +1 1 +2 0 +preserved side has array join +0 0 +1 1 +2 0 +datetime64 +1 1 +2 0 +derived bounds +2 +2 +2 +1 diff --git a/tests/queries/0_stateless/05331_and_compare_chain_outer_join_on.sql b/tests/queries/0_stateless/05331_and_compare_chain_outer_join_on.sql new file mode 100644 index 0000000000000..f3ebf42be036f --- /dev/null +++ b/tests/queries/0_stateless/05331_and_compare_chain_outer_join_on.sql @@ -0,0 +1,33 @@ +-- the compare chain must not add a preserved-side conjunct to outer join ON + +SET optimize_and_compare_chain = 1; +SET join_algorithm = 'full_sorting_merge'; + +SELECT 'left'; +SELECT l.number, r.number FROM numbers(3) AS l LEFT JOIN numbers(3) AS r ON l.number = r.number AND r.number < 2 ORDER BY l.number; + +SELECT 'right'; +SELECT l.number, r.number FROM numbers(3) AS l RIGHT JOIN numbers(3) AS r ON l.number = r.number AND l.number < 2 ORDER BY r.number; + +SELECT 'nested and'; +SELECT l.number, r.number FROM numbers(3) AS l LEFT JOIN numbers(3) AS r ON (l.number = r.number AND r.number < 2) AND r.number != 5 ORDER BY l.number; + +SELECT 'preserved side is a join'; +SELECT a.number, c.number FROM numbers(3) AS a INNER JOIN numbers(3) AS b ON a.number = b.number LEFT JOIN numbers(3) AS c ON b.number = c.number AND c.number < 2 ORDER BY a.number; + +SELECT 'preserved side has array join'; +SELECT x, r.number FROM numbers(1) AS l ARRAY JOIN [0, 1, 2] AS x LEFT JOIN numbers(3) AS r ON x = r.number AND r.number < 2 ORDER BY x; + +SELECT 'datetime64'; +DROP TABLE IF EXISTS t_chain_outer; +CREATE TABLE t_chain_outer (id UInt64, ts DateTime64(6, 'UTC')) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_chain_outer VALUES (1, '2020-01-01 00:00:01'), (2, '2020-01-01 00:00:03'); +SELECT l.id, r.id FROM t_chain_outer AS l LEFT JOIN t_chain_outer AS r ON l.id = r.id AND l.ts = r.ts AND r.ts < toDateTime('2020-01-01 00:00:02', 'UTC') ORDER BY l.id; +DROP TABLE t_chain_outer; + +-- 2 if the chain derives a bound in ON, 1 if not +SELECT 'derived bounds'; +SELECT count() FROM (EXPLAIN QUERY TREE SELECT 1 FROM numbers(3) AS l INNER JOIN numbers(3) AS r ON l.number = r.number AND r.number < 2) WHERE explain LIKE '%function_name: less,%'; +SELECT count() FROM (EXPLAIN QUERY TREE SELECT 1 FROM numbers(3) AS l LEFT JOIN numbers(3) AS r ON l.number = r.number AND l.number < 2) WHERE explain LIKE '%function_name: less,%'; +SELECT count() FROM (EXPLAIN QUERY TREE SELECT 1 FROM numbers(3) AS l LEFT SEMI JOIN numbers(3) AS r ON l.number = r.number AND r.number < 2) WHERE explain LIKE '%function_name: less,%'; +SELECT count() FROM (EXPLAIN QUERY TREE SELECT 1 FROM numbers(3) AS l FULL JOIN numbers(3) AS r ON l.number = r.number AND l.number < 2) WHERE explain LIKE '%function_name: less,%'; From 01ff2fed64e0e01f468e8fb24c9372e6eab20067 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 11 Oct 2026 21:48:03 +0000 Subject: [PATCH 262/296] Backport #117015 to 26.8: Bound the collections read from the Native protocol --- .../interfaces/specs/NativeProtocol.mdx | 23 ++ src/Client/Connection.cpp | 7 +- src/Core/BaseSettings.cpp | 13 + src/Core/BaseSettings.h | 33 +- src/Core/BlockInfo.cpp | 24 +- src/Core/Settings.cpp | 7 +- src/Core/tests/gtest_block_info.cpp | 70 ++++ src/Core/tests/gtest_settings.cpp | 9 +- src/IO/ReadHelpers.h | 9 +- src/Interpreters/ClientInfo.cpp | 44 ++- src/Interpreters/TablesStatus.cpp | 26 +- .../tests/gtest_client_info_read.cpp | 53 +++ src/Server/TCPHandler.cpp | 22 +- ...5_native_protocol_collection_limits.python | 310 ++++++++++++++++++ ...ative_protocol_collection_limits.reference | 7 + ...05055_native_protocol_collection_limits.sh | 10 + ...ed_async_insert_long_client_info.reference | 1 + ...stributed_async_insert_long_client_info.sh | 28 ++ 18 files changed, 659 insertions(+), 37 deletions(-) create mode 100644 src/Core/tests/gtest_block_info.cpp create mode 100644 tests/queries/0_stateless/05055_native_protocol_collection_limits.python create mode 100644 tests/queries/0_stateless/05055_native_protocol_collection_limits.reference create mode 100755 tests/queries/0_stateless/05055_native_protocol_collection_limits.sh create mode 100644 tests/queries/0_stateless/05258_distributed_async_insert_long_client_info.reference create mode 100755 tests/queries/0_stateless/05258_distributed_async_insert_long_client_info.sh diff --git a/docs/reference/interfaces/specs/NativeProtocol.mdx b/docs/reference/interfaces/specs/NativeProtocol.mdx index 6d165d1b21e89..778b5a0f5c39c 100644 --- a/docs/reference/interfaces/specs/NativeProtocol.mdx +++ b/docs/reference/interfaces/specs/NativeProtocol.mdx @@ -84,6 +84,29 @@ SSH challenge-response authentication is available from protocol version 54466 o For distributed query execution, servers authenticate to one another by proving knowledge of a shared secret — without putting the secret on the wire. Each Query carries a 32-byte SHA-256 `auth_hash` in [`Query`](#query) field 4, computed over a salt, nonce, the configured secret, and the query, which the receiving server recomputes and compares. This is gated by the `INTERSERVER_SECRET` feature (v54441). External clients always send an empty string here. See [Inter-server authentication](#inter-server-authentication). +### Receiver-side size limits {#receiver-side-size-limits} + +Almost every string and collection on the wire is length-prefixed, and a receiver may allocate for the declared size before the data behind it arrives. A peer therefore has to bound what it accepts by what the data can plausibly hold — in inter-server mode the server reads `ClientInfo`, the settings and the externally granted roles of a query *before* it verifies the query's `auth_hash`, so these limits apply to unauthenticated input. + +The limits the ClickHouse server and `clickhouse-client` enforce, and which an implementation should expect to be rejected against, are: + +| Field | Limit | +|-------|-------| +| Each `String` of [`ClientHello`](#clienthello) — user, password, database, client name — and the query id of a [`Query`](#query) | 64 KiB | +| Each `String` of [`ClientInfo`](#clientinfo) — user, query id, host and client names, quota key, user agent, referer, URL, tracestate, JWT | 1 GiB — the generic bound, but read as the bytes arrive rather than allocated up front; `ClientInfo` is forwarded from the other interfaces, whose values can be longer, and persisted in async `Distributed` insert queue files | +| Number of current roles in [`ClientInfo`](#clientinfo), and the name of each | 65536 entries, 64 KiB each | +| Number of [`Setting`](#setting) entries and of [`Parameter`](#parameter) entries in a [`Query`](#query) | 65536 each | +| The name of each [`Setting`](#setting) and of each [`Parameter`](#parameter) | 64 KiB | +| The value of a [`Setting`](#setting) or a [`Parameter`](#parameter), and the query text | 1 GiB — the generic bound; a parameter is query-sized data, and the query text itself has to be read at this bound before the `auth_hash` covering it can be checked | +| The serialized externally granted roles of a [`Query`](#query) | 1 MiB in total, 65536 role names, 64 KiB each | +| Number of tables in a `TablesStatusRequest` / `TablesStatusResponse`, and the length of each database and table name | 100000 tables, 64 KiB per identifier | +| Number of UUIDs in a `PartUUIDs` packet (client-side) | 1000000 | +| Number of `out_of_order_buckets` in the block info of a [`Data`](#data) block | 256 — the number of two-level aggregation buckets | +| Number of `password_complexity_rules` in [`ServerHello`](#serverhello) | see the [note there](#serverhello) | +| Any other `String` or collection without a dedicated limit | 1 GiB | + +Exceeding a limit is answered with an [`Exception`](#exception) carrying `TOO_LARGE_ARRAY_SIZE` (128) or `TOO_LARGE_STRING_SIZE` (131). The values above are what the current server enforces; they are not negotiated, and a client should not rely on the server accepting anything close to them. + ## Versioning and feature gates {#versioning-and-feature-gates} ### Version negotiation {#version-negotiation} diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 37b5593026278..c268c0545e7f7 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -1577,8 +1577,11 @@ Packet Connection::receivePacket() // materializing it (do not resize a vector to a peer-chosen size). UInt64 num_part_uuids = 0; readVarUInt(num_part_uuids, *in); - if (num_part_uuids > DEFAULT_MAX_STRING_SIZE) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size (maximum: {})", DEFAULT_MAX_STRING_SIZE); + /// The count comes from the server and is only used to skip the payload, but it + /// still decides how much the client reads from the socket, so keep it realistic. + static constexpr UInt64 max_part_uuids = 1000000; + if (num_part_uuids > max_part_uuids) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size (maximum: {})", max_part_uuids); in->ignore(num_part_uuids * sizeof(UUID)); return res; } diff --git a/src/Core/BaseSettings.cpp b/src/Core/BaseSettings.cpp index da6718854c747..01c77b06e63dc 100644 --- a/src/Core/BaseSettings.cpp +++ b/src/Core/BaseSettings.cpp @@ -14,6 +14,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int INCORRECT_DATA; + extern const int TOO_LARGE_ARRAY_SIZE; extern const int TYPE_MISMATCH; extern const int UNKNOWN_SETTING; } @@ -32,6 +33,13 @@ String BaseSettingsHelpers::readString(ReadBuffer & in) return str; } +String BaseSettingsHelpers::readName(ReadBuffer & in) +{ + String str; + readStringBinaryGrowing(str, in, MAX_SETTING_NAME_SIZE); + return str; +} + void BaseSettingsHelpers::writeFlags(Flags flags, WriteBuffer & out) { @@ -61,6 +69,11 @@ void BaseSettingsHelpers::throwSettingNotFound(std::string_view name) throw Exception(ErrorCodes::UNKNOWN_SETTING, "Unknown setting '{}'", String{name}); } +void BaseSettingsHelpers::throwTooManySettings(std::string_view what) +{ + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too many {} received (maximum: {})", String{what}, MAX_SETTINGS_IN_A_SEQUENCE); +} + void BaseSettingsHelpers::throwValuelessSettingIsNotBool(std::string_view name, std::string_view type) { throw Exception( diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 9462822af5c19..149fe72671f21 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -29,6 +29,25 @@ namespace DB class ReadBuffer; class WriteBuffer; +/** A settings sequence on the wire is a list of name/value pairs, and both the number of pairs and + * the size of each name and value come from the peer. There are a few thousand declared settings + * and a handful of custom ones per query, so bound the number of pairs: otherwise a peer can keep + * the server allocating one pair at a time without a limit, and in interserver mode the settings of + * a query are read before its secret hash is checked. + */ +static constexpr size_t MAX_SETTINGS_IN_A_SEQUENCE = 65536; + +/** The name of a setting or of a query parameter is an identifier, so a name declared longer than + * this is rejected before any of it is read. The bound is the same as for the strings of the `Hello` + * packet. + * + * The value of a setting or of a parameter keeps the generic string bound: it is query-sized data + * (a parameter is substituted into the query), and the query text itself has to be read at that + * bound before the interserver hash that covers it can be checked, so a tighter bound here would + * only break large parameters without lowering what a peer can make the server allocate. + */ +static constexpr size_t MAX_SETTING_NAME_SIZE = 64 * 1024; + /// Utility functions and types used by the BaseSettings template class. /// These are implementation details that handle serialization, error reporting, /// and metadata management for settings. @@ -36,6 +55,7 @@ struct BaseSettingsHelpers { /// Error handling [[noreturn]] static void throwSettingNotFound(std::string_view name); + [[noreturn]] static void throwTooManySettings(std::string_view what); [[noreturn]] static void throwValuelessSettingIsNotBool(std::string_view name, std::string_view type); [[noreturn]] static void throwValuelessSettingIsNotBool(std::string_view name); [[noreturn]] static void throwValuelessSettingHasValue(std::string_view name); @@ -45,6 +65,8 @@ struct BaseSettingsHelpers /// Serialization helpers static void writeString(std::string_view str, WriteBuffer & out); static String readString(ReadBuffer & in); + /// The name of a setting or a query parameter, bounded by `MAX_SETTING_NAME_SIZE`. + static String readName(ReadBuffer & in); /// Setting metadata flags enum Flags : UInt64 @@ -732,9 +754,12 @@ void BaseSettings::readBinary(ReadBuffer & in) size_t num_settings = 0; readVarUInt(num_settings, in); + if (num_settings > MAX_SETTINGS_IN_A_SEQUENCE) + BaseSettingsHelpers::throwTooManySettings("settings"); + for (size_t i = 0; i < num_settings; ++i) { - String read_name = BaseSettingsHelpers::readString(in); + String read_name = BaseSettingsHelpers::readName(in); std::string_view name = TTraits::resolveName(read_name); size_t index = accessor.find(name); @@ -752,12 +777,16 @@ void BaseSettings::read(ReadBuffer & in, SettingsWriteFormat format) { resetToDefault(); const auto & accessor = Traits::Accessor::instance(); + size_t num_settings = 0; while (true) { - String read_name = BaseSettingsHelpers::readString(in); + String read_name = BaseSettingsHelpers::readName(in); if (read_name.empty() /* empty string is a marker of the end of settings */) break; + if (++num_settings > MAX_SETTINGS_IN_A_SEQUENCE) + BaseSettingsHelpers::throwTooManySettings("settings"); + std::string_view name = TTraits::resolveName(read_name); size_t index = accessor.find(name); diff --git a/src/Core/BlockInfo.cpp b/src/Core/BlockInfo.cpp index 49641369590aa..39458b5d6b043 100644 --- a/src/Core/BlockInfo.cpp +++ b/src/Core/BlockInfo.cpp @@ -35,6 +35,28 @@ void BlockInfo::write(WriteBuffer & out, UInt64 server_protocol_revision) const writeVarUInt(0, out); } +namespace +{ + +template +void readField(T & value, ReadBuffer & in) +{ + readBinary(value, in); +} + +/** The number of out of order buckets comes from the peer, and the vector is resized to it before the + * bucket ids are read. A list of two-level aggregation bucket ids cannot name more buckets than + * there are (256), and in practice it holds a few of them. + */ +constexpr size_t MAX_OUT_OF_ORDER_BUCKETS = 256; + +void readField(std::vector & value, ReadBuffer & in) +{ + readVectorBinary(value, in, MAX_OUT_OF_ORDER_BUCKETS); +} + +} + /// Read values in binary form. void BlockInfo::read(ReadBuffer & in, UInt64 client_protocol_revision) { @@ -51,7 +73,7 @@ void BlockInfo::read(ReadBuffer & in, UInt64 client_protocol_revision) #define READ_FIELD(TYPE, NAME, DEFAULT, FIELD_NUM, MIN_PROTOCOL_REVISION) \ case FIELD_NUM: \ if (client_protocol_revision >= (MIN_PROTOCOL_REVISION)) \ - readBinary(NAME, in); \ + readField(NAME, in); \ break; APPLY_FOR_BLOCK_INFO_FIELDS(READ_FIELD) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index c00af6c3fa1a3..6e8a0616fa0b6 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -9953,11 +9953,16 @@ void writeQueryParameters(const NameToNameMap & parameters, WriteBuffer & out) NameToNameMap readQueryParameters(ReadBuffer & in) { NameToNameMap parameters; + size_t num_parameters = 0; while (true) { - String name = BaseSettingsHelpers::readString(in); + String name = BaseSettingsHelpers::readName(in); if (name.empty()) break; + + if (++num_parameters > MAX_SETTINGS_IN_A_SEQUENCE) + BaseSettingsHelpers::throwTooManySettings("query parameters"); + std::ignore = BaseSettingsHelpers::readFlags(in); String value; ReadBufferFromOwnString buf(BaseSettingsHelpers::readString(in)); diff --git a/src/Core/tests/gtest_block_info.cpp b/src/Core/tests/gtest_block_info.cpp new file mode 100644 index 0000000000000..51b9f843fef7a --- /dev/null +++ b/src/Core/tests/gtest_block_info.cpp @@ -0,0 +1,70 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ + extern const int TOO_LARGE_ARRAY_SIZE; +} + +namespace +{ + +/// A `BlockInfo` with only the `out_of_order_buckets` field, declaring `declared_count` bucket ids +/// and sending `sent_count` of them. +std::string outOfOrderBucketsWire(size_t declared_count, size_t sent_count) +{ + WriteBufferFromOwnString out; + writeVarUInt(3, out); + writeVarUInt(declared_count, out); + for (size_t i = 0; i < sent_count; ++i) + writeBinary(static_cast(i), out); + writeVarUInt(0, out); + return out.str(); +} + +} + +TEST(BlockInfo, OutOfOrderBucketsCountIsBounded) +{ + /// The count comes from the peer, and the vector would be resized to it before any bucket id + /// arrives: a gigabyte of `Int32` is four gigabytes. + const std::string wire = outOfOrderBucketsWire(1ULL << 30, 0); + ReadBufferFromString in(wire); + BlockInfo info; + try + { + info.read(in, DBMS_MIN_REVISION_WITH_OUT_OF_ORDER_BUCKETS_IN_AGGREGATION); + ADD_FAILURE() << "an out of order buckets count of a gigabyte was accepted"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::TOO_LARGE_ARRAY_SIZE) << e.displayText(); + } +} + +TEST(BlockInfo, AllBucketIdsStillRoundTrip) +{ + /// A list cannot name more than the 256 buckets of two-level aggregation, and such a list is read. + BlockInfo sent; + for (Int32 i = 0; i < 256; ++i) + sent.out_of_order_buckets.push_back(i); + + WriteBufferFromOwnString out; + sent.write(out, DBMS_MIN_REVISION_WITH_OUT_OF_ORDER_BUCKETS_IN_AGGREGATION); + const std::string wire = out.str(); + + ReadBufferFromString in(wire); + BlockInfo received; + received.read(in, DBMS_MIN_REVISION_WITH_OUT_OF_ORDER_BUCKETS_IN_AGGREGATION); + EXPECT_EQ(received.out_of_order_buckets, sent.out_of_order_buckets); + EXPECT_TRUE(in.eof()); +} diff --git a/src/Core/tests/gtest_settings.cpp b/src/Core/tests/gtest_settings.cpp index 7a75f2e11052f..70b38fb5dc1ad 100644 --- a/src/Core/tests/gtest_settings.cpp +++ b/src/Core/tests/gtest_settings.cpp @@ -21,6 +21,7 @@ namespace DB::ErrorCodes { extern const int ATTEMPT_TO_READ_AFTER_EOF; + extern const int TOO_LARGE_STRING_SIZE; extern const int INCORRECT_DATA; } @@ -344,12 +345,16 @@ GTEST_TEST(Settings, ADeclaredLengthOnTheSettingsWireIsNotAnAllocation) std::string_view covers; SettingsWriteFormat format; std::function write_wire; + /// A setting name is an identifier, so a name declared longer than `MAX_SETTING_NAME_SIZE` is + /// refused before it is read at all, instead of running out of buffer. + int expected_code = DB::ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF; }; const std::vector cases = { {"the setting name, i.e. every BaseSettingsHelpers::readString carrier", SettingsWriteFormat::STRINGS_WITH_FLAGS, - [&](WriteBuffer & out) { writeVarUInt(declared_size, out); }}, + [&](WriteBuffer & out) { writeVarUInt(declared_size, out); }, + DB::ErrorCodes::TOO_LARGE_STRING_SIZE}, {"SettingFieldString::readBinary", SettingsWriteFormat::BINARY, [&](WriteBuffer & out) @@ -409,7 +414,7 @@ GTEST_TEST(Settings, ADeclaredLengthOnTheSettingsWireIsNotAnAllocation) } catch (const DB::Exception & e) { - EXPECT_EQ(e.code(), DB::ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF) << e.displayText(); + EXPECT_EQ(e.code(), test_case.expected_code) << e.displayText(); } } } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 755861fa88c38..728e58eee3eb5 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -192,8 +192,12 @@ inline void readIPv6Binary(IPv6 & ip, ReadBuffer & buf) buf.readStrict(reinterpret_cast(&ip.toUnderType()), size); } +/// The element count is read from `buf` and the vector is resized to it before the elements are +/// read, so where the count comes from an untrusted peer, pass a `max_size` that the data can +/// plausibly have instead of relying on the default. A `String` element is resized to its declared +/// size before its value arrives in the same way, so `max_element_size` bounds it as well. template -void readVectorBinary(V & v, ReadBuffer & buf, size_t max_size = DEFAULT_MAX_STRING_SIZE) +void readVectorBinary(V & v, ReadBuffer & buf, size_t max_size = DEFAULT_MAX_STRING_SIZE, size_t max_element_size = DEFAULT_MAX_STRING_SIZE) { using T = typename V::value_type; @@ -209,6 +213,9 @@ void readVectorBinary(V & v, ReadBuffer & buf, size_t max_size = DEFAULT_MAX_STR /// std::vector is bit-packed and has no contiguous element storage, so it cannot use the bulk path. if constexpr (is_trivially_serializable && !std::is_same_v) readNBytes(reinterpret_cast(v.data()), size * sizeof(T), buf); + else if constexpr (std::is_same_v) + for (size_t i = 0; i < size; ++i) + readStringBinary(v[i], buf, max_element_size); else for (size_t i = 0; i < size; ++i) readBinary(v[i], buf); diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 69f5dd92722a6..62e95c7405b1c 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -341,6 +341,18 @@ void ClientInfo::write(WriteBuffer & out, UInt64 server_protocol_revision, bool } +/** The strings of `ClientInfo` come from the peer, and in interserver mode `ClientInfo` is read before + * the secret hash of the query is checked, so they are read growing as the bytes arrive instead of + * being resized to the declared size first: a declared size does not become an allocation on its + * own. Their size keeps the generic bound, because `ClientInfo` is also forwarded from the other + * interfaces (an HTTP query id or quota key can be long) to the shards of a `Distributed` table and + * persisted in the queue files of an async `Distributed` insert, so a tighter bound would make the + * server reject what it wrote itself. The number of current roles is bounded, and a role name is an + * identifier, so it is bounded as well. + */ +static constexpr size_t MAX_CLIENT_INFO_ROLE_NAME_SIZE = 64 * 1024; +static constexpr size_t MAX_CLIENT_INFO_ROLES = 65536; + void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision, bool with_trailing_fields) { if (client_protocol_revision < DBMS_MIN_REVISION_WITH_CLIENT_INFO) @@ -354,11 +366,11 @@ void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision, bool wit resolve_client_hostname_on_demand = false; - readBinary(initial_user, in); - readBinary(initial_query_id, in); + readStringBinaryGrowing(initial_user, in); + readStringBinaryGrowing(initial_query_id, in); String initial_address_string; - readBinary(initial_address_string, in); + readStringBinaryGrowing(initial_address_string, in); /// The wire address must never reach Poco's resolver (getservbyname/DNS, trapped to SIGILL). For a /// SECONDARY_QUERY the value is consumed verbatim (system.query_log, interserver authenticate), so a /// non-"ip:port" form is corrupted input and is rejected as INCORRECT_DATA. For an INITIAL_QUERY the @@ -384,9 +396,9 @@ void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision, bool wit if (interface == Interface::TCP) { - readBinary(os_user, in); - readBinary(client_hostname, in); - readBinary(client_name, in); + readStringBinaryGrowing(os_user, in); + readStringBinaryGrowing(client_hostname, in); + readStringBinaryGrowing(client_name, in); readVarUInt(client_version_major, in); readVarUInt(client_version_minor, in); readVarUInt(client_tcp_protocol_version, in); @@ -397,25 +409,25 @@ void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision, bool wit readBinary(read_http_method, in); http_method = HTTPMethod(read_http_method); - readBinary(http_user_agent, in); + readStringBinaryGrowing(http_user_agent, in); if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO) - readBinary(forwarded_for, in); + readStringBinaryGrowing(forwarded_for, in); if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_REFERER_IN_CLIENT_INFO) - readBinary(http_referer, in); + readStringBinaryGrowing(http_referer, in); /// See the note in `write`: absent from the embedded `ClientInfo` of the persisted async /// `Distributed` insert header, where they are stored as trailing header fields instead. if (with_trailing_fields && client_protocol_revision >= DBMS_MIN_REVISION_WITH_HTTP_HANDLER_IN_CLIENT_INFO) { - readBinary(http_handler_name, in); - readBinary(http_request_url, in); + readStringBinaryGrowing(http_handler_name, in); + readStringBinaryGrowing(http_request_url, in); } } if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO) - readBinary(quota_key, in); + readStringBinaryGrowing(quota_key, in); if (client_protocol_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_DISTRIBUTED_DEPTH) readVarUInt(distributed_depth, in); @@ -436,7 +448,7 @@ void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision, bool wit { readBinary(client_trace_context.trace_id, in); readBinary(client_trace_context.span_id, in); - readBinary(client_trace_context.tracestate, in); + readStringBinaryGrowing(client_trace_context.tracestate, in); readBinary(client_trace_context.trace_flags, in); } } @@ -461,11 +473,11 @@ void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision, bool wit UInt8 have_jwt = 0; readBinary(have_jwt, in); if (have_jwt) - readBinary(jwt, in); + readStringBinaryGrowing(jwt, in); } if (with_trailing_fields && client_protocol_revision >= DBMS_MIN_REVISION_WITH_CLIENT_AGENT_IN_CLIENT_INFO) - readBinary(client_agent, in); + readStringBinaryGrowing(client_agent, in); if (with_trailing_fields && client_protocol_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_INTERNAL_QUERY_FLAG) readBinary(is_internal, in); @@ -477,7 +489,7 @@ void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision, bool wit if (have_current_roles) { std::vector roles; - readVectorBinary(roles, in); + readVectorBinary(roles, in, MAX_CLIENT_INFO_ROLES, MAX_CLIENT_INFO_ROLE_NAME_SIZE); current_roles = std::move(roles); } else diff --git a/src/Interpreters/TablesStatus.cpp b/src/Interpreters/TablesStatus.cpp index 2b784a359f976..979c4fec9ee74 100644 --- a/src/Interpreters/TablesStatus.cpp +++ b/src/Interpreters/TablesStatus.cpp @@ -84,6 +84,14 @@ std::string TablesStatusRequest::getAuthDigest() const return data; } +/** The number of tables comes from the peer and each of them is inserted into a collection before + * the names that follow are read, so bound it: a request or a response about a hundred thousand + * tables is already far beyond what a query touches. A database or a table name is an identifier + * and is resized to its declared size before its value arrives, so bound it as well. + */ +static constexpr size_t MAX_TABLES_IN_TABLES_STATUS = 100000; +static constexpr size_t MAX_TABLE_NAME_SIZE_IN_TABLES_STATUS = 64 * 1024; + void TablesStatusRequest::read(ReadBuffer & in, UInt64 client_protocol_revision) { if (client_protocol_revision < DBMS_MIN_REVISION_WITH_TABLES_STATUS) @@ -92,14 +100,15 @@ void TablesStatusRequest::read(ReadBuffer & in, UInt64 client_protocol_revision) size_t size = 0; readVarUInt(size, in); - if (size > DEFAULT_MAX_STRING_SIZE) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large collection size."); + if (size > MAX_TABLES_IN_TABLES_STATUS) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Too large collection size: {} (maximum: {})", size, MAX_TABLES_IN_TABLES_STATUS); for (size_t i = 0; i < size; ++i) { QualifiedTableName table_name; - readBinary(table_name.database, in); - readBinary(table_name.table, in); + readStringBinary(table_name.database, in, MAX_TABLE_NAME_SIZE_IN_TABLES_STATUS); + readStringBinary(table_name.table, in, MAX_TABLE_NAME_SIZE_IN_TABLES_STATUS); tables.emplace(std::move(table_name)); } } @@ -129,14 +138,15 @@ void TablesStatusResponse::read(ReadBuffer & in, UInt64 server_protocol_revision size_t size = 0; readVarUInt(size, in); - if (size > DEFAULT_MAX_STRING_SIZE) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large collection size."); + if (size > MAX_TABLES_IN_TABLES_STATUS) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Too large collection size: {} (maximum: {})", size, MAX_TABLES_IN_TABLES_STATUS); for (size_t i = 0; i < size; ++i) { QualifiedTableName table_name; - readBinary(table_name.database, in); - readBinary(table_name.table, in); + readStringBinary(table_name.database, in, MAX_TABLE_NAME_SIZE_IN_TABLES_STATUS); + readStringBinary(table_name.table, in, MAX_TABLE_NAME_SIZE_IN_TABLES_STATUS); TableStatus status; status.read(in, server_protocol_revision); diff --git a/src/Interpreters/tests/gtest_client_info_read.cpp b/src/Interpreters/tests/gtest_client_info_read.cpp index 47d303e1056eb..a621661c0e320 100644 --- a/src/Interpreters/tests/gtest_client_info_read.cpp +++ b/src/Interpreters/tests/gtest_client_info_read.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -18,6 +19,8 @@ using namespace DB; namespace DB::ErrorCodes { extern const int INCORRECT_DATA; + extern const int TOO_LARGE_ARRAY_SIZE; + extern const int TOO_LARGE_STRING_SIZE; } namespace @@ -384,6 +387,56 @@ TEST(ClientInfoRead, CurrentRolesRoundTripsTriState) EXPECT_EQ(*some, (std::vector{"role_a", "role_b"})); } +/// A full `ClientInfo` wire payload whose `current_roles` declares `declared_roles` names, the first of +/// which declares `declared_name_size` bytes, with no role bytes following: the declared sizes come +/// from the peer, and in interserver mode they are read before the secret hash of the query is checked. +static String makeClientInfoWireWithOversizedRoles(UInt64 declared_roles, UInt64 declared_name_size) +{ + String wire = makeFullClientInfoWire(ClientInfo::QueryKind::SECONDARY_QUERY, "127.0.0.1:9000"); + /// Drop the trailing `have_current_roles = no`. + wire.pop_back(); + + WriteBufferFromString buf(wire, AppendModeTag{}); + writeBinary(static_cast(1), buf); + writeVarUInt(declared_roles, buf); + if (declared_roles) + writeVarUInt(declared_name_size, buf); + buf.finalize(); + return wire; +} + +TEST(ClientInfoRead, CurrentRolesCountIsBounded) +{ + /// The vector would be resized to the declared count before any role name arrives. + ClientInfo info; + ReadBufferFromOwnString in(makeClientInfoWireWithOversizedRoles(1ULL << 30, 0)); + try + { + info.read(in, DBMS_TCP_PROTOCOL_VERSION); + FAIL() << "a current roles count of a gigabyte was accepted"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::TOO_LARGE_ARRAY_SIZE) << e.displayText(); + } +} + +TEST(ClientInfoRead, CurrentRoleNameSizeIsBounded) +{ + /// A role name would be resized to its declared size before its value arrives. + ClientInfo info; + ReadBufferFromOwnString in(makeClientInfoWireWithOversizedRoles(1, 1ULL << 30)); + try + { + info.read(in, DBMS_TCP_PROTOCOL_VERSION); + FAIL() << "a role name of a gigabyte was accepted"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::TOO_LARGE_STRING_SIZE) << e.displayText(); + } +} + /// An older peer can forward a server-initiated query whose context was never filled with a /// version, so `client_version_*` arrives as 0.0.0 over the wire and `read` overwrites the seed /// taken from the session. `setClientVersionFromConnectionIfUnknown` must then restore the peer's diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 938759d9205a0..faf0dff76ddbe 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -2077,6 +2077,16 @@ std::unique_ptr TCPHandler::makeSession() /// from consuming excessive memory. See #52501. static constexpr size_t MAX_HELLO_STRING_SIZE = 64 * 1024; +/** The externally granted roles are a serialized list of role names. Both the size of the list and + * the number of names in it come from the peer, and in interserver mode they are read before the + * secret hash of the query is checked, so bound them. + */ +static constexpr size_t MAX_EXTRA_ROLES_SIZE = 1024 * 1024; +/// Only read on the interserver path, which is compiled out without SSL. +[[maybe_unused]] static constexpr size_t MAX_EXTRA_ROLES = 65536; +/// A role name is an identifier, and it is resized to its declared size before its value arrives. +[[maybe_unused]] static constexpr size_t MAX_EXTRA_ROLE_NAME_SIZE = 64 * 1024; + void TCPHandler::receiveHello() { /// Receive `hello` packet. @@ -2572,7 +2582,7 @@ void TCPHandler::processQuery(std::shared_ptr & state) // TODO: check if having `is_interserver_mode` doesn't break interoperability with the CH-client. if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES) { - readStringBinary(received_extra_roles, *in); + readStringBinary(received_extra_roles, *in, MAX_EXTRA_ROLES_SIZE); } /// Interserver secret. @@ -2687,7 +2697,7 @@ void TCPHandler::processQuery(std::shared_ptr & state) { ReadBufferFromString buffer(received_extra_roles); - readVectorBinary(external_roles, buffer); + readVectorBinary(external_roles, buffer, MAX_EXTRA_ROLES, MAX_EXTRA_ROLE_NAME_SIZE); LOG_DEBUG(log, "Parsed extra roles [{}]", fmt::join(external_roles, ", ")); } @@ -2834,7 +2844,7 @@ void TCPHandler::processUnexpectedQuery() UInt64 skip_uint_64 = 0; String skip_string; - readStringBinary(skip_string, *in); + readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); ClientInfo skip_client_info; if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) @@ -2845,6 +2855,10 @@ void TCPHandler::processUnexpectedQuery() : SettingsWriteFormat::BINARY; skip_settings.read(*in, settings_format); + /// The same layout as in `processQuery`: the externally granted roles precede the interserver hash. + if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES) + readStringBinary(skip_string, *in, MAX_EXTRA_ROLES_SIZE); + std::string skip_hash; bool interserver_secret = client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET; if (interserver_secret) @@ -2858,7 +2872,7 @@ void TCPHandler::processUnexpectedQuery() readStringBinary(skip_string, *in); if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS) - skip_settings.read(*in, settings_format); + std::ignore = readQueryParameters(*in); throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet Query received from client"); } diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.python b/tests/queries/0_stateless/05055_native_protocol_collection_limits.python new file mode 100644 index 0000000000000..df47b8da1ff3b --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.python @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 + +# The server allocates from counts and sizes that a client puts on the wire, before the corresponding +# data arrives, so a collection read from the protocol has to be bounded by what the data can +# plausibly hold. In interserver mode some of these fields are read before the secret hash of the +# query is checked. + +import os +import socket +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "900000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "05055 native protocol" +CLIENT_REVISION = 54459 # the first revision with query parameters + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + + ba.append(byte) + + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise RuntimeError("Socket is closed") + size -= len(cur) + res.extend(cur) + + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + + if not byte & 0x80: + return x + + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8", "replace") + + +def connect(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(21, ba) + writeVarUInt(9, ba) + writeVarUInt(CLIENT_REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) + writeStringBinary("default", ba) + writeStringBinary("", ba) + s.sendall(ba) + + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # version major + readVarUInt(s) # version minor + readVarUInt(s) # revision + readStringBinary(s) # timezone + readStringBinary(s) # display name + readVarUInt(s) # version patch + + ba = bytearray() + writeStringBinary("", ba) # addendum: quota key + s.sendall(ba) + return s + + +def serializeClientInfo(ba, query_id, os_user_declared_size=None): + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds + ba.append(1) # TCP + if os_user_declared_size is None: + writeStringBinary("os_user", ba) # os_user + else: + writeVarUInt(os_user_declared_size, ba) # os_user, but the value never follows + return + writeStringBinary("client_hostname", ba) # client_hostname + writeStringBinary(CLIENT_NAME, ba) # client_name + writeVarUInt(21, ba) + writeVarUInt(9, ba) + writeVarUInt(CLIENT_REVISION, ba) + writeStringBinary("", ba) # quota_key + writeVarUInt(0, ba) # distributed_depth + writeVarUInt(1, ba) # client_version_patch + ba.append(0) # no telemetry + writeVarUInt(0, ba) # collaborate_with_initiator + writeVarUInt(0, ba) # count_participating_replicas + writeVarUInt(0, ba) # number_of_current_replica + + +def sendQuery( + s, + query, + num_settings=0, + os_user_declared_size=None, + setting_name_declared_size=None, + parameter_name_declared_size=None, +): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + ba.append(1) # INITIAL_QUERY + serializeClientInfo(ba, query_id, os_user_declared_size) + if os_user_declared_size is not None: + s.sendall(ba) + return + + if setting_name_declared_size is not None: + writeVarUInt( + setting_name_declared_size, ba + ) # a setting name that never follows + s.sendall(ba) + return + + for _ in range(num_settings): + writeStringBinary("max_threads", ba) + writeVarUInt(0, ba) # flags + writeStringBinary("1", ba) + writeStringBinary("", ba) # end of settings + + writeStringBinary("", ba) # no interserver secret + writeVarUInt(2, ba) # stage: Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + + if parameter_name_declared_size is not None: + writeVarUInt( + parameter_name_declared_size, ba + ) # a parameter name that never follows + s.sendall(ba) + return + + writeStringBinary("", ba) # end of query parameters + s.sendall(ba) + + +def sendTablesStatusRequest(s, num_tables, database_declared_size=None): + ba = bytearray() + writeVarUInt(5, ba) # TablesStatusRequest + writeVarUInt(num_tables, ba) + if database_declared_size is not None: + writeVarUInt( + database_declared_size, ba + ) # a database name whose value never follows + s.sendall(ba) + + +def sendEmptyBlock(s): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) + writeVarUInt(1, ba) + ba.append(0) # is_overflows + writeVarUInt(2, ba) + writeVarUInt(0, ba) + ba.extend([0] * 4) # bucket_num + writeVarUInt(0, ba) # columns + writeVarUInt(0, ba) # rows + s.sendall(ba) + + +def readException(s): + readUInt(s, 4) # code + readStringBinary(s) # name + message = readStringBinary(s) + readStringBinary(s) # stack trace + readUInt(s) # has_nested + for known in ( + "Too many settings received", + "Too large string size", + "Too large collection size", + "Attempt to read after eof", + "Cannot read all data", + ): + if known in message: + return known + return "UNEXPECTED " + message[:120] + + +def outcome(s): + """The name of the exception the server answered with, or how the exchange ended.""" + try: + while True: + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + return readException(s) + if packet_type == 5: # EndOfStream + return "end of stream" + if packet_type == 1: # Data + return "data" + if packet_type == 3: # Progress + continue + return "unexpected packet {}".format(packet_type) + except socket.timeout: + return "TIMEOUT" + except RuntimeError: + return "closed" + + +def rejected(s): + """A `TablesStatusRequest` is read before there is a query state to attach an exception to, so + the server may either answer with one or simply close the connection; either way it stops + reading instead of waiting for data that never comes.""" + result = outcome(s) + if result in ("closed", "Too large collection size", "Too large string size"): + return "rejected" + return result + + +def endOfStream(s): + """The client closed its side in the middle of a packet: the server may either answer with the + exception about the end of the stream or simply close the connection.""" + result = outcome(s) + if result in ("closed", "Attempt to read after eof", "Cannot read all data"): + return "end of stream" + return result + + +def main(): + # A settings sequence longer than any real query. + s = connect() + sendQuery(s, "SELECT 1", num_settings=65537) + print("too many settings:", outcome(s)) + s.close() + + # A `ClientInfo` string that declares a gigabyte and never sends it. It is read growing as the + # bytes arrive, so the server waits for them instead of allocating the declared size, and stops + # at the end of the stream. + s = connect() + sendQuery(s, "SELECT 1", os_user_declared_size=1 << 30) + s.shutdown(socket.SHUT_WR) + print("client info string of a gigabyte:", endOfStream(s)) + s.close() + + # A setting name that declares a gigabyte and never sends it: the count bound alone does not + # cover a single pair. + s = connect() + sendQuery(s, "SELECT 1", setting_name_declared_size=1 << 30) + print("setting name of a gigabyte:", outcome(s)) + s.close() + + # The same for a query parameter name. + s = connect() + sendQuery(s, "SELECT 1", parameter_name_declared_size=1 << 30) + print("query parameter name of a gigabyte:", outcome(s)) + s.close() + + # A `TablesStatusRequest` about more tables than a server can hold. + s = connect() + sendTablesStatusRequest(s, 1 << 30) + print("too many tables in a tables status request:", rejected(s)) + s.close() + + # A `TablesStatusRequest` whose database name declares a gigabyte and never sends it. + s = connect() + sendTablesStatusRequest(s, 1, database_declared_size=1 << 30) + print("table status identifier of a gigabyte:", rejected(s)) + s.close() + + # A well-formed query still works. + s = connect() + sendQuery(s, "SELECT 1", num_settings=1) + sendEmptyBlock(s) + print("well-formed query:", outcome(s)) + s.close() + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference b/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference new file mode 100644 index 0000000000000..4fc7c418dc5b5 --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference @@ -0,0 +1,7 @@ +too many settings: Too many settings received +client info string of a gigabyte: end of stream +setting name of a gigabyte: Too large string size +query parameter name of a gigabyte: Too large string size +too many tables in a tables status request: rejected +table status identifier of a gigabyte: rejected +well-formed query: data diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh b/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh new file mode 100755 index 0000000000000..dc01e3f6e8b3d --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# NOTE: this sh wrapper is required because of shell_config + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +python3 "$CUR_DIR"/05055_native_protocol_collection_limits.python diff --git a/tests/queries/0_stateless/05258_distributed_async_insert_long_client_info.reference b/tests/queries/0_stateless/05258_distributed_async_insert_long_client_info.reference new file mode 100644 index 0000000000000..35ab43b0eb62a --- /dev/null +++ b/tests/queries/0_stateless/05258_distributed_async_insert_long_client_info.reference @@ -0,0 +1 @@ +3 6 diff --git a/tests/queries/0_stateless/05258_distributed_async_insert_long_client_info.sh b/tests/queries/0_stateless/05258_distributed_async_insert_long_client_info.sh new file mode 100755 index 0000000000000..b6b6442ae93e6 --- /dev/null +++ b/tests/queries/0_stateless/05258_distributed_async_insert_long_client_info.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Tags: distributed + +# `ClientInfo` of an async `Distributed` insert is persisted in the queue file and forwarded to the +# shard. A query id longer than 64 KiB is accepted over HTTP, so the server must be able to read back +# and forward what it wrote itself. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE data (x UInt64) ENGINE = MergeTree ORDER BY x; + CREATE TABLE dist AS data ENGINE = Distributed(test_shard_localhost, currentDatabase(), data); + SYSTEM STOP DISTRIBUTED SENDS dist; +" + +query_id="$(printf 'q%.0s' {1..100000})_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query_id=${query_id}&distributed_foreground_insert=0&prefer_localhost_replica=0" \ + -d "INSERT INTO dist VALUES (1), (2), (3)" + +${CLICKHOUSE_CLIENT} --query " + SYSTEM FLUSH DISTRIBUTED dist; + SELECT count(), sum(x) FROM data; + DROP TABLE dist; + DROP TABLE data; +" From ee292def993fc125a11b17dbbb75189baa967811 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 03:55:33 +0000 Subject: [PATCH 263/296] Backport #124157 to 26.8: Do not allocate memory from the counts in columns_substreams.txt --- src/Storages/MergeTree/ColumnsSubstreams.cpp | 28 ++++++---- src/Storages/MergeTree/ColumnsSubstreams.h | 7 ++- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 1 + .../tests/gtest_columns_substreams.cpp | 54 +++++++++++++++++++ ...lumns_substreams_column_mismatch.reference | 2 + ...5331_columns_substreams_column_mismatch.sh | 28 ++++++++++ 6 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 src/Storages/MergeTree/tests/gtest_columns_substreams.cpp create mode 100644 tests/queries/0_stateless/05331_columns_substreams_column_mismatch.reference create mode 100755 tests/queries/0_stateless/05331_columns_substreams_column_mismatch.sh diff --git a/src/Storages/MergeTree/ColumnsSubstreams.cpp b/src/Storages/MergeTree/ColumnsSubstreams.cpp index e29e957d94e72..cecc8f30f0b5a 100644 --- a/src/Storages/MergeTree/ColumnsSubstreams.cpp +++ b/src/Storages/MergeTree/ColumnsSubstreams.cpp @@ -11,6 +11,7 @@ namespace DB namespace ErrorCodes { + extern const int CORRUPTED_DATA; extern const int LOGICAL_ERROR; } @@ -234,8 +235,7 @@ void ColumnsSubstreams::readText(ReadBuffer & buf) size_t num_columns = 0; DB::readText(num_columns, buf); assertString(" columns:\n", buf); - columns_substreams.reserve(num_columns); - first_substream_positions.reserve(num_columns); + /// Counts are untrusted file content: containers grow as entries are parsed. for (size_t i = 0; i != num_columns; ++i) { size_t num_substreams = 0; @@ -245,18 +245,26 @@ void ColumnsSubstreams::readText(ReadBuffer & buf) readBackQuotedStringWithSQLStyle(entry->column, buf); assertString(":\n", buf); - entry->substreams.resize(num_substreams); - entry->substream_to_local_position.reserve(num_substreams); + if (num_substreams == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Invalid columns substreams: column {} has no substreams", entry->column); + for (size_t j = 0; j != num_substreams; ++j) { assertChar('\t', buf); - readString(entry->substreams[j], buf); + readString(entry->substreams.emplace_back(), buf); assertChar('\n', buf); - entry->substream_to_local_position[entry->substreams[j]] = j; + } + + /// Filled after parsing to reserve the exact size: the interned copy of the entry keeps the bucket count. + entry->substream_to_local_position.reserve(entry->substreams.size()); + for (size_t j = 0; j != entry->substreams.size(); ++j) + { + if (!entry->substream_to_local_position.emplace(entry->substreams[j], j).second) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Invalid columns substreams: column {} has duplicate substream {}", entry->column, entry->substreams[j]); } first_substream_positions.push_back(static_cast(total_substreams)); - total_substreams += num_substreams; + total_substreams += entry->substreams.size(); columns_substreams.emplace_back(std::move(entry)); } } @@ -341,15 +349,15 @@ void ColumnsSubstreams::internColumnEntries(const std::function & columns) const +void ColumnsSubstreams::validateColumns(const std::vector & columns, int error_code) const { if (columns.size() != columns_substreams.size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid columns substreams: expected {} columns, got {}", columns.size(), columns_substreams.size()); + throw Exception(error_code, "Invalid columns substreams: expected {} columns, got {}", columns.size(), columns_substreams.size()); for (size_t i = 0; i != columns_substreams.size(); ++i) { if (columns_substreams[i]->column != columns[i]) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected column at position {} in columns substreams: expected {}, got {}", i, columns[i], columns_substreams[i]->column); + throw Exception(error_code, "Unexpected column at position {} in columns substreams: expected {}, got {}", i, columns[i], columns_substreams[i]->column); } } diff --git a/src/Storages/MergeTree/ColumnsSubstreams.h b/src/Storages/MergeTree/ColumnsSubstreams.h index 2ac3103017bf8..b64186a82ca5e 100644 --- a/src/Storages/MergeTree/ColumnsSubstreams.h +++ b/src/Storages/MergeTree/ColumnsSubstreams.h @@ -9,6 +9,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + struct MergeTreeSettings; using MergeTreeSettingsPtr = std::shared_ptr; @@ -66,7 +71,7 @@ class ColumnsSubstreams bool empty() const { return !total_substreams; } /// Check that we have substreams for all columns and they have the same order as in provided list. - void validateColumns(const std::vector & columns) const; + void validateColumns(const std::vector & columns, int error_code = ErrorCodes::LOGICAL_ERROR) const; /// Check that all substream names have valid prefixes matching their column names. /// Every substream for a column must start with escapeForFileName(column_name) (or diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index d22df1d1f9ae8..4b7b5be4a128c 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -2615,6 +2615,7 @@ void IMergeTreeDataPart::loadColumnsSubstreams() } } + loaded_columns_substreams.validateColumns(getColumns().getNames(), ErrorCodes::CORRUPTED_DATA); setColumnsSubstreams(loaded_columns_substreams); } /// In Compact part with marks for substreams we must have substreams file. For other cases it's not mandatory. diff --git a/src/Storages/MergeTree/tests/gtest_columns_substreams.cpp b/src/Storages/MergeTree/tests/gtest_columns_substreams.cpp new file mode 100644 index 0000000000000..0af98ca581c80 --- /dev/null +++ b/src/Storages/MergeTree/tests/gtest_columns_substreams.cpp @@ -0,0 +1,54 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ + extern const int CORRUPTED_DATA; +} + +/// A count above the address space fails to parse at the end of the file instead of throwing `std::bad_alloc`. +TEST(ColumnsSubstreams, ReadTextThrowsOnCountsBeyondFile) +{ + const std::vector files = { + "columns substreams version: 1\n1000000000000000 columns:\n", + "columns substreams version: 1\n1 columns:\n1000000000000000 substreams for column `c`:\n\tc\n", + }; + + for (const auto & file : files) + { + SCOPED_TRACE(file); + ReadBufferFromString buf(file); + ColumnsSubstreams columns_substreams; + EXPECT_THROW(columns_substreams.readText(buf), Exception); + } +} + +/// Entries the writer cannot produce: a column without substreams, a substream recorded twice. +TEST(ColumnsSubstreams, ReadTextThrowsOnInvalidColumnEntry) +{ + const std::vector files = { + "columns substreams version: 1\n1 columns:\n0 substreams for column `c`:\n", + "columns substreams version: 1\n1 columns:\n2 substreams for column `c`:\n\tc\n\tc\n", + }; + + for (const auto & file : files) + { + SCOPED_TRACE(file); + ReadBufferFromString buf(file); + ColumnsSubstreams columns_substreams; + try + { + columns_substreams.readText(buf); + ADD_FAILURE() << "readText accepted an invalid column entry"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::CORRUPTED_DATA); + } + } +} diff --git a/tests/queries/0_stateless/05331_columns_substreams_column_mismatch.reference b/tests/queries/0_stateless/05331_columns_substreams_column_mismatch.reference new file mode 100644 index 0000000000000..37b2d4c14c491 --- /dev/null +++ b/tests/queries/0_stateless/05331_columns_substreams_column_mismatch.reference @@ -0,0 +1,2 @@ +0 +broken-on-start diff --git a/tests/queries/0_stateless/05331_columns_substreams_column_mismatch.sh b/tests/queries/0_stateless/05331_columns_substreams_column_mismatch.sh new file mode 100755 index 0000000000000..edb3d4fddbbd4 --- /dev/null +++ b/tests/queries/0_stateless/05331_columns_substreams_column_mismatch.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# A `columns_substreams.txt` with fewer columns than `columns.txt` detaches its part as broken and the +# table stays queryable: red if the mismatch is a logical error (aborts debug builds). + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +WORKING_DIR="${CLICKHOUSE_TMP}/05331_columns_substreams_column_mismatch" +rm -rf "${WORKING_DIR}" +mkdir -p "${WORKING_DIR}" + +${CLICKHOUSE_LOCAL} --path "${WORKING_DIR}" --multiquery -q " + CREATE TABLE t (id UInt64, val UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; + INSERT INTO t SELECT number, number FROM numbers(10); +" "$(find "${WORKING_DIR}" -name columns_substreams.txt)" + +${CLICKHOUSE_LOCAL} --path "${WORKING_DIR}" --multiquery -q " + SELECT count() FROM t; + SELECT reason FROM system.detached_parts WHERE database = currentDatabase() AND table = 't'; +" Date: Mon, 12 Oct 2026 07:49:30 +0000 Subject: [PATCH 264/296] Backport #124280 to 26.8: Bump `thrift` from v0.24.0 to v0.25.0 --- contrib/thrift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/thrift b/contrib/thrift index 13d30d6a6cdfb..26bf3f0c3cade 160000 --- a/contrib/thrift +++ b/contrib/thrift @@ -1 +1 @@ -Subproject commit 13d30d6a6cdfbd44c07977b8b762e88d9d5eb6df +Subproject commit 26bf3f0c3cade6f2c9fac82d4962751466282c23 From 307fbc224b3523d405bd5820319765cf103d5bfb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 09:14:27 +0000 Subject: [PATCH 265/296] Backport #124282 to 26.8: Bump `mongo-c-driver` from 2.5.4 to 2.5.5 --- .gitmodules | 2 +- contrib/mongo-c-driver | 2 +- contrib/mongo-c-driver-cmake/CMakeLists.txt | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index ea6e27a866283..cee2d45690c0d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -332,7 +332,7 @@ url = https://github.com/ClickHouse/mongo-cxx-driver.git [submodule "contrib/mongo-c-driver"] path = contrib/mongo-c-driver - url = https://github.com/ClickHouse/mongo-c-driver.git + url = https://github.com/ClickHouse/mongo-c-driver [submodule "contrib/sz3"] path = contrib/sz3 url = https://github.com/ClickHouse/SZ3.git diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index a00f0b91bd24f..70de9e880072b 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit a00f0b91bd24f8ddbe62ec3864cb3704385670f5 +Subproject commit 70de9e880072b120ccbb48c77de334775747d7a9 diff --git a/contrib/mongo-c-driver-cmake/CMakeLists.txt b/contrib/mongo-c-driver-cmake/CMakeLists.txt index a79ef97c8fe2c..43ac3f5cd0283 100644 --- a/contrib/mongo-c-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-c-driver-cmake/CMakeLists.txt @@ -6,12 +6,12 @@ endif() set(libbson_VERSION_MAJOR 2) set(libbson_VERSION_MINOR 5) -set(libbson_VERSION_PATCH 4) -set(libbson_VERSION 2.5.4) +set(libbson_VERSION_PATCH 5) +set(libbson_VERSION 2.5.5) set(libmongoc_VERSION_MAJOR 2) set(libmongoc_VERSION_MINOR 5) -set(libmongoc_VERSION_PATCH 4) -set(libmongoc_VERSION 2.5.4) +set(libmongoc_VERSION_PATCH 5) +set(libmongoc_VERSION 2.5.5) set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src") set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src") From 7bd92db6cb261a5f363439ae95d2a056fe4a0cd5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 09:37:50 +0000 Subject: [PATCH 266/296] Backport #118195 to 26.8: Tolerate non-UTF-8 bytes when the integration harness decodes captured output --- tests/integration/helpers/cluster.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index e056c5b311dc9..a32f1617a384d 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -234,7 +234,7 @@ def run_and_check( logging.debug("Env:%s", env) if not nothrow: raise Exception( - f"Command [{shell_args}] return non-zero code {res.returncode}: {res.stderr.decode('utf-8')}" + f"Command [{shell_args}] return non-zero code {res.returncode}: {err}" ) return out @@ -4451,9 +4451,9 @@ def shutdown( if self.docker_logs_proc is not None: self.docker_logs_proc.kill() - if not sanitizer_assert_instance: + if not sanitizer_assert_instance and not ignore_sanitizer: # Search for sinitizer signs in docker.log if it's still empty - with open(self.docker_logs_path, "r") as f: + with open(self.docker_logs_path, "r", errors="replace") as f: for line in f: if SANITIZER_SIGN in line: sanitizer_assert_instance = line.split("|")[0].strip() @@ -6153,7 +6153,7 @@ def has_new_rows_in_log(): status = handle.status if status == "exited": raise Exception( - f"Instance `{self.name}' failed to start. Container status: {status}, logs: {handle.logs().decode('utf-8')}" + f"Instance `{self.name}' failed to start. Container status: {status}, logs: {handle.logs().decode('utf-8', errors='replace')}" ) deadline = start_time + timeout @@ -6166,7 +6166,7 @@ def has_new_rows_in_log(): if current_time >= deadline: raise Exception( f"Timed out while waiting for instance `{self.name}' with ip address {self.ip_address} to start. " - f"Container status: {status}, logs: {handle.logs().decode('utf-8')}" + f"Container status: {status}, logs: {handle.logs().decode('utf-8', errors='replace')}" ) socket_timeout = min(timeout, deadline - current_time) From 56cc84c726a6fb4737f1850fe32fe7614726b8e8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 09:56:25 +0000 Subject: [PATCH 267/296] Backport #120885 to 26.8: Guard the constant-to-literal walks with checkStackSize --- src/Analyzer/Utils.cpp | 4 ++++ src/Columns/ColumnObject.cpp | 4 ++++ .../05218_dynamic_json_constant_deep_recursion.reference | 1 + .../05218_dynamic_json_constant_deep_recursion.sql | 8 ++++++++ 4 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference create mode 100644 tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index 7344b0435ab67..2ec2414d1f543 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -1362,6 +1363,9 @@ namespace Field getFieldFromColumnForASTLiteralImpl(const ColumnPtr & column, size_t row, const DataTypePtr & data_type, bool is_inside_object) { + /// Nesting is part of the value rather than of the query text, so no parser limit bounds it. + checkStackSize(); + if (isColumnConst(*column)) return getFieldFromColumnForASTLiteralImpl(assert_cast(*column).getDataColumnPtr(), 0, data_type, is_inside_object); diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 164211e5afb55..af2242c747b23 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace DB @@ -272,6 +273,9 @@ MutableColumnPtr ColumnObject::cloneResized(size_t size) const Field ColumnObject::operator[](size_t n) const { + /// Nesting is part of the value rather than of the query text, so no parser limit bounds it. + checkStackSize(); + Object object; for (const auto & [path, column] : typed_paths) diff --git a/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference new file mode 100644 index 0000000000000..99b8764f122fa --- /dev/null +++ b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference @@ -0,0 +1 @@ +{"a":{"a":{}}} diff --git a/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql new file mode 100644 index 0000000000000..6178d3e164508 --- /dev/null +++ b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql @@ -0,0 +1,8 @@ +SELECT finalizeAggregation(CAST(unhex('0130008008200000000101613000800820000000010161300080082000000000') AS AggregateFunction(any, Dynamic))); + +-- Turning a Dynamic constant into a literal recurses once per JSON level, and the depth comes from the +-- value, so it must be reported rather than overflow the stack. +SELECT finalizeAggregation(CAST(unhex(concat('01', repeat('3000800820000000010161', 4000), '300080082000000000')) AS AggregateFunction(any, Dynamic))); -- { serverError TOO_DEEP_RECURSION } + +-- Constant folding reads the value as a Field, which walks the same nesting through the columns. +SELECT if(finalizeAggregation(CAST(unhex(concat('01', repeat('3000800820000000010161', 4000), '300080082000000000')) AS AggregateFunction(any, Dynamic))), 1, 2); -- { serverError TOO_DEEP_RECURSION } From 3b6f3f6a7f6e9a56ee946cfe8f73f70cbdb1709b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 11:17:45 +0000 Subject: [PATCH 268/296] Backport #124991 to 26.8: Fix a filter on a Hive partition column pruning by the data file's column of the same name (revert of revert with a fix) --- src/Formats/FormatFilterInfo.cpp | 2 +- src/Formats/FormatFilterInfo.h | 1 + .../QueryPlan/ReadFromObjectStorageStep.cpp | 3 +- .../StorageObjectStorageSource.cpp | 2 +- src/Storages/StorageFile.cpp | 3 +- src/Storages/StorageURL.cpp | 3 +- src/Storages/prepareReadingFromFormat.cpp | 63 ++++++++++++++ src/Storages/prepareReadingFromFormat.h | 4 + ...oning_filter_on_partition_column.reference | 76 +++++++++++++++++ ...artitioning_filter_on_partition_column.sql | 85 +++++++++++++++++++ 10 files changed, 237 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05325_hive_partitioning_filter_on_partition_column.reference create mode 100644 tests/queries/0_stateless/05325_hive_partitioning_filter_on_partition_column.sql diff --git a/src/Formats/FormatFilterInfo.cpp b/src/Formats/FormatFilterInfo.cpp index b585592a77d95..fb276762cc4a0 100644 --- a/src/Formats/FormatFilterInfo.cpp +++ b/src/Formats/FormatFilterInfo.cpp @@ -83,7 +83,7 @@ FormatFilterInfo::FormatFilterInfo() = default; bool FormatFilterInfo::hasFilter() const { - return filter_actions_dag != nullptr; + return filter_actions_dag || row_level_filter || prewhere_info; } namespace diff --git a/src/Formats/FormatFilterInfo.h b/src/Formats/FormatFilterInfo.h index 620a0b0253aa9..2126610d3696c 100644 --- a/src/Formats/FormatFilterInfo.h +++ b/src/Formats/FormatFilterInfo.h @@ -122,6 +122,7 @@ struct FormatFilterInfo std::exception_ptr init_exception; public: + /// True if the format may skip rows of the file. bool hasFilter() const; /// Creates `key_condition` and `additional_columns` with std::call_once semantics. diff --git a/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp b/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp index a27873b024ca1..034058bbcaceb 100644 --- a/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp +++ b/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp @@ -122,8 +122,9 @@ void ReadFromObjectStorageStep::initializePipeline(QueryPipelineBuilder & pipeli // here create for node -> query -> level thread pool auto parser_shared_resources = std::make_shared(context->getSettingsRef(), num_streams); + /// Iceberg reads the row lineage columns from the data file. auto format_filter_info = std::make_shared( - filter_actions_dag, + info.getFormatFilter(filter_actions_dag, /*keep_row_lineage_columns=*/ configuration->isIcebergConfiguration()), context, configuration->getColumnMapperForCurrentSchema(storage_snapshot->metadata, context), query_info.row_level_filter, diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 99db81265c8cc..d6e804216ffb1 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -894,7 +894,7 @@ Chunk StorageObjectStorageSource::generate() } if (reader.getInputFormat() && read_context->getSettingsRef()[Setting::use_cache_for_count_from_files] - && !format_filter_info->filter_actions_dag + && !format_filter_info->hasFilter() && !hasAttachedDeletes(*reader.getObjectInfo()) && !reader.getObjectInfo()->rows_to_read) addNumRowsToCache(*reader.getObjectInfo(), total_rows_in_file); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 64565be6ad13d..f7d241e18f655 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -2383,7 +2383,8 @@ void ReadFromFile::initializePipeline(QueryPipelineBuilder & pipeline, const Bui progress_callback(FileProgress(0, storage->total_bytes_to_read)); auto parser_shared_resources = std::make_shared(ctx->getSettingsRef(), num_streams); - auto format_filter_info = std::make_shared(filter_actions_dag, ctx, nullptr, query_info.row_level_filter, query_info.prewhere_info); + auto format_filter_info = std::make_shared( + info.getFormatFilter(filter_actions_dag), ctx, nullptr, query_info.row_level_filter, query_info.prewhere_info); for (size_t i = 0; i < num_streams; ++i) { diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index cd59a8c92aa71..8b409a96a27ec 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1386,7 +1386,8 @@ void ReadFromURL::initializePipeline(QueryPipelineBuilder & pipeline, const Buil pipes.reserve(num_streams); auto parser_shared_resources = std::make_shared(settings, num_streams); - auto format_filter_info = std::make_shared(filter_actions_dag, context, nullptr, query_info.row_level_filter, query_info.prewhere_info); + auto format_filter_info = std::make_shared( + info.getFormatFilter(filter_actions_dag), context, nullptr, query_info.row_level_filter, query_info.prewhere_info); for (size_t i = 0; i < num_streams; ++i) { diff --git a/src/Storages/prepareReadingFromFormat.cpp b/src/Storages/prepareReadingFromFormat.cpp index 39298dbe05bcf..a6b19de7d6f93 100644 --- a/src/Storages/prepareReadingFromFormat.cpp +++ b/src/Storages/prepareReadingFromFormat.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include #include @@ -15,6 +17,7 @@ #include #include #include +#include #include #include @@ -348,6 +351,66 @@ SerializationInfoByName getSerializationHintsForFileLikeStorage(const StorageMet return res; } +namespace +{ + +bool isRowLineageColumn(const String & name) +{ + return name == "_row_id" || name == "_last_updated_sequence_number"; +} + +} + +std::shared_ptr ReadFromFormatInfo::getFormatFilter( + const std::shared_ptr & filter_actions_dag, bool keep_row_lineage_columns) const +{ + if (!filter_actions_dag || (hive_partition_columns_to_read_from_file_path.empty() && requested_virtual_columns.empty())) + return filter_actions_dag; + + auto is_added_after_format = [&](const String & name) + { + return hive_partition_columns_to_read_from_file_path.contains(name) + || (requested_virtual_columns.contains(name) && !(keep_row_lineage_columns && isRowLineageColumn(name))); + }; + + auto reads_added_column = [&](const ActionsDAG::Node * atom) + { + std::unordered_set visited; + std::vector stack{atom}; + while (!stack.empty()) + { + const auto * node = stack.back(); + stack.pop_back(); + if (!visited.insert(node).second) + continue; + if (node->type == ActionsDAG::ActionType::INPUT && is_added_after_format(node->result_name)) + return true; + stack.insert(stack.end(), node->children.begin(), node->children.end()); + /// `indexHint` keeps its arguments in its own DAG, not in `children`, and the format uses them for pruning. + if (node->type == ActionsDAG::ActionType::FUNCTION && node->function_base->getName() == "indexHint") + { + const auto & adaptor = assert_cast(*node->function_base); + for (const auto & inner : assert_cast(*adaptor.getFunction()).getActions().getNodes()) + stack.push_back(&inner); + } + } + return false; + }; + + auto atoms = ActionsDAG::extractConjunctionAtoms(filter_actions_dag->getOutputs().at(0)); + ActionsDAG::NodeRawConstPtrs kept; + for (const auto * atom : atoms) + if (!reads_added_column(atom)) + kept.push_back(atom); + + if (kept.size() == atoms.size()) + return filter_actions_dag; + if (kept.empty()) + return nullptr; + auto dag = ActionsDAG::buildFilterActionsDAG(kept); + return std::make_shared(std::move(*dag)); +} + void ReadFromFormatInfo::serialize(IQueryPlanStep::Serialization & ctx) const { source_header.getNamesAndTypesList().writeTextWithNamesInStorage(ctx.out); diff --git a/src/Storages/prepareReadingFromFormat.h b/src/Storages/prepareReadingFromFormat.h index f33a95d65226b..445f79d694248 100644 --- a/src/Storages/prepareReadingFromFormat.h +++ b/src/Storages/prepareReadingFromFormat.h @@ -44,6 +44,10 @@ namespace DB /// The list of hive partition columns. It shall be read from the path regardless if it is present in the file NamesAndTypesList hive_partition_columns_to_read_from_file_path; + /// The filter without the conjuncts on hive partition and virtual columns (except row lineage ones if `keep_row_lineage_columns`): + /// they are added after the format, and a column of the data file with the same name holds other values. + std::shared_ptr getFormatFilter( + const std::shared_ptr & filter_actions_dag, bool keep_row_lineage_columns = false) const; PrewhereInfoPtr prewhere_info; FilterDAGInfoPtr row_level_filter; }; diff --git a/tests/queries/0_stateless/05325_hive_partitioning_filter_on_partition_column.reference b/tests/queries/0_stateless/05325_hive_partitioning_filter_on_partition_column.reference new file mode 100644 index 0000000000000..d864b117e4482 --- /dev/null +++ b/tests/queries/0_stateless/05325_hive_partitioning_filter_on_partition_column.reference @@ -0,0 +1,76 @@ +-- { echoOn } +SELECT key, count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') GROUP BY key; +9 1000 +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9; +1000 +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 10; +0 +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9 AND v < 10; +10 +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9 OR v < 10; +1000 +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key IN (9); +1000 +SELECT count(), sum(v) FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet) WHERE key = 9; +1000 499500 +SELECT count() FROM file(currentDatabase() || '/05325/nullable/key=9/data.parquet', Parquet, 'key Nullable(Int64), v Int64') WHERE key IS NOT NULL; +1000 +SELECT count() FROM file(currentDatabase() || '/05325/only/a=1/key=9/data.parquet', Parquet, 'key Int64') WHERE indexHint(key = 9); +1000 +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE indexHint(key = 9); +1000 +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE indexHint(key = 9) AND v < 1000; +1000 +CREATE TABLE t_05325_s3 (key Int64, v Int64) ENGINE = S3(s3_conn, filename = currentDatabase() || '/05325/key=9/data.parquet', format = Parquet); +SELECT count() FROM t_05325_s3 WHERE key = 9 AND v < 10 SETTINGS use_query_condition_cache = 1, log_comment = '05325_qcc_1'; +10 +SELECT count() FROM t_05325_s3 WHERE key = 9 AND v < 10 SETTINGS use_query_condition_cache = 1, log_comment = '05325_qcc_2'; +10 +DROP TABLE t_05325_s3; +SELECT count() FROM url('http://localhost:11111/test/' || currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9; +1000 +SELECT _file, count() FROM file(currentDatabase() || '/05325/virtual/data.parquet', Parquet, 'v Int64') GROUP BY _file; +data.parquet 1000 +SELECT count() FROM file(currentDatabase() || '/05325/virtual/data.parquet', Parquet, 'v Int64') WHERE _file = 'data.parquet'; +1000 +SELECT count() FROM file(currentDatabase() || '/05325/virtual/data.parquet', Parquet, 'v Int64') WHERE indexHint(_file = 'data.parquet'); +1000 +CREATE TABLE t_05325_file AS file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64'); +CREATE ROW POLICY p_05325_file ON t_05325_file USING key = 9 TO ALL; +SELECT count() FROM t_05325_file; +1000 +DROP ROW POLICY p_05325_file ON t_05325_file; +DROP TABLE t_05325_file; +SELECT count(), any(old_path) FROM t_05325_iceberg; +1000 evil +SELECT count() FROM t_05325_iceberg WHERE _path != 'evil'; +1000 +DROP TABLE t_05325_iceberg; +-- The format still prunes row groups on the columns of the data file. +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9 AND v < 10 SETTINGS log_comment = '05325_prune'; +10 +SYSTEM FLUSH LOGS query_log; +SELECT ProfileEvents['ParquetPrunedRowGroups'] > 0 FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '05325_prune' AND type = 'QueryFinish'; +1 +SELECT sum(ProfileEvents['QueryConditionCacheHits'] + ProfileEvents['QueryConditionCacheMisses']) > 0 FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '05325_qcc_2' AND type = 'QueryFinish'; +1 +-- A PREWHERE applied by the format does not let the source cache the filtered row count as the file's row count. +SELECT count(), sum(v) FROM file(currentDatabase() || '/05325/count/key=9/data.parquet', Parquet, 'key Int64, v Int64') PREWHERE v < 100 + 0 * rand() WHERE key = 9; +100 4950 +SELECT count() FROM file(currentDatabase() || '/05325/count/key=9/data.parquet', Parquet, 'key Int64, v Int64') SETTINGS optimize_count_from_files = 1, use_cache_for_count_from_files = 1, optimize_trivial_count_query = 1; +1000 +SELECT count() FROM file(currentDatabase() || '/05325/count/key=9/data.parquet', Parquet, 'key Int64, v Int64') PREWHERE v < 100 + 0 * rand() WHERE key = 9; +100 +-- Neither does a join runtime filter nor the top-K filter applied by the format. +SELECT count() > 0 FROM (EXPLAIN actions = 1, pretty = 0 SELECT count() FROM file(currentDatabase() || '/05325/join/key=9/data.parquet', Parquet, 'key Int64, v Int64') AS f INNER JOIN (SELECT toInt64(number) AS v FROM numbers(10)) AS t ON f.v = t.v WHERE f.key = 9 SETTINGS enable_join_runtime_filters = 1, join_runtime_filter_min_probe_rows = 0, join_algorithm = 'hash,parallel_hash', query_plan_join_swap_table = 0, optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1, query_plan_max_step_description_length = 1000) WHERE explain ILIKE '%prewhere filter column: %__applyfilter(%'; +1 +SELECT count() FROM file(currentDatabase() || '/05325/join/key=9/data.parquet', Parquet, 'key Int64, v Int64') AS f INNER JOIN (SELECT toInt64(number) AS v FROM numbers(10)) AS t ON f.v = t.v WHERE f.key = 9 SETTINGS enable_join_runtime_filters = 1, join_runtime_filter_min_probe_rows = 0, join_algorithm = 'hash,parallel_hash', query_plan_join_swap_table = 0, optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1; +10 +SELECT count() FROM file(currentDatabase() || '/05325/join/key=9/data.parquet', Parquet, 'key Int64, v Int64') SETTINGS optimize_count_from_files = 1, use_cache_for_count_from_files = 1, optimize_trivial_count_query = 1; +1000 +SELECT v FROM file(currentDatabase() || '/05325/topk/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9 ORDER BY v LIMIT 3 SETTINGS use_top_k_dynamic_filtering = 1, query_plan_max_limit_for_top_k_optimization = 1000, input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_filter_push_down = 1, max_threads = 1, max_parsing_threads = 1; +0 +1 +2 +SELECT count() FROM file(currentDatabase() || '/05325/topk/key=9/data.parquet', Parquet, 'key Int64, v Int64') SETTINGS optimize_count_from_files = 1, use_cache_for_count_from_files = 1, optimize_trivial_count_query = 1; +1000 diff --git a/tests/queries/0_stateless/05325_hive_partitioning_filter_on_partition_column.sql b/tests/queries/0_stateless/05325_hive_partitioning_filter_on_partition_column.sql new file mode 100644 index 0000000000000..58346008152eb --- /dev/null +++ b/tests/queries/0_stateless/05325_hive_partitioning_filter_on_partition_column.sql @@ -0,0 +1,85 @@ +-- Tags: no-fasttest, no-replicated-database +-- Tag no-fasttest: depends on S3, Parquet and Iceberg +-- Tag no-replicated-database: the replicated DDL worker evaluates the `IcebergLocal` arguments without the session, so it does not see the `TEMPORARY TABLE iceberg_path` + +-- A filter or a row policy on a column that is added after the data file is read (a Hive partition column, +-- a virtual column) uses the value of that column, even when the data file has a column with the same name. + +-- The path says `key = 9`, the data files store `key = 10`, NULL and `_file = 'evil'`. +INSERT INTO FUNCTION file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') SELECT 10, number FROM numbers(1000) SETTINGS engine_file_truncate_on_insert = 1, output_format_parquet_row_group_size = 100; +INSERT INTO FUNCTION file(currentDatabase() || '/05325/nullable/key=9/data.parquet', Parquet, 'key Nullable(Int64), v Int64') SELECT NULL, number FROM numbers(1000) SETTINGS engine_file_truncate_on_insert = 1, output_format_parquet_row_group_size = 100; +INSERT INTO FUNCTION file(currentDatabase() || '/05325/virtual/data.parquet', Parquet, '_file String, v Int64') SELECT 'evil', number FROM numbers(1000) SETTINGS engine_file_truncate_on_insert = 1, output_format_parquet_row_group_size = 100; +INSERT INTO FUNCTION file(currentDatabase() || '/05325/count/key=9/data.parquet', Parquet, 'key Int64, v Int64') SELECT 10, number FROM numbers(1000) SETTINGS engine_file_truncate_on_insert = 1, output_format_parquet_row_group_size = 100; +INSERT INTO FUNCTION file(currentDatabase() || '/05325/join/key=9/data.parquet', Parquet, 'v Int64') SELECT number AS v FROM numbers(1000) SETTINGS engine_file_truncate_on_insert = 1, output_format_parquet_row_group_size = 100; +INSERT INTO FUNCTION file(currentDatabase() || '/05325/topk/key=9/data.parquet', Parquet, 'v Int64') SELECT number AS v FROM numbers(1000) SETTINGS engine_file_truncate_on_insert = 1, output_format_parquet_row_group_size = 100; +INSERT INTO FUNCTION file(currentDatabase() || '/05325/only/a=1/key=9/data.parquet', Parquet, 'key Int64') SELECT 10 FROM numbers(1000) SETTINGS engine_file_truncate_on_insert = 1, output_format_parquet_row_group_size = 100; +INSERT INTO FUNCTION s3(s3_conn, filename = currentDatabase() || '/05325/key=9/data.parquet', format = Parquet, structure = 'key Int64, v Int64') SELECT 10, number FROM numbers(1000) SETTINGS s3_truncate_on_insert = 1, output_format_parquet_row_group_size = 100; + +-- The Iceberg data file is written before `RENAME COLUMN _path TO old_path`, so it stores a column `_path`. +SET allow_insert_into_iceberg = 1; +CREATE TEMPORARY TABLE iceberg_path AS +WITH if(changed, trimBoth(value), 'user_files/') AS user_files_path +SELECT concat( + if(startsWith(user_files_path, '/'), '', (SELECT path FROM system.disks WHERE name = 'default')), + user_files_path, '/', currentDatabase(), '/05325/iceberg/') AS path +FROM system.server_settings WHERE name = 'user_files_path'; +CREATE TABLE t_05325_iceberg (_path String, v Int64) ENGINE = IcebergLocal((SELECT path FROM iceberg_path), 'Parquet'); +INSERT INTO t_05325_iceberg SELECT 'evil', number FROM numbers(1000) SETTINGS output_format_parquet_row_group_size = 100; +ALTER TABLE t_05325_iceberg RENAME COLUMN _path TO old_path; + +-- The row count cache ignores an entry registered in the same second the file was written. +SELECT sleep(1) FORMAT Null; + +SET use_hive_partitioning = 1, optimize_count_from_files = 0; +SET input_format_parquet_filter_push_down = 1, input_format_parquet_bloom_filter_push_down = 1, input_format_parquet_page_filter_push_down = 1, input_format_parquet_dictionary_filter_push_down = 1048576; + +-- { echoOn } +SELECT key, count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') GROUP BY key; +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9; +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 10; +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9 AND v < 10; +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9 OR v < 10; +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key IN (9); +SELECT count(), sum(v) FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet) WHERE key = 9; +SELECT count() FROM file(currentDatabase() || '/05325/nullable/key=9/data.parquet', Parquet, 'key Nullable(Int64), v Int64') WHERE key IS NOT NULL; +SELECT count() FROM file(currentDatabase() || '/05325/only/a=1/key=9/data.parquet', Parquet, 'key Int64') WHERE indexHint(key = 9); +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE indexHint(key = 9); +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE indexHint(key = 9) AND v < 1000; + +CREATE TABLE t_05325_s3 (key Int64, v Int64) ENGINE = S3(s3_conn, filename = currentDatabase() || '/05325/key=9/data.parquet', format = Parquet); +SELECT count() FROM t_05325_s3 WHERE key = 9 AND v < 10 SETTINGS use_query_condition_cache = 1, log_comment = '05325_qcc_1'; +SELECT count() FROM t_05325_s3 WHERE key = 9 AND v < 10 SETTINGS use_query_condition_cache = 1, log_comment = '05325_qcc_2'; +DROP TABLE t_05325_s3; +SELECT count() FROM url('http://localhost:11111/test/' || currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9; + +SELECT _file, count() FROM file(currentDatabase() || '/05325/virtual/data.parquet', Parquet, 'v Int64') GROUP BY _file; +SELECT count() FROM file(currentDatabase() || '/05325/virtual/data.parquet', Parquet, 'v Int64') WHERE _file = 'data.parquet'; +SELECT count() FROM file(currentDatabase() || '/05325/virtual/data.parquet', Parquet, 'v Int64') WHERE indexHint(_file = 'data.parquet'); + +CREATE TABLE t_05325_file AS file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64'); +CREATE ROW POLICY p_05325_file ON t_05325_file USING key = 9 TO ALL; +SELECT count() FROM t_05325_file; +DROP ROW POLICY p_05325_file ON t_05325_file; +DROP TABLE t_05325_file; + +SELECT count(), any(old_path) FROM t_05325_iceberg; +SELECT count() FROM t_05325_iceberg WHERE _path != 'evil'; +DROP TABLE t_05325_iceberg; + +-- The format still prunes row groups on the columns of the data file. +SELECT count() FROM file(currentDatabase() || '/05325/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9 AND v < 10 SETTINGS log_comment = '05325_prune'; +SYSTEM FLUSH LOGS query_log; +SELECT ProfileEvents['ParquetPrunedRowGroups'] > 0 FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '05325_prune' AND type = 'QueryFinish'; +SELECT sum(ProfileEvents['QueryConditionCacheHits'] + ProfileEvents['QueryConditionCacheMisses']) > 0 FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '05325_qcc_2' AND type = 'QueryFinish'; + +-- A PREWHERE applied by the format does not let the source cache the filtered row count as the file's row count. +SELECT count(), sum(v) FROM file(currentDatabase() || '/05325/count/key=9/data.parquet', Parquet, 'key Int64, v Int64') PREWHERE v < 100 + 0 * rand() WHERE key = 9; +SELECT count() FROM file(currentDatabase() || '/05325/count/key=9/data.parquet', Parquet, 'key Int64, v Int64') SETTINGS optimize_count_from_files = 1, use_cache_for_count_from_files = 1, optimize_trivial_count_query = 1; +SELECT count() FROM file(currentDatabase() || '/05325/count/key=9/data.parquet', Parquet, 'key Int64, v Int64') PREWHERE v < 100 + 0 * rand() WHERE key = 9; + +-- Neither does a join runtime filter nor the top-K filter applied by the format. +SELECT count() > 0 FROM (EXPLAIN actions = 1, pretty = 0 SELECT count() FROM file(currentDatabase() || '/05325/join/key=9/data.parquet', Parquet, 'key Int64, v Int64') AS f INNER JOIN (SELECT toInt64(number) AS v FROM numbers(10)) AS t ON f.v = t.v WHERE f.key = 9 SETTINGS enable_join_runtime_filters = 1, join_runtime_filter_min_probe_rows = 0, join_algorithm = 'hash,parallel_hash', query_plan_join_swap_table = 0, optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1, query_plan_max_step_description_length = 1000) WHERE explain ILIKE '%prewhere filter column: %__applyfilter(%'; +SELECT count() FROM file(currentDatabase() || '/05325/join/key=9/data.parquet', Parquet, 'key Int64, v Int64') AS f INNER JOIN (SELECT toInt64(number) AS v FROM numbers(10)) AS t ON f.v = t.v WHERE f.key = 9 SETTINGS enable_join_runtime_filters = 1, join_runtime_filter_min_probe_rows = 0, join_algorithm = 'hash,parallel_hash', query_plan_join_swap_table = 0, optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1; +SELECT count() FROM file(currentDatabase() || '/05325/join/key=9/data.parquet', Parquet, 'key Int64, v Int64') SETTINGS optimize_count_from_files = 1, use_cache_for_count_from_files = 1, optimize_trivial_count_query = 1; +SELECT v FROM file(currentDatabase() || '/05325/topk/key=9/data.parquet', Parquet, 'key Int64, v Int64') WHERE key = 9 ORDER BY v LIMIT 3 SETTINGS use_top_k_dynamic_filtering = 1, query_plan_max_limit_for_top_k_optimization = 1000, input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_filter_push_down = 1, max_threads = 1, max_parsing_threads = 1; +SELECT count() FROM file(currentDatabase() || '/05325/topk/key=9/data.parquet', Parquet, 'key Int64, v Int64') SETTINGS optimize_count_from_files = 1, use_cache_for_count_from_files = 1, optimize_trivial_count_query = 1; From adb580b1beee9be24cb0ef3e2613998d24687001 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 16:53:56 +0000 Subject: [PATCH 269/296] Update autogenerated version to 26.8.25.14 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 1c632249d671a..ac9b215343d89 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54537) +SET(VERSION_REVISION 54538) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 25) -SET(VERSION_GITHASH 575872da1f0ecad95f1cb3fc07be77772d4fe4c1) -SET(VERSION_DESCRIBE v26.8.25.1-lts) -SET(VERSION_STRING 26.8.25.1) +SET(VERSION_PATCH 26) +SET(VERSION_GITHASH fe4c23110e19371e9275dc2a8d535d7208b4e53e) +SET(VERSION_DESCRIBE v26.8.26.1-lts) +SET(VERSION_STRING 26.8.26.1) # end of autochange From f09d0ce3584ea6de5431dfdd949d507a61c30f96 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 17:02:49 +0000 Subject: [PATCH 270/296] Backport #124439 to 26.8: Add use_aggregation_memory_tracker setting to gate 26.8 arrayCount lambda regression --- src/Core/Settings.cpp | 3 ++ src/Core/SettingsChangesHistory.cpp | 1 + src/Interpreters/Aggregator.cpp | 8 +++-- src/Interpreters/Aggregator.h | 5 ++- src/Interpreters/InterpreterSelectQuery.cpp | 4 ++- ...st_aggregator_parallel_partition_merge.cpp | 3 +- src/Planner/Planner.cpp | 4 ++- src/Processors/QueryPlan/AggregatingStep.cpp | 9 ++++- .../Sources/LazyReadReplacingFinalSource.cpp | 4 ++- .../TTL/TTLAggregationAlgorithm.cpp | 3 +- ..._two_level_decision_query_memory.reference | 2 ++ ...gation_two_level_decision_query_memory.sql | 34 +++++++++++++++++++ 12 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05326_aggregation_two_level_decision_query_memory.reference create mode 100644 tests/queries/0_stateless/05326_aggregation_two_level_decision_query_memory.sql diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 6e8a0616fa0b6..5a9e7e0a46242 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -1280,6 +1280,9 @@ From what number of keys, a two-level aggregation starts. 0 - the threshold is n )", 0) \ DECLARE(UInt64, group_by_two_level_threshold_bytes, 50000000, R"( From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered. +)", 0) \ + DECLARE(Bool, use_aggregation_memory_tracker, false, R"( +Measure the size of the aggregation state compared with `group_by_two_level_threshold_bytes` with the memory tracker dedicated to the aggregation. When disabled, the size is measured as the growth of the memory usage of the whole query since the aggregation was created, so an aggregation with a small state can be converted to two-level. )", 0) \ DECLARE(Bool, distributed_aggregation_memory_efficient, true, R"( Is the memory-saving mode of distributed aggregation enabled. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a33162430ed93..28c8ee4969fce 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -47,6 +47,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.8", { + {"use_aggregation_memory_tracker", false, false, "New setting. In this release it is disabled by default, which restores the two-level conversion of `GROUP BY` from before 26.7 (https://github.com/ClickHouse/ClickHouse/pull/107490)."}, {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.8 restores the previous strict behavior."}, {"analyzer_compatibility_allow_cte_redefinition", false, false, "New compatibility setting. When enabled, the analyzer accepts a CTE name defined more than once in a single `WITH` clause and lets a later definition shadow the earlier ones, as the query analysis before v24.3 did."}, {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index b526a8ac129d6..c41978e614779 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -416,7 +416,8 @@ Aggregator::Params::Params( bool enable_packed_string_keys_, bool enable_adaptive_aggregator_, UInt64 adaptive_aggregator_freeze_threshold_, - UInt64 adaptive_aggregator_freeze_threshold_bytes_) + UInt64 adaptive_aggregator_freeze_threshold_bytes_, + bool use_aggregation_memory_tracker_) : keys(keys_) , keys_size(keys.size()) , aggregates(aggregates_) @@ -442,6 +443,7 @@ Aggregator::Params::Params( , enable_adaptive_aggregator(enable_adaptive_aggregator_) , adaptive_aggregator_freeze_threshold(adaptive_aggregator_freeze_threshold_) , adaptive_aggregator_freeze_threshold_bytes(adaptive_aggregator_freeze_threshold_bytes_) + , use_aggregation_memory_tracker(use_aggregation_memory_tracker_) , enable_producing_buckets_out_of_order_in_aggregation(enable_producing_buckets_out_of_order_in_aggregation_) , enable_parallel_single_level_merge(enable_parallel_single_level_merge_) , serialize_string_with_zero_byte(serialize_string_with_zero_byte_) @@ -2380,7 +2382,9 @@ bool Aggregator::executeOnBlock(Columns columns, Int64 current_memory_usage = getCurrentQueryMemoryUsage(); /// Here all the results in the sum are taken into account, from different threads. - Int64 result_size_bytes = use_own_tracker ? memory_tracker->get() : current_memory_usage - memory_usage_before_aggregation; + Int64 result_size_bytes = use_own_tracker && params.use_aggregation_memory_tracker + ? memory_tracker->get() + : current_memory_usage - memory_usage_before_aggregation; if (adaptive && !adaptive->isBaseline()) { diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 48c2e5c8a9f26..f12c6acbe42cf 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -153,6 +153,8 @@ class Aggregator final UInt64 adaptive_aggregator_freeze_threshold = 0; UInt64 adaptive_aggregator_freeze_threshold_bytes = 0; + bool use_aggregation_memory_tracker = true; + /// Bucket-local Top-K of the final conversion, set by the `aggregation_bucket_top_k` /// plan optimization (never by users) when the plan proves this aggregation feeds /// `ORDER BY LIMIT n`: each two-level bucket materializes @@ -234,7 +236,8 @@ class Aggregator final bool enable_packed_string_keys_, bool enable_adaptive_aggregator_, UInt64 adaptive_aggregator_freeze_threshold_, - UInt64 adaptive_aggregator_freeze_threshold_bytes_); + UInt64 adaptive_aggregator_freeze_threshold_bytes_, + bool use_aggregation_memory_tracker_); /// Only parameters that matter during merge. Params( diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f3589d75b5b19..df906bbd51b04 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -189,6 +189,7 @@ namespace Setting extern const SettingsBool enable_adaptive_aggregator; extern const SettingsUInt64 adaptive_aggregator_freeze_threshold; extern const SettingsUInt64 adaptive_aggregator_freeze_threshold_bytes; + extern const SettingsBool use_aggregation_memory_tracker; extern const SettingsBool optimize_move_to_prewhere; extern const SettingsBool optimize_move_to_prewhere_if_final; extern const SettingsBool optimize_uniq_to_count; @@ -3158,7 +3159,8 @@ static Aggregator::Params getAggregatorParams( settings[Setting::enable_packed_string_keys_in_aggregation], settings[Setting::enable_adaptive_aggregator], settings[Setting::adaptive_aggregator_freeze_threshold], - settings[Setting::adaptive_aggregator_freeze_threshold_bytes]}; + settings[Setting::adaptive_aggregator_freeze_threshold_bytes], + settings[Setting::use_aggregation_memory_tracker]}; } void InterpreterSelectQuery::executeAggregation( diff --git a/src/Interpreters/tests/gtest_aggregator_parallel_partition_merge.cpp b/src/Interpreters/tests/gtest_aggregator_parallel_partition_merge.cpp index 3ee4abc7736e2..07a7b0cdfe517 100644 --- a/src/Interpreters/tests/gtest_aggregator_parallel_partition_merge.cpp +++ b/src/Interpreters/tests/gtest_aggregator_parallel_partition_merge.cpp @@ -74,7 +74,8 @@ Aggregator::Params makeParams(const Names & keys, const AggregateDescriptions & /*enable_packed_string_keys_=*/true, /*enable_adaptive_aggregator_=*/false, /*adaptive_aggregator_freeze_threshold_=*/0, - /*adaptive_aggregator_freeze_threshold_bytes_=*/0); + /*adaptive_aggregator_freeze_threshold_bytes_=*/0, + /*use_aggregation_memory_tracker_=*/true); } /// Renders one finalized chunk as "key[,key]" -> "value[,value]" rows. diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 246a63ac50012..efd05807a1bd9 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -174,6 +174,7 @@ namespace Setting extern const SettingsBool enable_adaptive_aggregator; extern const SettingsUInt64 adaptive_aggregator_freeze_threshold; extern const SettingsUInt64 adaptive_aggregator_freeze_threshold_bytes; + extern const SettingsBool use_aggregation_memory_tracker; extern const SettingsUInt64 max_bytes_to_transfer; extern const SettingsUInt64 max_rows_to_transfer; extern const SettingsOverflowMode transfer_overflow_mode; @@ -704,7 +705,8 @@ Aggregator::Params getAggregatorParams(const PlannerContextPtr & planner_context settings[Setting::enable_packed_string_keys_in_aggregation], settings[Setting::enable_adaptive_aggregator], settings[Setting::adaptive_aggregator_freeze_threshold], - settings[Setting::adaptive_aggregator_freeze_threshold_bytes]); + settings[Setting::adaptive_aggregator_freeze_threshold_bytes], + settings[Setting::use_aggregation_memory_tracker]); return aggregator_params; } diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 08fe3c68f92fd..1eb0d6d97b01b 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,11 @@ namespace QueryPlanSerializationSetting extern const QueryPlanSerializationSettingsBool enable_packed_string_keys_in_aggregation; } +namespace Setting +{ + extern const SettingsBool use_aggregation_memory_tracker; +} + namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -1366,7 +1372,8 @@ QueryPlanStepPtr AggregatingStep::deserialize(Deserialization & ctx) ctx.settings[QueryPlanSerializationSetting::enable_packed_string_keys_in_aggregation], ctx.settings[QueryPlanSerializationSetting::enable_adaptive_aggregator], ctx.settings[QueryPlanSerializationSetting::adaptive_aggregator_freeze_threshold], - ctx.settings[QueryPlanSerializationSetting::adaptive_aggregator_freeze_threshold_bytes]}; + ctx.settings[QueryPlanSerializationSetting::adaptive_aggregator_freeze_threshold_bytes], + ctx.context->getSettingsRef()[Setting::use_aggregation_memory_tracker]}; auto aggregating_step = std::make_unique( ctx.input_headers.front(), diff --git a/src/Processors/Sources/LazyReadReplacingFinalSource.cpp b/src/Processors/Sources/LazyReadReplacingFinalSource.cpp index 0f35284741284..cd4f8c180cd8a 100644 --- a/src/Processors/Sources/LazyReadReplacingFinalSource.cpp +++ b/src/Processors/Sources/LazyReadReplacingFinalSource.cpp @@ -28,6 +28,7 @@ namespace Setting { extern const SettingsUInt64 adaptive_aggregator_freeze_threshold; extern const SettingsUInt64 adaptive_aggregator_freeze_threshold_bytes; +extern const SettingsBool use_aggregation_memory_tracker; extern const SettingsUInt64 aggregation_memory_efficient_merge_threads; extern const SettingsBool collect_hash_table_stats_during_aggregation; extern const SettingsBool enable_adaptive_aggregator; @@ -288,7 +289,8 @@ QueryPlan LazyReadReplacingFinalSource::buildPlanFromReadingStep( /*enable_packed_string_keys_=*/settings[Setting::enable_packed_string_keys_in_aggregation], /*enable_adaptive_aggregator_=*/settings[Setting::enable_adaptive_aggregator], /*adaptive_aggregator_freeze_threshold_=*/settings[Setting::adaptive_aggregator_freeze_threshold], - /*adaptive_aggregator_freeze_threshold_bytes_=*/settings[Setting::adaptive_aggregator_freeze_threshold_bytes]); + /*adaptive_aggregator_freeze_threshold_bytes_=*/settings[Setting::adaptive_aggregator_freeze_threshold_bytes], + /*use_aggregation_memory_tracker_=*/settings[Setting::use_aggregation_memory_tracker]); auto merge_threads = settings[Setting::max_threads]; auto temporary_data_merge_threads = settings[Setting::aggregation_memory_efficient_merge_threads] diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index c28834e920a42..be161ca394212 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -125,7 +125,8 @@ TTLAggregationAlgorithm::TTLAggregationAlgorithm( settings[Setting::enable_packed_string_keys_in_aggregation], /* enable_adaptive_aggregator */ false, /* adaptive_aggregator_freeze_threshold */ 0, - /* adaptive_aggregator_freeze_threshold_bytes */ 0); + /* adaptive_aggregator_freeze_threshold_bytes */ 0, + /* use_aggregation_memory_tracker */ true); aggregator = std::make_unique(header, params); diff --git a/tests/queries/0_stateless/05326_aggregation_two_level_decision_query_memory.reference b/tests/queries/0_stateless/05326_aggregation_two_level_decision_query_memory.reference new file mode 100644 index 0000000000000..64c0b7b6efe8d --- /dev/null +++ b/tests/queries/0_stateless/05326_aggregation_two_level_decision_query_memory.reference @@ -0,0 +1,2 @@ +aggregation_memory_tracker 0 +query_memory 1 diff --git a/tests/queries/0_stateless/05326_aggregation_two_level_decision_query_memory.sql b/tests/queries/0_stateless/05326_aggregation_two_level_decision_query_memory.sql new file mode 100644 index 0000000000000..06f0ad1bb1849 --- /dev/null +++ b/tests/queries/0_stateless/05326_aggregation_two_level_decision_query_memory.sql @@ -0,0 +1,34 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/123756: with `use_aggregation_memory_tracker = 0` the two-level conversion of `GROUP BY` also counts the memory held by the rest of the query. + +SET max_threads = 1; +SET max_untracked_memory = 0; +SET max_bytes_before_external_group_by = 1000000000; +SET max_bytes_ratio_before_external_group_by = 0; +SET group_by_two_level_threshold = 100000; +SET group_by_two_level_threshold_bytes = 100000; + +SELECT r.k AS g, groupArray(l.s) +FROM (SELECT number AS id, leftPad(toString(number), 10, '0') AS s FROM numbers(1000)) AS l +JOIN (SELECT number AS id, toFixedString(toString(number % 10), 64) AS k FROM numbers(1000)) AS r +ON l.id = r.id +GROUP BY g +FORMAT Null +SETTINGS log_comment = 'aggregation_memory_tracker', use_aggregation_memory_tracker = 1; + +SELECT r.k AS g, groupArray(l.s) +FROM (SELECT number AS id, leftPad(toString(number), 10, '0') AS s FROM numbers(1000)) AS l +JOIN (SELECT number AS id, toFixedString(toString(number % 10), 64) AS k FROM numbers(1000)) AS r +ON l.id = r.id +GROUP BY g +FORMAT Null +SETTINGS log_comment = 'query_memory', use_aggregation_memory_tracker = 0; + +SYSTEM FLUSH LOGS query_log; + +SELECT log_comment, ProfileEvents['AggregationConvertedToTwoLevel'] > 0 AS two_level +FROM system.query_log +WHERE log_comment IN ('aggregation_memory_tracker', 'query_memory') + AND type = 'QueryFinish' + AND current_database = currentDatabase() +ORDER BY log_comment +SETTINGS max_rows_to_read = 0; From aff3ad133194ee9a21f6d579d3d2620e0e23e388 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 18:49:26 +0000 Subject: [PATCH 271/296] Backport #123403 to 26.8: Do not let use_skip_indexes = 0 disable part min-max pruning --- .../QueryPlan/ReadFromMergeTree.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 3 +-- ...x_pruning_use_skip_indexes_repro.reference | 4 ++++ ..._minmax_pruning_use_skip_indexes_repro.sql | 23 +++++++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05326_minmax_pruning_use_skip_indexes_repro.reference create mode 100644 tests/queries/0_stateless/05326_minmax_pruning_use_skip_indexes_repro.sql diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 748dfddbb1cf5..38c99f94d19bc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -2767,7 +2767,7 @@ void ReadFromMergeTree::buildIndexes( return KeyCondition{ wrapped, query_context, minmax_columns.getNames(), minmax_expression_actions, /* single_point_ = */ false, - /* skip_analysis_ = */ skip_partition_pruning_ || !query_context->getSettingsRef()[Setting::use_partition_pruning] || !query_context->getSettingsRef()[Setting::use_skip_indexes]}; + /* skip_analysis_ = */ skip_partition_pruning_ || !query_context->getSettingsRef()[Setting::use_partition_pruning]}; }; indexes->minmax_idx_condition = std::make_shared>(filter_dag_ptr, std::move(key_condition_factory), metadata_snapshot, query_context, skip_constant_folding); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ba93e48c657b6..5186618190e73 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -267,7 +267,6 @@ namespace Setting extern const SettingsBool use_statistics_cache; extern const SettingsBool use_partition_pruning; extern const SettingsBool use_constant_folding_in_index_analysis; - extern const SettingsBool use_skip_indexes; } namespace MergeTreeSetting @@ -10728,7 +10727,7 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( wrapped, query_context, minmax_columns.getNames(), MergeTreeData::getMinMaxExpr(metadata_snapshot->getPartitionKey(), data_settings, ExpressionActionsSettings(query_context)), /*single_point=*/false, - /*skip_analysis=*/!query_context->getSettingsRef()[Setting::use_partition_pruning] || !query_context->getSettingsRef()[Setting::use_skip_indexes]}; + /*skip_analysis=*/!query_context->getSettingsRef()[Setting::use_partition_pruning]}; }; auto inverted_dag = std::make_shared(filter_dag->getOutputs().front(), query_context, /* boolean_context */ true); minmax_idx_condition = std::make_shared>(inverted_dag, std::move(key_condition_factory), metadata_snapshot, query_context, /*skip_folding_=*/!query_context->getSettingsRef()[Setting::use_constant_folding_in_index_analysis]); diff --git a/tests/queries/0_stateless/05326_minmax_pruning_use_skip_indexes_repro.reference b/tests/queries/0_stateless/05326_minmax_pruning_use_skip_indexes_repro.reference new file mode 100644 index 0000000000000..44a43c32cbc30 --- /dev/null +++ b/tests/queries/0_stateless/05326_minmax_pruning_use_skip_indexes_repro.reference @@ -0,0 +1,4 @@ +Parts: 1 | Granules: 1 +Parts: 1/30 +Parts: 1/1 +Parts: 1/1 diff --git a/tests/queries/0_stateless/05326_minmax_pruning_use_skip_indexes_repro.sql b/tests/queries/0_stateless/05326_minmax_pruning_use_skip_indexes_repro.sql new file mode 100644 index 0000000000000..33414460428b9 --- /dev/null +++ b/tests/queries/0_stateless/05326_minmax_pruning_use_skip_indexes_repro.sql @@ -0,0 +1,23 @@ +-- Reproducer from https://github.com/ClickHouse/ClickHouse/issues/123363: +-- `use_skip_indexes = 0` must not disable the part-level min-max index on the partition-key columns. + +SET enable_parallel_replicas = 0, use_query_condition_cache = 0; + +DROP TABLE IF EXISTS t; + +CREATE TABLE t (ts DateTime, k UInt64) +ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY k +SETTINGS auto_statistics_types = ''; + +SYSTEM STOP MERGES t; + +-- 720 hourly rows = 30 daily parts; every part contains k = 0..9 +INSERT INTO t SELECT toDateTime('2026-01-01 00:00:00') + number * 3600, number % 10 FROM numbers(720); + +SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t WHERE k = 3 AND ts::Date IN (SELECT toDate('2026-01-05')) + SETTINGS use_skip_indexes = 0 +) WHERE explain LIKE '%Parts:%'; + +DROP TABLE t; From 203381df9440ae3c00216eebe872d0702184970f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 19:10:09 +0000 Subject: [PATCH 272/296] Backport #125090 to 26.8: Fix use-after-free of the read buffer when creating an object storage reader throws --- src/Processors/Formats/IInputFormat.h | 4 +++- .../StorageObjectStorageSource.cpp | 2 +- ...reader_exception_buffer_lifetime.reference | 1 + ...torage_reader_exception_buffer_lifetime.sh | 22 +++++++++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05384_object_storage_reader_exception_buffer_lifetime.reference create mode 100755 tests/queries/0_stateless/05384_object_storage_reader_exception_buffer_lifetime.sh diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index fecd232ad721f..93dc66dd9b4c1 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -122,7 +122,9 @@ class IInputFormat : public ISource /// in custom serializations (e.g. Sparse) for parsing and avoid extra conversion. virtual void setSerializationHints(const SerializationInfoByName & /*hints*/) {} - void addBuffer(std::unique_ptr buffer) { owned_buffers.emplace_back(std::move(buffer)); } + /// Not by value: if the insertion throws, `buffer` must stay with the caller, since this format still reads it. + template + void addBuffer(std::unique_ptr && buffer) { owned_buffers.emplace_back(std::move(buffer)); } void setErrorsLogger(const InputFormatErrorsLoggerPtr & errors_logger_) { errors_logger = errors_logger_; } diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index d6e804216ffb1..5fc14e1df78f5 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1043,9 +1043,9 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade break; } + std::unique_ptr read_buf; /// Destroyed last: `builder` and `source` hold the input format reading it. QueryPipelineBuilder builder; std::shared_ptr source; - std::unique_ptr read_buf; auto try_get_num_rows_from_cache = [&]() -> std::optional { diff --git a/tests/queries/0_stateless/05384_object_storage_reader_exception_buffer_lifetime.reference b/tests/queries/0_stateless/05384_object_storage_reader_exception_buffer_lifetime.reference new file mode 100644 index 0000000000000..d567f8a0b01f3 --- /dev/null +++ b/tests/queries/0_stateless/05384_object_storage_reader_exception_buffer_lifetime.reference @@ -0,0 +1 @@ +4950 diff --git a/tests/queries/0_stateless/05384_object_storage_reader_exception_buffer_lifetime.sh b/tests/queries/0_stateless/05384_object_storage_reader_exception_buffer_lifetime.sh new file mode 100755 index 0000000000000..03038a8d04c27 --- /dev/null +++ b/tests/queries/0_stateless/05384_object_storage_reader_exception_buffer_lifetime.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Depends on S3 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Reading many CSV files from S3 while memory allocations fail at random must not crash the server. +prefix="${CLICKHOUSE_TEST_UNIQUE_NAME}" +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3(s3_conn, filename = '${prefix}/f_{_partition_id}.csv', format = 'CSV', structure = 'x UInt64', partition_strategy = 'wildcard') PARTITION BY x SELECT number FROM numbers(100) SETTINGS s3_truncate_on_insert = 1" + +query="SELECT x FROM s3(s3_conn, filename = '${prefix}/f_*.csv', format = 'CSV', structure = 'x UInt64') FORMAT Null" +start=$SECONDS +for _ in {1..100}; do + $CLICKHOUSE_CLIENT --max_threads=1 --input_format_parallel_parsing=0 --max_untracked_memory=0 \ + --memory_tracker_fault_probability=0.001 -q "$query" >/dev/null 2>&1 ||: + (( SECONDS - start < 60 )) || break +done + +$CLICKHOUSE_CLIENT -q "SELECT sum(x) FROM s3(s3_conn, filename = '${prefix}/f_*.csv', format = 'CSV', structure = 'x UInt64')" +exit 0 From 82170ccfc64399ffbe3c30ac5dd7fde84ae2ef5b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 12 Oct 2026 19:19:09 +0000 Subject: [PATCH 273/296] Backport #125129 to 26.8: Fix INTERPOLATE when a value converted for one target is read by another item --- src/Planner/Planner.cpp | 21 +++++++------ ..._expression_read_by_other_target.reference | 30 +++++++++++++++++++ ...verted_expression_read_by_other_target.sql | 22 ++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05359_interpolate_converted_expression_read_by_other_target.reference create mode 100644 tests/queries/0_stateless/05359_interpolate_converted_expression_read_by_other_target.sql diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 246a63ac50012..c815ba530cdf8 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1374,6 +1374,9 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan, { ActionsDAG rename_dag; + /// A later item resolves names against every node of the DAG, so the conversions and outputs are added after all items are visited. + std::vector> interpolated_columns; + for (auto & interpolate_node : interpolate_list_nodes) { auto & interpolate_node_typed = interpolate_node->as(); @@ -1394,16 +1397,16 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan, const auto * expression_to_interpolate = expression_to_interpolate_expression_nodes[0]; const auto * interpolate_expression = interpolate_expression_nodes[0]; - if (!interpolate_expression->result_type->equals(*expression_to_interpolate->result_type)) - { - interpolate_expression = &interpolate_actions_dag.addCast(*interpolate_expression, - expression_to_interpolate->result_type, - interpolate_expression->result_name, - planner_context->getQueryContext()); - } + interpolated_columns.emplace_back(interpolate_expression, expression_to_interpolate->result_type, interpolate_node_typed.getExpressionName()); + } + + for (const auto & [interpolate_expression, target_type, target_name] : interpolated_columns) + { + const auto * interpolated_column = interpolate_expression; + if (!interpolated_column->result_type->equals(*target_type)) + interpolated_column = &interpolate_actions_dag.addCast(*interpolated_column, target_type, {}, planner_context->getQueryContext()); - const auto * alias_node = &interpolate_actions_dag.addAlias(*interpolate_expression, interpolate_node_typed.getExpressionName()); - interpolate_actions_dag.getOutputs().push_back(alias_node); + interpolate_actions_dag.getOutputs().push_back(&interpolate_actions_dag.addAlias(*interpolated_column, target_name)); } if (!rename_dag.getOutputs().empty()) diff --git a/tests/queries/0_stateless/05359_interpolate_converted_expression_read_by_other_target.reference b/tests/queries/0_stateless/05359_interpolate_converted_expression_read_by_other_target.reference new file mode 100644 index 0000000000000..65f2f7e65d695 --- /dev/null +++ b/tests/queries/0_stateless/05359_interpolate_converted_expression_read_by_other_target.reference @@ -0,0 +1,30 @@ +swap of targets of different types +0 1 300 +1 44 1 +2 2 1300 +3 20 2 +4 3 2300 +swap with a Nullable target +0 1 300 +1 300 1 +2 2 1300 +3 1300 2 +4 3 2300 +swap with a String target +0 1 300 +1 300 1 +2 2 1300 +3 1300 2 +4 3 2300 +a column converted for an earlier target +0 1 300 7 +1 44 0 301 +2 2 1300 7 +3 20 0 1301 +4 3 2300 7 +a literal converted for an earlier target +0 0 0 +1 1 1 +2 1 10 +3 1 11 +4 2 20 diff --git a/tests/queries/0_stateless/05359_interpolate_converted_expression_read_by_other_target.sql b/tests/queries/0_stateless/05359_interpolate_converted_expression_read_by_other_target.sql new file mode 100644 index 0000000000000..955917b04aa9c --- /dev/null +++ b/tests/queries/0_stateless/05359_interpolate_converted_expression_read_by_other_target.sql @@ -0,0 +1,22 @@ +-- An INTERPOLATE expression converted to its target's type must not replace the expression it converts for the other INTERPOLATE items. +-- Every gap has one filled row, so each filled row is computed from a source row. + +SELECT 'swap of targets of different types'; +SELECT g, a, b FROM (SELECT toUInt16(number * 2) AS g, toUInt8(number + 1) AS a, toUInt32(number * 1000 + 300) AS b FROM numbers(3)) +ORDER BY g WITH FILL INTERPOLATE (a AS b, b AS a); + +SELECT 'swap with a Nullable target'; +SELECT g, a, b FROM (SELECT toUInt16(number * 2) AS g, toNullable(toUInt32(number + 1)) AS a, toUInt32(number * 1000 + 300) AS b FROM numbers(3)) +ORDER BY g WITH FILL INTERPOLATE (a AS b, b AS a); + +SELECT 'swap with a String target'; +SELECT g, a, b FROM (SELECT toUInt16(number * 2) AS g, toString(number + 1) AS a, toUInt32(number * 1000 + 300) AS b FROM numbers(3)) +ORDER BY g WITH FILL INTERPOLATE (b AS a, a AS b); + +SELECT 'a column converted for an earlier target'; +SELECT g, a, b, c FROM (SELECT toUInt16(number * 2) AS g, toUInt8(number + 1) AS a, toUInt32(number * 1000 + 300) AS b, toUInt32(7) AS c FROM numbers(3)) +ORDER BY g WITH FILL INTERPOLATE (a AS b, c AS b + 1); + +SELECT 'a literal converted for an earlier target'; +SELECT g, s, y FROM (SELECT toUInt16(number * 2) AS g, toString(number) AS s, toUInt64(number * 10) AS y FROM numbers(3)) +ORDER BY g WITH FILL INTERPOLATE (s AS 1, y AS y + 1); From cacbd830f926aef91b8893c3f6f59a96629e3499 Mon Sep 17 00:00:00 2001 From: Hechem Selmi Date: Mon, 12 Oct 2026 19:56:55 +0000 Subject: [PATCH 274/296] Pin use_aggregation_memory_tracker = 1 in 04326_aggregation_two_level_decision. --- .../queries/0_stateless/04326_aggregation_two_level_decision.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/04326_aggregation_two_level_decision.sql b/tests/queries/0_stateless/04326_aggregation_two_level_decision.sql index 606da91b2f94a..f74634032d6ce 100644 --- a/tests/queries/0_stateless/04326_aggregation_two_level_decision.sql +++ b/tests/queries/0_stateless/04326_aggregation_two_level_decision.sql @@ -9,6 +9,7 @@ SET max_bytes_before_external_group_by = 1000000000; -- Ensure conversion to two SET max_bytes_ratio_before_external_group_by = 0; SET group_by_two_level_threshold = 100000; SET group_by_two_level_threshold_bytes = 100000; +SET use_aggregation_memory_tracker = 1; -- Small aggregate state over join stays single-level. SELECT r.k AS g, groupArray(l.s) From f04e0863f41049b9fc1aca27fd55fd794e4aa2ba Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 00:20:45 +0000 Subject: [PATCH 275/296] Backport #118021 to 26.8: Add an opt-in access control improvement so that readonly restricts SET ROLE --- docs/concepts/features/interfaces/http.mdx | 2 + .../_server_settings_outside_source.mdx | 2 + programs/server/config.xml | 4 + src/Access/AccessControl.cpp | 1 + src/Access/AccessControl.h | 5 + .../SQLDefinedHandlerFromAST.cpp | 22 ++-- .../Access/InterpreterSetRoleQuery.cpp | 16 +++ .../__init__.py | 0 .../configs/readonly_restricts_set_role.xml | 5 + .../readonly_restricts_set_role_absent.xml | 5 + .../test_readonly_restricts_set_role/test.py | 116 ++++++++++++++++++ ...04824_handler_safe_method_session_state.sh | 6 +- 12 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 tests/integration/test_readonly_restricts_set_role/__init__.py create mode 100644 tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml create mode 100644 tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml create mode 100644 tests/integration/test_readonly_restricts_set_role/test.py diff --git a/docs/concepts/features/interfaces/http.mdx b/docs/concepts/features/interfaces/http.mdx index dbadd8c4a7c99..ee87cf7216b68 100644 --- a/docs/concepts/features/interfaces/http.mdx +++ b/docs/concepts/features/interfaces/http.mdx @@ -644,6 +644,8 @@ curl -sS "http://localhost:8123?role=my_role&role=my_other_role" --data-binary " In this case, `?role=my_role&role=my_other_role` works similarly to executing `SET ROLE my_role, my_other_role` before the statement. +The `role` query parameter is not affected by the `access_control_improvements.readonly_restricts_set_role` server setting, which refuses only the `SET ROLE` statement. + ## HTTP response codes caveats {#http_response_codes_caveats} Because of limitations of the HTTP protocol, a HTTP 200 response code doesn't guarantee that a query was successful. diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 23c3fedd9bc7d..71a2cfa7fc796 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1809,6 +1809,7 @@ Settings for optional improvements in the access control system. | Setting | Description | Default | |-------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| | `on_cluster_queries_require_cluster_grant` | Sets whether `ON CLUSTER` queries require the `CLUSTER` grant. | `true` | +| `readonly_restricts_set_role` | Sets whether the `SET ROLE` statement is refused when the session or the query is readonly. When `false`, `readonly` does not restrict `SET ROLE`. The `role` HTTP query parameter is not affected. | `false` | | `role_cache_expiration_time_seconds` | Sets the number of seconds since last access, that a role is stored in the Role Cache. | `600` | | `select_from_information_schema_requires_grant` | Sets whether `SELECT * FROM information_schema.
` requires any grants and can be executed by any user. If set to true, then this query requires `GRANT SELECT ON information_schema.
`, just as for ordinary tables. | `true` | | `select_from_system_db_requires_grant` | Sets whether `SELECT * FROM system.
` requires any grants and can be executed by any user. If set to true then this query requires `GRANT SELECT ON system.
` just as for non-system tables. Exceptions: a few system tables (`tables`, `columns`, `databases`, and some constant tables like `one`, `contributors`) are still accessible for everyone; and if there is a `SHOW` privilege (e.g. `SHOW USERS`) granted then the corresponding system table (i.e. `system.users`) will be accessible. | `true` | @@ -1828,6 +1829,7 @@ Example: true true false + false 600 ``` diff --git a/programs/server/config.xml b/programs/server/config.xml index 40eb34dd922c6..014895d4ec6de 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -902,6 +902,10 @@ however you can change this behaviour by setting this to true --> false + + false + 600 diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 6d67074ceeed8..c869445e76824 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -311,6 +311,7 @@ void AccessControl::setupFromMainConfig(const Poco::Util::AbstractConfiguration setTableEnginesRequireGrant(config_.getBool("access_control_improvements.table_engines_require_grant", false)); setEnableReadWriteGrants(config_.getBool("access_control_improvements.enable_read_write_grants", false)); setThrowOnUnmatchedRowPolicies(config_.getBool("access_control_improvements.throw_on_unmatched_row_policies", false)); + setReadonlyRestrictsSetRole(config_.getBool("access_control_improvements.readonly_restricts_set_role", false)); /// Set `true` by default because the feature is backward incompatible only when older version replicas are in the same cluster. setEnableUserNameAccessType(config_.getBool("access_control_improvements.enable_user_name_access_type", true)); diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index 2bd9eb104fb8f..ae303fbf116c5 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -217,6 +217,10 @@ class AccessControl : public MultipleAccessStorage void setThrowOnInvalidReplicatedAccessEntities(bool enable) { throw_on_invalid_replicated_access_entities = enable; } bool shouldThrowOnInvalidReplicatedAccessEntities() const { return throw_on_invalid_replicated_access_entities; } + /// Sets whether `SET ROLE` is refused when the session or the query is readonly. + void setReadonlyRestrictsSetRole(bool enable) { readonly_restricts_set_role = enable; } + bool doesReadonlyRestrictSetRole() const { return readonly_restricts_set_role; } + std::shared_ptr getContextAccess(const ContextAccessParams & params) const; std::shared_ptr getEnabledRoles( @@ -314,6 +318,7 @@ class AccessControl : public MultipleAccessStorage std::atomic_bool enable_user_name_access_type = true; std::atomic_bool enable_read_write_grants = false; std::atomic_bool allow_impersonate_user = false; + std::atomic_bool readonly_restricts_set_role = false; }; } diff --git a/src/Common/SQLDefinedHandlers/SQLDefinedHandlerFromAST.cpp b/src/Common/SQLDefinedHandlers/SQLDefinedHandlerFromAST.cpp index 2a4da26ddfcbb..41485ef1c4e82 100644 --- a/src/Common/SQLDefinedHandlers/SQLDefinedHandlerFromAST.cpp +++ b/src/Common/SQLDefinedHandlers/SQLDefinedHandlerFromAST.cpp @@ -100,7 +100,10 @@ bool queryKindRequiresMutatingMethod(IAST::QueryKind kind) case IAST::QueryKind::Check: /// Session- and transaction-mutating statements (`USE`, `SET` / `SET ROLE`, `BEGIN` / `COMMIT` / `ROLLBACK` / /// `SET TRANSACTION SNAPSHOT`) run under `readonly = 2` - the mode the HTTP execution path sets for safe - /// methods such as `GET` - so this readonly-mirror predicate reports them as runnable over a safe method. + /// methods such as `GET` - apart from `SET ROLE` when + /// `access_control_improvements.readonly_restricts_set_role` is enabled. `QueryKind::Set` does not + /// distinguish `SET ROLE` from plain `SET`, so this readonly-mirror predicate reports the kind as + /// runnable over a safe method either way. /// Their session-visible side effects are fenced off separately by `queryKindHasSideEffectsUnderReadonly`, /// which requires *every* method of such a handler to be a mutating one. case IAST::QueryKind::Use: @@ -223,11 +226,12 @@ bool queryWrapsBodyConsumingStatement(const IAST & query) /// - `BACKUP` writes an archive to disk or object storage and `RESTORE` writes data into tables, yet /// `BackupsWorker` rejects them only under the strict, user-set `readonly = 1`. /// - Session- and transaction-mutating statements: `SET` changes session settings (allowed under `readonly = 2`, -/// which forbids only changing `readonly` itself), `SET ROLE` changes the active roles, `USE` changes the +/// which forbids only changing `readonly` itself), `SET ROLE` changes the active roles (which `readonly` blocks +/// only when `access_control_improvements.readonly_restricts_set_role` is enabled), `USE` changes the /// session database, and `BEGIN` / `COMMIT` / `ROLLBACK` / `SET TRANSACTION SNAPSHOT` mutate the current -/// transaction - none of which `readonly` blocks. When the client uses `session_id`, these effects persist +/// transaction - none of which `readonly` blocks by default. When the client uses `session_id`, these effects persist /// across requests, so a `GET` could invisibly commit a transaction or alter session state. -/// The runtime `readonly` enforcement cannot fence any of these off. HTTP requires safe methods to be +/// The runtime `readonly` enforcement cannot be relied on to fence these off. HTTP requires safe methods to be /// side-effect-free: `GET` is expected to have no effects, and a handler declared for `GET` is also served for /// `HEAD` (see `HTTPHandlerFactory`), where the suppressed response body would hide the effect entirely. Such /// queries therefore must not be reachable over safe methods at all. @@ -471,10 +475,10 @@ SQLDefinedHandlerPtr makeSQLDefinedHandler(const ASTCreateHandlerQuery & create) create.handler_name, fmt::join(handler->methods, ", ")); } - /// The `readonly` enforcement above cannot fence off queries whose side effects survive `readonly = 2` - the - /// mode that safe methods set: `BACKUP` / `RESTORE` write durable data, and `SET` / `SET ROLE` / `USE` / - /// transaction control / `CREATE TEMPORARY TABLE` / `CREATE TEMPORARY VIEW` / an `INSERT`, `DROP` or - /// `TRUNCATE` that may target an existing temporary table mutate session state that + /// The `readonly` enforcement above cannot be relied on to fence off queries whose side effects survive + /// `readonly = 2` - the mode that safe methods set: `BACKUP` / `RESTORE` write durable data, and `SET` / + /// `SET ROLE` / `USE` / transaction control / `CREATE TEMPORARY TABLE` / `CREATE TEMPORARY VIEW` / an + /// `INSERT`, `DROP` or `TRUNCATE` that may target an existing temporary table mutate session state that /// persists across requests when `session_id` is in use. A safe method must never trigger them - `GET` is /// expected to be side-effect-free, and a declared `GET` is also served for `HEAD` (see `HTTPHandlerFactory`), /// where the suppressed response body would hide the effect entirely. So require *every* allowed method of @@ -483,7 +487,7 @@ SQLDefinedHandlerPtr makeSQLDefinedHandler(const ASTCreateHandlerQuery & create) && !std::all_of(handler->methods.begin(), handler->methods.end(), isMutatingHTTPMethod)) { throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Handler `{}` runs {}. The read-only mode of safe HTTP methods does not prevent these side effects, " + "Handler `{}` runs {}. The read-only mode of safe HTTP methods cannot be relied on to prevent these side effects, " "but the handler's allowed HTTP methods ({}) include read-only ones. " "List only mutating methods (POST, PUT, or DELETE) in the METHODS clause.", create.handler_name, diff --git a/src/Interpreters/Access/InterpreterSetRoleQuery.cpp b/src/Interpreters/Access/InterpreterSetRoleQuery.cpp index 016f8a70214ef..b68fbd19f7046 100644 --- a/src/Interpreters/Access/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/Access/InterpreterSetRoleQuery.cpp @@ -5,13 +5,20 @@ #include #include #include +#include #include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 readonly; +} + namespace ErrorCodes { + extern const int READONLY; extern const int SET_NON_GRANTED_ROLE; } @@ -31,6 +38,15 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) { auto session_context = getContext()->getSessionContext(); + /// `readonly` is read from the query context: a client `--readonly 1` raises it there only, while a + /// mid-session `SET readonly = 1` also reaches it because a query context inherits session settings. + if (getContext()->getAccessControl().doesReadonlyRestrictSetRole() + && getContext()->getSettingsRef()[Setting::readonly]) + throw Exception( + ErrorCodes::READONLY, + "Cannot execute SET ROLE in readonly mode. " + "This restriction is enabled by access_control_improvements.readonly_restricts_set_role"); + if (query.kind == ASTSetRoleQuery::Kind::SET_ROLE_DEFAULT) session_context->setCurrentRolesDefault(); else diff --git a/tests/integration/test_readonly_restricts_set_role/__init__.py b/tests/integration/test_readonly_restricts_set_role/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml new file mode 100644 index 0000000000000..80d7145131cdf --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml @@ -0,0 +1,5 @@ + + + true + + diff --git a/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml new file mode 100644 index 0000000000000..76b2225c37b3e --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/tests/integration/test_readonly_restricts_set_role/test.py b/tests/integration/test_readonly_restricts_set_role/test.py new file mode 100644 index 0000000000000..c36af86fa7203 --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/test.py @@ -0,0 +1,116 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# Every instance gets a copy of the production `config.xml`, which sets the element explicitly, so +# `node_off` removes it to run at the compatibility default an upgraded deployment has. +node_on = cluster.add_instance( + "node_on", main_configs=["configs/readonly_restricts_set_role.xml"] +) +node_off = cluster.add_instance( + "node_off", main_configs=["configs/readonly_restricts_set_role_absent.xml"] +) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + for node in (node_on, node_off): + node.query( + "CREATE ROLE ro_role, other_role;" + "CREATE USER probe IDENTIFIED WITH plaintext_password BY 'x';" + "GRANT ro_role, other_role TO probe;" + "ALTER USER probe DEFAULT ROLE ro_role;" + ) + + yield cluster + + finally: + cluster.shutdown() + + +def as_probe(node, sql, readonly): + return node.query_and_get_answer_with_error( + sql, user="probe", password="x", settings={"readonly": readonly} + ) + + +@pytest.mark.parametrize( + "readonly, statement", + [ + (1, "SET ROLE other_role"), + # The other `ASTSetRoleQuery::Kind` reaching the check. + (1, "SET ROLE DEFAULT"), + # `readonly = 2` is what the HTTP execution path sets for safe methods. + (2, "SET ROLE other_role"), + ], +) +def test_set_role_refused_when_enabled(readonly, statement): + _, error = as_probe(node_on, statement, readonly) + assert "READONLY" in error + + +def test_set_role_allowed_without_readonly(): + answer, error = as_probe(node_on, "SET ROLE other_role; SELECT currentRoles()", 0) + assert error == "" + assert answer.strip() == "['other_role']" + + +def test_set_role_allowed_at_default_settings(): + answer, error = as_probe(node_off, "SET ROLE other_role; SELECT currentRoles()", 1) + assert error == "" + assert answer.strip() == "['other_role']" + + +def in_session(node, session_id): + # Every request is a POST so the HTTP method contributes no `readonly` of its own + # (`setReadOnlyIfHTTPMethodIdempotent` exempts POST), leaving the session's own value as the + # only source. Reading the roles back needs the same session, which is why this is HTTP. + def post(sql): + return node.http_query_and_get_answer_with_error( + sql, method="POST", params={"session_id": session_id}, user="probe", password="x" + ) + + assert post("SET readonly = 1")[1] is None + _, error = post("SET ROLE other_role") + roles, roles_error = post("SELECT currentRoles()") + assert roles_error is None + return error, roles.strip() + + +def test_set_role_refused_mid_session(): + error, roles = in_session(node_on, "refused_mid_session") + assert error and "READONLY" in error + assert roles == "['ro_role']" + + +def test_set_role_allowed_mid_session_at_default_settings(): + error, roles = in_session(node_off, "allowed_mid_session") + assert error is None + assert roles == "['other_role']" + + +def test_http_role_parameter_not_restricted_when_enabled(): + # The `role` URL parameter reaches `Context::setCurrentRoles` without passing through + # `InterpreterSetRoleQuery`, so the restriction does not cover it. + def post(sql, **extra): + return node_on.http_query_and_get_answer_with_error( + sql, + method="POST", + params={"session_id": "http_role_parameter", **extra}, + user="probe", + password="x", + ) + + assert post("SET readonly = 1")[1] is None + answer, error = post("SELECT currentRoles()", role="other_role") + assert error is None + assert answer.strip() == "['other_role']" + # The parameter applies to one request, so the session keeps its default role. + answer, error = post("SELECT currentRoles()") + assert error is None + assert answer.strip() == "['ro_role']" diff --git a/tests/queries/0_stateless/04824_handler_safe_method_session_state.sh b/tests/queries/0_stateless/04824_handler_safe_method_session_state.sh index 5d3ebaf796bda..6fdfccbcebf05 100755 --- a/tests/queries/0_stateless/04824_handler_safe_method_session_state.sh +++ b/tests/queries/0_stateless/04824_handler_safe_method_session_state.sh @@ -6,9 +6,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Session-mutating statements (`SET`, `SET ROLE`, `USE`, `BEGIN` / `COMMIT` / `ROLLBACK` / # `SET TRANSACTION SNAPSHOT`) run under the `readonly = 2` mode that the HTTP execution path sets for safe -# methods, so the runtime `readonly` enforcement cannot fence them off. But they change session or transaction -# state that persists across requests when `session_id` is in use, and HTTP requires safe methods to be -# side-effect-free: a handler declared for GET is also served for HEAD, where the suppressed response body +# methods, so the runtime `readonly` enforcement cannot be relied on to fence them off. But they change session +# or transaction state that persists across requests when `session_id` is in use, and HTTP requires safe methods +# to be side-effect-free: a handler declared for GET is also served for HEAD, where the suppressed response body # would hide the effect entirely. So such a handler must list only mutating methods (POST, PUT, DELETE) - # both at CREATE and at ALTER time. From d6526231e990db90527bee2f24d520508ac59480 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 02:06:59 +0000 Subject: [PATCH 276/296] Backport #117238 to 26.8: Stop claiming injectivity for `toString` of a folding date-time and `toFixedString` of a `String` --- .../UniqInjectiveFunctionsEliminationPass.cpp | 4 +- src/Functions/FunctionsConversion.h | 53 +++++- src/Functions/concat.cpp | 6 +- src/Functions/toFixedString.h | 35 +++- .../RemoveInjectiveFunctionsVisitor.cpp | 15 +- .../RemoveInjectiveFunctionsVisitor.h | 8 +- src/Interpreters/TreeOptimizer.cpp | 19 ++- .../getASTFunctionArgumentColumns.cpp | 72 ++++++++ .../getASTFunctionArgumentColumns.h | 36 ++++ .../Optimizations/actionsDAGUtils.cpp | 11 +- .../QueryPlan/Optimizations/actionsDAGUtils.h | 5 + .../convertOuterJoinToInnerJoin.cpp | 4 +- .../Optimizations/optimizeReadInOrder.cpp | 2 +- ...tring_datetime_and_tofixedstring.reference | 68 ++++++++ ...ms_tostring_datetime_and_tofixedstring.sql | 159 ++++++++++++++++++ 15 files changed, 479 insertions(+), 18 deletions(-) create mode 100644 src/Interpreters/getASTFunctionArgumentColumns.cpp create mode 100644 src/Interpreters/getASTFunctionArgumentColumns.h create mode 100644 tests/queries/0_stateless/05227_injective_claims_tostring_datetime_and_tofixedstring.reference create mode 100644 tests/queries/0_stateless/05227_injective_claims_tostring_datetime_and_tofixedstring.sql diff --git a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp index 6717f98c0b7e3..6067598db3125 100644 --- a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp +++ b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp @@ -65,7 +65,9 @@ class UniqInjectiveFunctionsEliminationVisitor : public InDepthQueryTreeVisitorW return false; const auto & arg_function = arg_typed->getFunction(); - if (!arg_function->isInjective({})) + /// Pass the argument types: whether a function is injective can depend on them, as it + /// does for `toString` of a date-time in a time zone with a fall-back transition. + if (!arg_function->isInjective(arg_typed->getArgumentColumns())) return false; /// The `Null` combinator makes `uniq*` skip rows where a Nullable argument is NULL: `uniq(tuple(x))` diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 913b78ffaa5d5..4534a4915990c 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -3148,6 +3148,42 @@ bool convertIsCompilableImpl(const DataTypes & types, const DataTypePtr & result llvm::Value * convertCompileImpl(llvm::IRBuilderBase & builder, const ValuesWithType & arguments, const DataTypePtr & result_type); #endif +/// Whether rendering a value of this type as text maps distinct values onto the same string: +/// +/// - a date-time in a time zone with a UTC offset transition: in a fall-back hour two distinct +/// instants have the same local wall-clock representation; +/// - a floating-point number, because every `NaN` payload is rendered as the same `nan`, while hash +/// tables compare floating-point keys bitwise and keep distinct payloads apart; +/// - a `Date32`, because day numbers out of the type range are saturated to `0000-01-01` and +/// `9999-12-31` when formatted (see `ToStringMonotonicity`); +/// - a type-erased type such as `Variant`, `Dynamic` or `Object`, whose alternatives render into a +/// common text space: `toString(1::Variant(Int64, String))` equals `toString('1'::Variant(Int64, String))`; +/// - a `Bool` when `bool_true_representation` and `bool_false_representation` coincide in `format_settings`. +/// +/// An unknown type is assumed to collapse, so that an undecidable case is never claimed to be injective. +inline bool renderingCollapsesDistinctValues(const DataTypePtr & type, const FormatSettings & format_settings) +{ + if (!type) + return true; + + bool collapses = false; + auto check = [&](const IDataType & nested) + { + if (const auto * date_time = typeid_cast(&nested)) + collapses = collapses || !date_time->getTimeZone().hasFixedOffset(); + else if (const auto * date_time64 = typeid_cast(&nested)) + collapses = collapses || !date_time64->getTimeZone().hasFixedOffset(); + else if (isFloat(nested) || isDate32(nested) || isVariant(nested) || isDynamic(nested) || isObject(nested)) + collapses = true; + else if (isUInt8(nested) && nested.getName() == "Bool") + collapses = collapses || format_settings.bool_true_representation == format_settings.bool_false_representation; + }; + + check(*type); + type->forEachChild(check); + return collapses; +} + template class FunctionConvert final : public IFunction { @@ -3186,7 +3222,22 @@ class FunctionConvert final : public IFunction bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } - bool isInjective(const ColumnsWithTypeAndName &) const override { return std::is_same_v; } + bool isInjective(const ColumnsWithTypeAndName & sample_columns) const override + { + if constexpr (std::is_same_v) + { + /// Only the single-argument form is claimed. A caller that passes no arguments gets no + /// claim, because the answer depends on them, and the multi-argument forms are rendered + /// in the time zone given by the second argument - `toString(dt, 'Europe/Amsterdam')` + /// folds even for a fixed-offset `dt`, and `toString(x, NULL)` maps every row to `NULL`. + if (sample_columns.size() != 1) + return false; + + return !renderingCollapsesDistinctValues(sample_columns.front().type, settings.format_settings); + } + else + return false; + } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & arguments) const override { return !(IsDataTypeDateOrDateTime && isNumber(*arguments[0].type)); diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index f389d54bc6f9c..3adf383980eaa 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -214,8 +214,10 @@ class ConcatOverloadResolver final : public IFunctionOverloadResolver bool isInjective(const ColumnsWithTypeAndName & arguments) const override { - /// When called with a single argument, concat delegates to toString which is injective. - return arguments.size() == 1; + /// When called with a single argument, concat delegates to toString, so the claim has to be + /// the one toString makes: it depends on the argument type, because rendering a date-time in a + /// time zone that has a UTC offset transition maps two distinct instants onto one string. + return arguments.size() == 1 && to_string->isInjective(arguments); } FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override diff --git a/src/Functions/toFixedString.h b/src/Functions/toFixedString.h index c4ab91fa12e8b..90e382ae987f8 100644 --- a/src/Functions/toFixedString.h +++ b/src/Functions/toFixedString.h @@ -1,10 +1,13 @@ #pragma once #include #include +#include +#include #include #include #include #include +#include #include @@ -38,7 +41,37 @@ class FunctionToFixedString final : public IFunction } size_t getNumberOfArguments() const override { return 2; } - bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } + + /// Injective only for a `FixedString(M)` argument that fits into the target width. A `String` is + /// NUL-padded to the target width, so 'a' and 'a\0' both become the same `FixedString(2)`. Every + /// value of a `FixedString(M)`, in contrast, has the same length, so padding appends the same + /// suffix to all of them - but only when `M <= N`; otherwise execution throws, and eliminating + /// the call would silently turn a query that must fail into one that succeeds. + /// A caller that passes no arguments gets no claim, because the answer depends on them. + bool isInjective(const ColumnsWithTypeAndName & sample_columns) const override + { + if (sample_columns.size() != 2) + return false; + + const auto * fixed_string = typeid_cast( + removeNullable(recursiveRemoveLowCardinality(sample_columns[0].type)).get()); + if (!fixed_string) + return false; + + /// The target width has to be known to compare it with the source width. It also has to be + /// an unsigned integer: `getReturnTypeImpl` rejects everything else, and reading a negative + /// or a `NULL` width here would either wrap around to a huge `UInt64` or throw, in both + /// cases eliminating a call that must raise + /// `Second argument for function toFixedString must be unsigned integer`. + if (!isUInt(sample_columns[1].type)) + return false; + + const auto & width_column = sample_columns[1].column; + if (!width_column || !isColumnConst(*width_column)) + return false; + + return fixed_string->getN() <= width_column->getUInt(0); + } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override diff --git a/src/Interpreters/RemoveInjectiveFunctionsVisitor.cpp b/src/Interpreters/RemoveInjectiveFunctionsVisitor.cpp index 8d0303799095a..45f38f83d8339 100644 --- a/src/Interpreters/RemoveInjectiveFunctionsVisitor.cpp +++ b/src/Interpreters/RemoveInjectiveFunctionsVisitor.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -17,7 +18,12 @@ static bool isUniq(const ASTFunction & func) } /// Remove injective functions of one argument: replace with a child -static bool removeInjectiveFunction(ASTPtr & ast, ContextPtr context, const FunctionFactory & function_factory) +static bool removeInjectiveFunction( + ASTPtr & ast, + ContextPtr context, + const NamesAndTypesList & source_columns, + const NameSet & array_join_result_names, + const FunctionFactory & function_factory) { const ASTFunction * func = ast->as(); if (!func) @@ -26,7 +32,10 @@ static bool removeInjectiveFunction(ASTPtr & ast, ContextPtr context, const Func if (!func->arguments || func->arguments->children.size() != 1) return false; - if (!function_factory.get(func->name, context)->isInjective({})) + /// The claim can depend on the argument, so resolve it as far as the AST allows. An argument + /// that stays unresolved leaves the function unclaimed. + auto argument_columns = tryGetASTFunctionArgumentColumns(*func, source_columns, array_join_result_names); + if (!argument_columns || !function_factory.get(func->name, context)->isInjective(*argument_columns)) return false; ast = func->arguments->children[0]; @@ -47,7 +56,7 @@ void RemoveInjectiveFunctionsMatcher::visit(ASTFunction & func, ASTPtr &, const for (auto & arg : func.arguments->children) { - while (removeInjectiveFunction(arg, data.getContext(), function_factory)) + while (removeInjectiveFunction(arg, data.getContext(), data.source_columns, data.array_join_result_names, function_factory)) ; } } diff --git a/src/Interpreters/RemoveInjectiveFunctionsVisitor.h b/src/Interpreters/RemoveInjectiveFunctionsVisitor.h index 4b6fc0a643800..5dac8f03619fa 100644 --- a/src/Interpreters/RemoveInjectiveFunctionsVisitor.h +++ b/src/Interpreters/RemoveInjectiveFunctionsVisitor.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -15,7 +16,12 @@ class RemoveInjectiveFunctionsMatcher public: struct Data : public WithContext { - explicit Data(ContextPtr context_) : WithContext(context_) {} + Data(ContextPtr context_, const NamesAndTypesList & source_columns_, NameSet array_join_result_names_) + : WithContext(context_), source_columns(source_columns_), array_join_result_names(std::move(array_join_result_names_)) {} + + const NamesAndTypesList & source_columns; + /// See `getArrayJoinResultNames`. + NameSet array_join_result_names; }; static void visit(ASTPtr & ast, const Data & data); diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 249cb8609e44a..eeb30eb20f137 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -98,13 +99,16 @@ void appendUnusedGroupByColumn(ASTSelectQuery * select_query) } /// Eliminates injective function calls and constant expressions from group by statement. -void optimizeGroupBy(ASTSelectQuery * select_query, ContextPtr context) +void optimizeGroupBy(ASTSelectQuery * select_query, const NamesAndTypesList & source_columns, ContextPtr context) { const FunctionFactory & function_factory = FunctionFactory::instance(); if (!select_query->groupBy()) return; + /// An `ARRAY JOIN` result may shadow a source column of the same name; see `tryGetASTFunctionArgumentColumns`. + const NameSet array_join_result_names = getArrayJoinResultNames(*select_query); + /// Skip when a GROUP BY modifier produces rows where a grouping key is absent from the set /// being aggregated: CUBE/ROLLUP subtotals, GROUPING SETS non-member sets, and the WITH TOTALS /// row. In such a row the key is output as its column default. Rewriting f(g) -> g makes the @@ -180,7 +184,10 @@ void optimizeGroupBy(ASTSelectQuery * select_query, ContextPtr context) if (!function_builder) function_builder = function_factory.get(function->name, context); - if (!function_builder->isInjective({})) + /// The claim can depend on the arguments, so resolve as many of them as the AST + /// allows. An argument that stays unresolved leaves the function unclaimed. + auto argument_columns = tryGetASTFunctionArgumentColumns(*function, source_columns, array_join_result_names); + if (!argument_columns || !function_builder->isInjective(*argument_columns)) { ++i; continue; @@ -552,9 +559,9 @@ void optimizeMultiIfToIf(ASTPtr & query) OptimizeMultiIfToIfVisitor(data).visit(query); } -void optimizeInjectiveFunctionsInsideUniq(ASTPtr & query, ContextPtr context) +void optimizeInjectiveFunctionsInsideUniq(ASTPtr & query, const NamesAndTypesList & source_columns, ContextPtr context) { - RemoveInjectiveFunctionsVisitor::Data data(context); + RemoveInjectiveFunctionsVisitor::Data data(context, source_columns, getArrayJoinResultNames(query->as())); RemoveInjectiveFunctionsVisitor(data).visit(query); } @@ -692,7 +699,7 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, optimizeDateFilters(select_query, tables_with_columns, context); /// GROUP BY injective function elimination. - optimizeGroupBy(select_query, context); + optimizeGroupBy(select_query, result.source_columns, context); /// GROUP BY functions of other keys elimination. if (settings[Setting::optimize_group_by_function_keys]) @@ -703,7 +710,7 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, /// Remove injective functions inside uniq if (settings[Setting::optimize_injective_functions_inside_uniq]) - optimizeInjectiveFunctionsInsideUniq(query, context); + optimizeInjectiveFunctionsInsideUniq(query, result.source_columns, context); /// Eliminate min/max/any aggregators of functions of GROUP BY keys. /// GROUPING SETS with a single set is classified as an ordinary GROUP BY by ExpressionAnalyzer diff --git a/src/Interpreters/getASTFunctionArgumentColumns.cpp b/src/Interpreters/getASTFunctionArgumentColumns.cpp new file mode 100644 index 0000000000000..56d776b072b99 --- /dev/null +++ b/src/Interpreters/getASTFunctionArgumentColumns.cpp @@ -0,0 +1,72 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NameSet getArrayJoinResultNames(const ASTSelectQuery & select_query) +{ + NameSet result_names; + + const auto & array_join_expression_list = select_query.arrayJoinExpressionList().first; + if (!array_join_expression_list) + return result_names; + + for (const auto & expression : array_join_expression_list->children) + result_names.insert(expression->getAliasOrColumnName()); + + return result_names; +} + +std::optional tryGetASTFunctionArgumentColumns( + const ASTFunction & function, const NamesAndTypesList & source_columns, const NameSet & array_join_result_names) +{ + ColumnsWithTypeAndName arguments; + + if (!function.arguments) + return arguments; + + arguments.reserve(function.arguments->children.size()); + for (const auto & child : function.arguments->children) + { + if (const auto * literal = child->as()) + { + auto type = applyVisitor(FieldToDataType(), literal->value); + arguments.emplace_back(type->createColumnConst(1, literal->value), type, ""); + } + else if (const auto * identifier = child->as()) + { + /// A compound name - a table-qualified `JOIN` key such as `r.dt`, or a subcolumn such as + /// `tuple.dt` - is not decidable here. This runs before `collectJoinedColumns`, so + /// `source_columns` describes only the source side, and matching by the last part alone + /// would resolve `r.dt` against an unrelated source column named `dt` and read its type. + if (identifier->compound()) + return {}; + + /// This also runs before `getArrayJoinedColumns`, so a plain name may still denote an + /// `ARRAY JOIN` result rather than the source column it shadows: after `ARRAY JOIN arr AS dt` + /// the runtime `dt` is an element of `arr`, whatever the type of the source `dt`. + if (array_join_result_names.contains(identifier->name())) + return {}; + + auto name_and_type = source_columns.tryGetByName(identifier->name()); + if (!name_and_type) + return {}; + arguments.emplace_back(ColumnPtr{}, name_and_type->type, name_and_type->name); + } + else + return {}; + } + + return arguments; +} + +} diff --git a/src/Interpreters/getASTFunctionArgumentColumns.h b/src/Interpreters/getASTFunctionArgumentColumns.h new file mode 100644 index 0000000000000..fce9c1d915e78 --- /dev/null +++ b/src/Interpreters/getASTFunctionArgumentColumns.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +#include + + +namespace DB +{ + +class ASTFunction; +class ASTSelectQuery; + +/// The result names an `ARRAY JOIN` clause of the query introduces: the alias of every `ARRAY JOIN` +/// expression, or its column name when it has none. An identifier with such a name denotes the +/// array-joined element, not the source column of the same name, and `ARRAY JOIN arr AS value` is +/// allowed even when `value` already is a source column. +NameSet getArrayJoinResultNames(const ASTSelectQuery & select_query); + +/// The arguments of a function call in the shape `IFunctionBase` and `IFunctionOverloadResolver` +/// methods expect: the result type of every argument, and its constant column where it has one. +/// Whether a function is injective can depend on them - `toString` of a date-time in a time zone +/// with a UTC offset transition is not injective, and neither is `toString(x, NULL)` - so a caller +/// that can resolve them has no reason to pass nothing. +/// +/// Only what the AST alone decides is resolved: a plain identifier that names a source column, and +/// a literal. For anything else - including a compound identifier, which may name a joined column +/// or a subcolumn rather than the source column of the same short name, and an identifier that +/// may name an `ARRAY JOIN` result (`array_join_result_names`, see `getArrayJoinResultNames`) - +/// the answer is undecidable at this point, and `std::nullopt` is returned so that the caller +/// claims nothing. +std::optional tryGetASTFunctionArgumentColumns( + const ASTFunction & function, const NamesAndTypesList & source_columns, const NameSet & array_join_result_names); + +} diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp index a4462d6004c3d..fe48718589230 100644 --- a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp @@ -551,9 +551,18 @@ std::optionalcolumn, child->result_type, child->result_name}); + return arguments; +} + bool isInjectiveFunction(const ActionsDAG::Node * node) { - if (node->function_base->isInjective({})) + if (node->function_base->isInjective(getFunctionArgumentColumns(*node))) return true; const auto & name = node->function_base->getName(); diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h index c9e14970fb20b..d42332060a2d9 100644 --- a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h @@ -91,6 +91,11 @@ std::optional & allowed_inputs, const ActionsDAG::NodeRawConstPtrs & nodes); +/// The arguments of a function node in the shape `IFunctionBase` methods expect: the result type of +/// every child, and its constant column where the child has one. Whether a function is injective or +/// monotonic can depend on them, so a caller that has a node has no reason to pass nothing. +ColumnsWithTypeAndName getFunctionArgumentColumns(const ActionsDAG::Node & node); + bool isInjectiveFunction(const ActionsDAG::Node * node); /// Our objective is to replace injective function nodes in `actions` results with its children diff --git a/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp b/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp index 472bd4c0c1ef2..d742cf003073f 100644 --- a/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -116,7 +117,8 @@ size_t tryConvertAnyOuterJoinToInnerJoin( if (current_node->type == ActionsDAG::ActionType::INPUT) join_keys_interesting_side.insert(current_node->result_name); - if (current_node->type == ActionsDAG::ActionType::FUNCTION && current_node->function_base->isInjective({})) + if (current_node->type == ActionsDAG::ActionType::FUNCTION + && current_node->function_base->isInjective(getFunctionArgumentColumns(*current_node))) { for (const auto * child : current_node->children) nodes_to_process.push(child); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index a9fec9089b4a3..fab0836a8d946 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -217,7 +217,7 @@ void appendFixedColumnsFromFilterExpression(const ActionsDAG::Node & filter_expr const ActionsDAG::Node * maybe_injective = maybe_fixed_column; while (maybe_injective->type == ActionsDAG::ActionType::FUNCTION && maybe_injective->children.size() == 1 - && maybe_injective->function_base->isInjective({})) + && maybe_injective->function_base->isInjective(getFunctionArgumentColumns(*maybe_injective))) { maybe_injective = maybe_injective->children.front(); fixed_columns.insert(maybe_injective); diff --git a/tests/queries/0_stateless/05227_injective_claims_tostring_datetime_and_tofixedstring.reference b/tests/queries/0_stateless/05227_injective_claims_tostring_datetime_and_tofixedstring.reference new file mode 100644 index 0000000000000..ba9f9a8135edc --- /dev/null +++ b/tests/queries/0_stateless/05227_injective_claims_tostring_datetime_and_tofixedstring.reference @@ -0,0 +1,68 @@ +ground truth +1 +1 +uniq elimination +1 +1 +order by truncation +2018-10-28 02:30:00 2 +2018-10-28 02:30:00 1 +2018-10-28 02:30:00 2 +2018-10-28 02:30:00 1 +group by +1 +1 +1 +1 +limit by +1 +1 +1 +1 +independent window partitions +200 +200 +1600 +1600 +a fixed-offset time zone is still injective +1 +1 +an explicit time zone argument +1 +1 +1 +1 +an out-of-range Date32 +1 +1 +1 +1 +a too narrow target width still throws +1 +a width that is not an unsigned integer still throws +a qualified JOIN key +1 +1 +1 +1 +an ARRAY JOIN alias that shadows a source column +1 +1 +1 +1 +1 +an ARRAY JOIN alias that shadows a sorting key column +1 +2 +distinct NaN payloads +1 +1 +1 +1 +coinciding boolean representations +1 +1 +1 +1 +1 +2 diff --git a/tests/queries/0_stateless/05227_injective_claims_tostring_datetime_and_tofixedstring.sql b/tests/queries/0_stateless/05227_injective_claims_tostring_datetime_and_tofixedstring.sql new file mode 100644 index 0000000000000..e6edcc7addfe6 --- /dev/null +++ b/tests/queries/0_stateless/05227_injective_claims_tostring_datetime_and_tofixedstring.sql @@ -0,0 +1,159 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/116931 +-- https://github.com/ClickHouse/ClickHouse/issues/116935 +-- https://github.com/ClickHouse/ClickHouse/issues/116828 +-- https://github.com/ClickHouse/ClickHouse/issues/103686 +-- `toString` of a date-time is not injective in a time zone with a UTC offset transition: in a +-- fall-back hour two distinct instants render to the same wall-clock string. `toFixedString` of a +-- `String` is not injective either: the value is NUL-padded, so 'a' and 'a\0' give the same +-- `FixedString(2)`. Every pair below prints the result at the default settings and then with the +-- optimization that trusted the claim disabled; the two must agree. + +DROP TABLE IF EXISTS t_tz_fold; +CREATE TABLE t_tz_fold (dt DateTime('Europe/Amsterdam'), v UInt32) ENGINE = Memory; +-- 2018-10-28 in Amsterdam: 03:00 CEST falls back to 02:00 CET. +INSERT INTO t_tz_fold VALUES (1540686600, 2), (1540690200, 1); + +SELECT 'ground truth'; +SELECT toString(toDateTime(1540686600, 'Europe/Amsterdam')) = toString(toDateTime(1540690200, 'Europe/Amsterdam')); +SELECT toFixedString('a', 2) = toFixedString('a\0', 2); + +SELECT 'uniq elimination'; +SELECT uniqExact(toString(dt)) FROM t_tz_fold; +SELECT uniqExact(toString(dt)) FROM t_tz_fold SETTINGS optimize_injective_functions_inside_uniq = 0; + +SELECT 'order by truncation'; +SELECT toString(dt) AS s, max(v) AS m FROM t_tz_fold GROUP BY dt ORDER BY s, m DESC; +SELECT toString(dt) AS s, max(v) AS m FROM t_tz_fold GROUP BY dt ORDER BY s, m DESC SETTINGS optimize_truncate_order_by_after_group_by_keys = 0; + +SELECT 'group by'; +SELECT count() FROM (SELECT toString(dt) AS s FROM t_tz_fold GROUP BY s); +SELECT count() FROM (SELECT toString(dt) AS s FROM t_tz_fold GROUP BY s) SETTINGS optimize_injective_functions_in_group_by = 0; +SELECT count() FROM (SELECT 1 FROM (SELECT arrayJoin(['a', 'a\0']) AS x) GROUP BY toFixedString(x, 2)); +SELECT count() FROM (SELECT 1 FROM (SELECT arrayJoin(['a', 'a\0']) AS x) GROUP BY toFixedString(x, 2)) SETTINGS optimize_injective_functions_in_group_by = 0; + +SELECT 'limit by'; +SELECT count() FROM (SELECT toString(dt) AS s, v FROM t_tz_fold ORDER BY v LIMIT 1 BY s); +SELECT count() FROM (SELECT toString(dt) AS s, v FROM t_tz_fold ORDER BY v LIMIT 1 BY s) SETTINGS optimize_injective_functions_in_limit_by = 0; +SELECT count() FROM (SELECT arrayJoin(['a', 'a\0']) AS x LIMIT 1 BY toFixedString(x, 2)); +SELECT count() FROM (SELECT arrayJoin(['a', 'a\0']) AS x LIMIT 1 BY toFixedString(x, 2)) SETTINGS optimize_injective_functions_in_limit_by = 0; + +SELECT 'independent window partitions'; +DROP TABLE IF EXISTS t_window_fold; +CREATE TABLE t_window_fold (ts DateTime('America/New_York'), v UInt64) ENGINE = MergeTree ORDER BY tuple() +PARTITION BY intDiv(toUnixTimestamp(ts), 3600); +-- 1762061400 is 2025-11-02 01:30:00 EDT; 3600 s later is 01:30:00 EST - another hourly partition +-- with the same rendering. +INSERT INTO t_window_fold SELECT toDateTime(1762061400 + 3600 * intDiv(number, 100), 'America/New_York'), number FROM numbers(1600); + +SELECT c FROM (SELECT DISTINCT toString(ts) AS s, count() OVER (PARTITION BY toString(ts)) AS c FROM t_window_fold) WHERE s = '2025-11-02 01:30:00' SETTINGS max_threads = 16; +SELECT c FROM (SELECT DISTINCT toString(ts) AS s, count() OVER (PARTITION BY toString(ts)) AS c FROM t_window_fold) WHERE s = '2025-11-02 01:30:00' SETTINGS max_threads = 16, allow_window_partitions_independently = 0; + +DROP TABLE IF EXISTS t_window_pad; +CREATE TABLE t_window_pad (s String) ENGINE = MergeTree ORDER BY tuple() PARTITION BY length(s); +INSERT INTO t_window_pad SELECT concat('k', repeat('\0', number % 16)) FROM numbers(1600); + +SELECT DISTINCT count() OVER (PARTITION BY toFixedString(s, 16)) FROM t_window_pad SETTINGS max_threads = 16; +SELECT DISTINCT count() OVER (PARTITION BY toFixedString(s, 16)) FROM t_window_pad SETTINGS max_threads = 16, allow_window_partitions_independently = 0; + +SELECT 'a fixed-offset time zone is still injective'; +SELECT count() > 0 FROM (EXPLAIN QUERY TREE SELECT uniqExact(toString(toDateTime(number, 'UTC'))) FROM numbers(2)) WHERE explain LIKE '%uniqExact%' AND explain NOT LIKE '%toString%' SETTINGS enable_analyzer = 1; +SELECT count() > 0 FROM (EXPLAIN QUERY TREE SELECT uniqExact(toString(toDateTime(number, 'Europe/Amsterdam'))) FROM numbers(2)) WHERE explain LIKE '%toString%' SETTINGS enable_analyzer = 1; + +SELECT 'an explicit time zone argument'; +-- The rendering zone comes from the second argument, so a fixed-offset `DateTime('UTC')` folds all +-- the same, and `toString(x, NULL)` maps every row onto `NULL`. +DROP TABLE IF EXISTS t_tz_arg; +CREATE TABLE t_tz_arg (dt DateTime('UTC')) ENGINE = Memory; +INSERT INTO t_tz_arg VALUES (1540686600), (1540690200); +SELECT count() FROM (SELECT 1 FROM t_tz_arg GROUP BY toString(dt, 'Europe/Amsterdam')); +SELECT count() FROM (SELECT 1 FROM t_tz_arg GROUP BY toString(dt, 'Europe/Amsterdam')) SETTINGS optimize_injective_functions_in_group_by = 0; +SELECT count() FROM (SELECT 1 FROM t_tz_arg GROUP BY toString(dt, NULL)); +SELECT count() FROM (SELECT 1 FROM t_tz_arg GROUP BY toString(dt, NULL)) SETTINGS optimize_injective_functions_in_group_by = 0; + +SELECT 'an out-of-range Date32'; +-- 2000000000 and 1999999999 are both out of the `Date32` range and are saturated to `9999-12-31` +-- when formatted. `RowBinary` is the way to get such day numbers into a column at all. +SELECT count() FROM (SELECT 1 FROM format(RowBinary, 'd Date32', unhex('00943577FF933577')) GROUP BY toString(d)); +SELECT count() FROM (SELECT 1 FROM format(RowBinary, 'd Date32', unhex('00943577FF933577')) GROUP BY toString(d)) SETTINGS optimize_injective_functions_in_group_by = 0; +SELECT uniqExact(toString(d)) FROM format(RowBinary, 'd Date32', unhex('00943577FF933577')); +SELECT uniqExact(toString(d)) FROM format(RowBinary, 'd Date32', unhex('00943577FF933577')) SETTINGS optimize_injective_functions_inside_uniq = 0; + +SELECT 'a too narrow target width still throws'; +-- `toFixedString(FixedString(3), 2)` must not be eliminated: it has to raise the exception. +SELECT count() FROM (SELECT 1 FROM (SELECT materialize(toFixedString('abc', 3)) AS f) GROUP BY toFixedString(f, 2)); -- { serverError TOO_LARGE_STRING_SIZE } +SELECT count() FROM (SELECT materialize(toFixedString('abc', 3)) AS f LIMIT 1 BY toFixedString(f, 2)); -- { serverError TOO_LARGE_STRING_SIZE } +SELECT uniqExact(toFixedString(f, 2)) FROM (SELECT materialize(toFixedString('abc', 3)) AS f); -- { serverError TOO_LARGE_STRING_SIZE } +-- A fitting target width stays injective. +SELECT count() > 0 FROM (EXPLAIN QUERY TREE SELECT uniqExact(toFixedString(materialize(toFixedString('abc', 3)), 4))) WHERE explain LIKE '%uniqExact%' AND explain NOT LIKE '%toFixedString%' SETTINGS enable_analyzer = 1; + +SELECT 'a width that is not an unsigned integer still throws'; +-- The claim must not be made before the width is known to be unsigned: reading a negative width +-- wraps it around to a huge `UInt64`, and eliminating the call would turn a query that must fail +-- into one that succeeds. +SELECT count() FROM (SELECT 1 FROM (SELECT materialize(toFixedString('abc', 3)) AS f) GROUP BY toFixedString(f, -1)); -- { serverError ILLEGAL_COLUMN } +SELECT uniqExact(toFixedString(f, -1)) FROM (SELECT materialize(toFixedString('abc', 3)) AS f); -- { serverError ILLEGAL_COLUMN } +SELECT count() FROM (SELECT materialize(toFixedString('abc', 3)) AS f LIMIT 1 BY toFixedString(f, -1)); -- { serverError ILLEGAL_COLUMN } + +SELECT 'a qualified JOIN key'; +-- A right-hand key must be typed as the right-hand column: `l.dt` is a `UInt32` and would make +-- `toString` look injective, while the right-hand `DateTime` folds. +DROP TABLE IF EXISTS t_join_left; +DROP TABLE IF EXISTS t_join_right; +CREATE TABLE t_join_left (id UInt32, dt UInt32) ENGINE = Memory; +CREATE TABLE t_join_right (id UInt32, dt DateTime('Europe/Amsterdam')) ENGINE = Memory; +INSERT INTO t_join_left VALUES (1, 100), (2, 200); +INSERT INTO t_join_right VALUES (1, 1540686600), (2, 1540690200); +SELECT count() FROM (SELECT 1 FROM t_join_left INNER JOIN t_join_right ON t_join_left.id = t_join_right.id GROUP BY toString(t_join_right.dt)); +SELECT count() FROM (SELECT 1 FROM t_join_left INNER JOIN t_join_right ON t_join_left.id = t_join_right.id GROUP BY toString(t_join_right.dt)) SETTINGS optimize_injective_functions_in_group_by = 0; +SELECT uniqExact(toString(t_join_right.dt)) FROM t_join_left INNER JOIN t_join_right ON t_join_left.id = t_join_right.id; +SELECT uniqExact(toString(t_join_right.dt)) FROM t_join_left INNER JOIN t_join_right ON t_join_left.id = t_join_right.id SETTINGS optimize_injective_functions_inside_uniq = 0; + +DROP TABLE t_join_left; +DROP TABLE t_join_right; + +SELECT 'an ARRAY JOIN alias that shadows a source column'; +-- `ARRAY JOIN arr AS dt` is allowed when `dt` already is a source column. The identifier must be +-- typed as the array-joined element - a folding `DateTime` - and not as the source `UInt32` it +-- shadows. +DROP TABLE IF EXISTS t_array_join_shadow; +CREATE TABLE t_array_join_shadow (dt UInt32, arr Array(DateTime('Europe/Amsterdam'))) ENGINE = Memory; +INSERT INTO t_array_join_shadow VALUES (1, [1540686600, 1540690200]); +SELECT count() FROM (SELECT 1 FROM t_array_join_shadow ARRAY JOIN arr AS dt GROUP BY toString(dt)); +SELECT count() FROM (SELECT 1 FROM t_array_join_shadow ARRAY JOIN arr AS dt GROUP BY toString(dt)) SETTINGS optimize_injective_functions_in_group_by = 0; +SELECT uniqExact(toString(dt)) FROM t_array_join_shadow ARRAY JOIN arr AS dt; +SELECT uniqExact(toString(dt)) FROM t_array_join_shadow ARRAY JOIN arr AS dt SETTINGS optimize_injective_functions_inside_uniq = 0; +-- The source column of the same name is unaffected when the `ARRAY JOIN` result is named otherwise. +SELECT count() FROM (SELECT 1 FROM t_array_join_shadow ARRAY JOIN arr AS elem GROUP BY toString(dt)); +DROP TABLE t_array_join_shadow; + +SELECT 'an ARRAY JOIN alias that shadows a sorting key column'; +-- A bare `tenant = 'x'` constrains the array-joined element, not the source sorting key `tenant`, so +-- the stream is not ordered by `ts` alone. +DROP TABLE IF EXISTS t_array_join_order; +CREATE TABLE t_array_join_order (tenant String, ts UInt32, arr Array(String)) ENGINE = MergeTree ORDER BY (tenant, ts); +INSERT INTO t_array_join_order VALUES ('a', 2, ['x']), ('b', 1, ['x']); +SELECT ts FROM t_array_join_order ARRAY JOIN arr AS tenant WHERE tenant = 'x' ORDER BY ts; +DROP TABLE t_array_join_order; + +SELECT 'distinct NaN payloads'; +-- Every `NaN` is rendered as `nan`, while the raw floating-point keys are compared bitwise. +SELECT uniqExact(toString(x)) FROM (SELECT arrayJoin([reinterpretAsFloat64(toUInt64(0x7FF0000000000001)), reinterpretAsFloat64(toUInt64(0x7FF8000000000001))]) AS x); +SELECT uniqExact(toString(x)) FROM (SELECT arrayJoin([reinterpretAsFloat64(toUInt64(0x7FF0000000000001)), reinterpretAsFloat64(toUInt64(0x7FF8000000000001))]) AS x) SETTINGS optimize_injective_functions_inside_uniq = 0; +SELECT count() FROM (SELECT 1 FROM (SELECT arrayJoin([reinterpretAsFloat64(toUInt64(0x7FF0000000000001)), reinterpretAsFloat64(toUInt64(0x7FF8000000000001))]) AS x) GROUP BY toString(x)); +SELECT count() FROM (SELECT 1 FROM (SELECT arrayJoin([reinterpretAsFloat64(toUInt64(0x7FF0000000000001)), reinterpretAsFloat64(toUInt64(0x7FF8000000000001))]) AS x) GROUP BY toString(x)) SETTINGS optimize_injective_functions_in_group_by = 0; + + +SELECT 'coinciding boolean representations'; +-- With equal `bool_true_representation` and `bool_false_representation` both values of a `Bool` render as the same string. +SELECT uniqExact(toString(b)) FROM (SELECT arrayJoin([false, true]) AS b) SETTINGS bool_true_representation = 'x', bool_false_representation = 'x'; +SELECT uniqExact(toString(b)) FROM (SELECT arrayJoin([false, true]) AS b) SETTINGS bool_true_representation = 'x', bool_false_representation = 'x', optimize_injective_functions_inside_uniq = 0; +SELECT count() FROM (SELECT 1 FROM (SELECT arrayJoin([false, true]) AS b) GROUP BY toString(b)) SETTINGS bool_true_representation = 'x', bool_false_representation = 'x'; +SELECT count() FROM (SELECT 1 FROM (SELECT arrayJoin([false, true]) AS b) GROUP BY toString(b)) SETTINGS bool_true_representation = 'x', bool_false_representation = 'x', optimize_injective_functions_in_group_by = 0; +SELECT count() FROM (SELECT b FROM (SELECT arrayJoin([false, true]) AS b) LIMIT 1 BY toString(b)) SETTINGS bool_true_representation = 'x', bool_false_representation = 'x'; +-- Distinct representations stay injective. +SELECT count() FROM (SELECT 1 FROM (SELECT arrayJoin([false, true]) AS b) GROUP BY toString(b)) SETTINGS bool_true_representation = 'yes', bool_false_representation = 'no'; + +DROP TABLE t_tz_fold; +DROP TABLE t_window_fold; +DROP TABLE t_window_pad; +DROP TABLE t_tz_arg; From 2bbbc5e9fef7628ba2fddf54824a26060ac9ff55 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 03:15:19 +0000 Subject: [PATCH 277/296] Backport #123649 to 26.8: Fix distributed queries failing when Native binary type encoding is enabled --- src/Client/ClientBase.cpp | 4 +- src/Formats/FormatFactory.cpp | 11 ++++ src/Formats/FormatFactory.h | 3 + src/Server/TCPHandler.cpp | 19 ++++--- src/Server/TCPHandler.h | 2 +- .../test_native_type_names_secondary_query.py | 48 ++++++++++++++++ ...ive_binary_types_between_servers.reference | 22 ++++++++ ...317_native_binary_types_between_servers.sh | 55 +++++++++++++++++++ 8 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 tests/integration/test_backward_compatibility/test_native_type_names_secondary_query.py create mode 100644 tests/queries/0_stateless/05317_native_binary_types_between_servers.reference create mode 100755 tests/queries/0_stateless/05317_native_binary_types_between_servers.sh diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 09408551b6ea1..cd2640da724c0 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2905,14 +2905,13 @@ void ClientBase::processParsedSingleQuery( } } client_context->setSettings(old_settings); - connection->setFormatSettings(getFormatSettings(client_context)); + connection->setFormatSettings(getNativeWireFormatSettings(client_context)); }); /// Capture whether this query was parsed via the `clickhouse_json` dialect *before* applying any /// in-query `SET` (which may change `dialect`/`enable_json_ast_dialect`). The outbound /// transport dialect is pinned to match the outbound text in `pinOutboundDialectForJSONDialect`. current_query_parsed_as_json_dialect = client_context->getSettingsRef()[Setting::dialect] == Dialect::clickhouse_json; InterpreterSetQuery::applySettingsFromQuery(parsed_query, client_context); - connection->setFormatSettings(getFormatSettings(client_context)); /// Deliberately without a round trip: this runs before every query. The only case that needs /// the stronger check is a session that continues after a failed query - the protocol can be @@ -2923,6 +2922,7 @@ void ClientBase::processParsedSingleQuery( else if (!connection->checkConnectedWithoutRoundTrip()) connect(); + connection->setFormatSettings(getNativeWireFormatSettings(client_context)); applySettingsFromServerIfNeeded(); // after connect() and applySettingsFromQuery() /// With `use_client_time_zone`, DateTime string literals must be interpreted in the client time diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index a36d3af0f3e35..e3ed598c84a9a 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -479,6 +479,17 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se return format_settings; } +FormatSettings getNativeWireFormatSettings(const ContextPtr & context) +{ + auto format_settings = getFormatSettings(context); + if (context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) + { + format_settings.native.encode_types_in_binary_format = false; + format_settings.native.decode_types_in_binary_format = false; + } + return format_settings; +} + FileBucketInfoPtr FormatFactory::getFileBucketInfo(const String & format) { auto creator = getCreators(format); diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 7ff210bffb735..abd855c159456 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -63,6 +63,9 @@ using BucketSplitter = std::shared_ptr; FormatSettings getFormatSettings(const ContextPtr & context); FormatSettings getFormatSettings(const ContextPtr & context, const Settings & settings); +/// For the `Native` blocks of the native protocol. A secondary query carries type names in both directions, +/// because a server-side `Connection` never takes format settings. +FormatSettings getNativeWireFormatSettings(const ContextPtr & context); /** Allows to create an IInputFormat or IOutputFormat by the name of the format. * Note: format and compression are independent things. diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 938759d9205a0..b598e4658b527 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -901,7 +901,7 @@ void TCPHandler::runImpl() block, getCompressionCodec(query_settings, query_state->compression), client_tcp_protocol_version, - getFormatSettings(query_state->query_context), + getNativeWireFormatSettings(query_state->query_context), !query_settings[Setting::low_cardinality_allow_in_native_format]); }); @@ -1319,7 +1319,7 @@ bool TCPHandler::receivePacketsExpectData(QueryState & state) { bool empty_block = false; if (state.skipping_data) - empty_block = !processUnexpectedData(); + empty_block = !processUnexpectedData(state); else empty_block = !processData(state, packet_type == Protocol::Client::Scalar); if (empty_block) @@ -2758,7 +2758,7 @@ void TCPHandler::processQuery(std::shared_ptr & state) block, getCompressionCodec(query_settings, current_state->compression), client_tcp_protocol_version, - getFormatSettings(current_state->query_context), + getNativeWireFormatSettings(current_state->query_context), !query_settings[Setting::low_cardinality_allow_in_native_format]); }); } @@ -2975,7 +2975,7 @@ bool TCPHandler::processData(QueryState & state, bool scalar) } -bool TCPHandler::processUnexpectedData() +bool TCPHandler::processUnexpectedData(QueryState & state) { String skip_external_table_name; readStringBinary(skip_external_table_name, *in); @@ -2986,7 +2986,8 @@ bool TCPHandler::processUnexpectedData() else maybe_compressed_in = in; - auto skip_block_in = std::make_shared(*maybe_compressed_in, client_tcp_protocol_version); + auto skip_block_in = std::make_shared( + *maybe_compressed_in, client_tcp_protocol_version, getNativeWireFormatSettings(state.query_context)); bool empty_block = skip_block_in->read().empty(); return !empty_block; } @@ -3014,7 +3015,7 @@ void TCPHandler::initBlockInput(QueryState & state) *state.maybe_compressed_in, header, client_tcp_protocol_version, - getFormatSettings(state.query_context)); + getNativeWireFormatSettings(state.query_context)); } } @@ -3074,7 +3075,7 @@ void TCPHandler::initBlockOutput(QueryState & state, const Block & block) *state.maybe_compressed_out, client_tcp_protocol_version, std::make_shared(block.cloneEmpty()), - getFormatSettings(state.query_context), + getNativeWireFormatSettings(state.query_context), !query_settings[Setting::low_cardinality_allow_in_native_format]); } } @@ -3098,7 +3099,7 @@ void TCPHandler::initLogsBlockOutput( *logs_buf, client_tcp_protocol_version, std::make_shared(block.cloneEmpty()), - getFormatSettings(state.query_context), + getNativeWireFormatSettings(state.query_context), !query_settings[Setting::low_cardinality_allow_in_native_format]); } } @@ -3117,7 +3118,7 @@ void TCPHandler::initProfileEventsBlockOutput(QueryState & state, const Block & const Settings & query_settings = state.query_context->getSettingsRef(); state.profile_events_block_out = std::make_unique( - *profile_events_buf, client_tcp_protocol_version, std::make_shared(block.cloneEmpty()), getFormatSettings(state.query_context), !query_settings[Setting::low_cardinality_allow_in_native_format]); + *profile_events_buf, client_tcp_protocol_version, std::make_shared(block.cloneEmpty()), getNativeWireFormatSettings(state.query_context), !query_settings[Setting::low_cardinality_allow_in_native_format]); } } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 8c4212893a830..e9e7eb9a22d55 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -303,7 +303,7 @@ class TCPHandler : public Poco::Net::TCPServerConnection void readTemporaryTables(QueryState & state) TSA_REQUIRES(callback_mutex); void skipData(QueryState & state) TSA_REQUIRES(callback_mutex); - bool processUnexpectedData(); + bool processUnexpectedData(QueryState & state); [[noreturn]] void processUnexpectedQuery(); [[noreturn]] void processUnexpectedHello(); [[noreturn]] void processUnexpectedTablesStatusRequest(); diff --git a/tests/integration/test_backward_compatibility/test_native_type_names_secondary_query.py b/tests/integration/test_backward_compatibility/test_native_type_names_secondary_query.py new file mode 100644 index 0000000000000..9a3221a5725eb --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_native_type_names_secondary_query.py @@ -0,0 +1,48 @@ +"""An older server queries a newer shard with the binary type encoding of the Native format enabled.""" + +import pytest + +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +new_node = cluster.add_instance("new_node") +old_node = cluster.add_instance( + "old_node", + image="clickhouse/clickhouse-server", + tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, + with_installed_binary=True, +) + +BINARY_TYPES = { + "output_format_native_encode_types_in_binary_format": 1, + "input_format_native_decode_types_in_binary_format": 1, +} + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_old_initiator_new_shard(start_cluster): + new_node.query("DROP TABLE IF EXISTS t_05317 SYNC") + new_node.query( + "CREATE TABLE t_05317 (x UInt64, s String) ENGINE = MergeTree ORDER BY x" + ) + + old_node.http_query( + "INSERT INTO FUNCTION remote('new_node', default, t_05317) VALUES (42, 'str')", + method="POST", + params=BINARY_TYPES, + ) + assert ( + old_node.http_query( + "SELECT x, s FROM remote('new_node', default, t_05317)", + params=BINARY_TYPES, + ) + == "42\tstr\n" + ) diff --git a/tests/queries/0_stateless/05317_native_binary_types_between_servers.reference b/tests/queries/0_stateless/05317_native_binary_types_between_servers.reference new file mode 100644 index 0000000000000..91b473e3133c5 --- /dev/null +++ b/tests/queries/0_stateless/05317_native_binary_types_between_servers.reference @@ -0,0 +1,22 @@ +remote +42 UInt64 +str String +parallel replicas +42 UInt64 +str String +1 +remote insert +7 +file read on a shard +5 +remote with server logs +42 UInt64 +str String +server logs received +secondary query from a client +42 UInt64 +str String +7 +8 +external data skipped after an exception +TOO_BIG_AST diff --git a/tests/queries/0_stateless/05317_native_binary_types_between_servers.sh b/tests/queries/0_stateless/05317_native_binary_types_between_servers.sh new file mode 100755 index 0000000000000..fe96095c637ec --- /dev/null +++ b/tests/queries/0_stateless/05317_native_binary_types_between_servers.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Server-to-server queries and a client's secondary query work with the binary type encoding of the Native format +# enabled, and a shard still applies these settings to the formats it reads itself. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +CLIENT_BINARY_TYPES="$CLICKHOUSE_CLIENT --output_format_native_encode_types_in_binary_format 1 --input_format_native_decode_types_in_binary_format 1" + +$CLIENT_BINARY_TYPES <<'EOF' +DROP TABLE IF EXISTS t_src; +DROP TABLE IF EXISTS t_dst; +CREATE TABLE t_src (d Dynamic) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_src VALUES (42::UInt64), ('str'); +CREATE TABLE t_dst (x UInt64) ENGINE = MergeTree ORDER BY x; + +SELECT 'remote'; +SELECT d, dynamicType(d) FROM remote('127.0.0.2', currentDatabase(), t_src) ORDER BY toString(d); + +SELECT 'parallel replicas'; +SELECT d, dynamicType(d) FROM t_src ORDER BY toString(d) +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, cluster_for_parallel_replicas = 'parallel_replicas', load_balancing = 'in_order', distributed_replica_max_ignored_errors = 1000, + parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_local_plan = 0, + automatic_parallel_replicas_mode = 0, enable_parallel_blocks_marshalling = 1, log_comment = '05317_parallel_replicas'; + +SYSTEM FLUSH LOGS query_log; +SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log +WHERE event_date >= yesterday() AND current_database = currentDatabase() AND log_comment = '05317_parallel_replicas' + AND type = 'QueryFinish' AND query_id = initial_query_id +SETTINGS enable_parallel_replicas = 0; + +SELECT 'remote insert'; +INSERT INTO FUNCTION remote('127.0.0.2', currentDatabase(), t_dst) VALUES (7); +SELECT x FROM t_dst; + +SELECT 'file read on a shard'; +INSERT INTO FUNCTION file(currentDatabase() || '_05317.native', Native) SELECT 5::UInt64 AS x SETTINGS engine_file_truncate_on_insert = 1; +SELECT x FROM remote('127.0.0.2', file(currentDatabase() || '_05317.native', Native)); +EOF + +echo 'remote with server logs' +$CLIENT_BINARY_TYPES --server_logs_file="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_05317_server_logs.txt" -q "SELECT d, dynamicType(d) FROM remote('127.0.0.2', currentDatabase(), t_src) ORDER BY toString(d) SETTINGS send_logs_level = 'trace'" +[ -s "${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_05317_server_logs.txt" ] && echo 'server logs received' +rm -f "${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_05317_server_logs.txt" + +echo 'secondary query from a client' +$CLIENT_BINARY_TYPES --query_kind secondary_query -q "INSERT INTO t_dst VALUES (8)" +$CLIENT_BINARY_TYPES --query_kind secondary_query -q "SELECT d, dynamicType(d) FROM t_src ORDER BY toString(d) SETTINGS enable_parallel_replicas = 0" +$CLICKHOUSE_CLIENT -q "SELECT x FROM t_dst ORDER BY x" + +echo 'external data skipped after an exception' +seq 1 1000000 | $CLIENT_BINARY_TYPES --max_ast_elements 2 --external --file=- --name=ext --structure='x UInt64' -q "SELECT x FROM ext" 2>&1 | grep -o -m1 'TOO_BIG_AST' + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_src; DROP TABLE t_dst" From 7f06edeb38533a1cde5b860a81960de046132509 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 13 Oct 2026 09:59:40 +0200 Subject: [PATCH 278/296] Revert "Merge pull request #121187 from ClickHouse/backport/26.8/116724" This reverts commit bedf2ab54b8a0c34afbf2d907eaf933324f43cd6, reversing changes made to 5efdd4fdb4431f29751b5e4c323c794a744bf699. --- base/poco/NetSSL_OpenSSL/src/SSLManager.cpp | 4 +- .../_server_settings_outside_source.mdx | 2 +- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 27 +------- src/Core/ServerSettings.cpp | 2 +- .../configs/ssl_config_ca_signed.xml | 15 ---- .../test_accept_invalid_certificate/test.py | 69 ------------------- .../test_acme_tls/configs/config.xml | 3 - .../test_acme_tls/configs/config_multi.xml | 3 - .../configs/config.d/ssl_conf.xml | 3 - .../configs/generate_certs.sh | 14 ---- .../configs/server.crt | 34 ++++----- .../configs/server.key | 52 +++++++------- .../configs/ssl_conf_verify.xml | 23 ------- .../test_keeper_secure_client/test.py | 16 ----- .../test_ssl_cert_authentication/test.py | 2 - 15 files changed, 47 insertions(+), 222 deletions(-) delete mode 100644 tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml delete mode 100755 tests/integration/test_keeper_secure_client/configs/generate_certs.sh delete mode 100644 tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml diff --git a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp index 80d85855fe3c9..94b66cdcf5f1e 100644 --- a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp @@ -340,9 +340,7 @@ void SSLManager::initDefaultContext(bool server) { _ptrDefaultClientContext->enableSessionCache(cacheSessions); } - /// Only an outbound connection has a requested host name to check: on an accepted socket this - /// would run against the client's address, which a client certificate does not name. - bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, !server); + bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, false); if (server) _ptrDefaultServerContext->enableExtendedCertificateVerification(extendedVerification); else diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 71a2cfa7fc796..12b0fecf8bb76 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1170,7 +1170,7 @@ Keys for server/client settings: | `certificateFile` | Path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contains the certificate. | | | `cipherList` | Supported OpenSSL encryptions. | `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH` | | `disableProtocols` | Protocols that are not allowed to be used. | | -| `extendedVerification` | If enabled, verify that the certificate CN or SAN matches the peer hostname. | `true` for `openSSL.client`, `false` for `openSSL.server` | +| `extendedVerification` | If enabled, verify that the certificate CN or SAN matches the peer hostname. | `false` | | `fips` | Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. | `false` | | `invalidCertificateHandler` | Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` RejectCertificateHandler ` . | `RejectCertificateHandler` | | `loadDefaultCAFile` | Wether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `/etc/ssl/cert.pem` (resp. the directory `/etc/ssl/certs`) or in file (resp. directory) specified by the environment variable `SSL_CERT_FILE` (resp. `SSL_CERT_DIR`). | `true` | diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 039ff73ba69c7..db582884434c4 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -372,25 +372,6 @@ void triggerWatchCallback( } } -#if USE_SSL -/// The host part of ":", split lexically at the same character Poco::Net::SocketAddress -/// splits it at, so that a port spelled as a service name still resolves. The result is matched -/// against a certificate, so an IPv6 literal loses its brackets; a shape naming no host is empty. -std::string peerHostName(const std::string & host_and_port) -{ - if (host_and_port.starts_with('/')) - return {}; - - if (host_and_port.starts_with('[')) - { - size_t closing_bracket = host_and_port.find(']'); - return closing_bracket == std::string::npos ? std::string{} : host_and_port.substr(1, closing_bracket - 1); - } - - return host_and_port.substr(0, host_and_port.find(':')); -} -#endif - } template @@ -636,13 +617,7 @@ void ZooKeeper::connect( if (node.secure) { #if USE_SSL - auto secure_socket = Poco::Net::SecureStreamSocket(); - /// The certificate names the configured host while the socket connects to the - /// address it resolved to, so the name has to be carried explicitly. This is - /// also what puts the host into the SNI extension. - if (const auto peer_host_name = peerHostName(node.host); !peer_host_name.empty()) - secure_socket.setPeerHostName(peer_host_name); - socket = secure_socket; + socket = Poco::Net::SecureStreamSocket(); #else throw Poco::Exception( "Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support."); diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 209c0521f77b5..c9e35783f3b8b 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -1912,7 +1912,7 @@ Configured as `named_collections_storage.type` (`` (resp. the directory ``) or in file (resp. directory) specified by the environment variable `` (resp. ``).)", 0, "openSSL.client.loadDefaultCAFile") \ DECLARE(String, openssl_client_chipher_list, "ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH", R"(Supported OpenSSL encryptions.)", 0, "openSSL.client.cipherList") \ DECLARE(Bool, openssl_client_cache_sessions, false, R"(Enables or disables caching sessions. Must be used in combination with ``. Acceptable values: ``, ``.)", 0, "openSSL.client.cacheSessions") \ - DECLARE(Bool, openssl_client_extended_verification, true, R"(If enabled, verify that the certificate CN or SAN matches the peer hostname.)", 0, "openSSL.client.extendedVerification") \ + DECLARE(Bool, openssl_client_extended_verification, false, R"(If enabled, verify that the certificate CN or SAN matches the peer hostname.)", 0, "openSSL.client.extendedVerification") \ DECLARE(Bool, openssl_client_required_tls_v1, false, R"(Require a TLSv1 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1") \ DECLARE(Bool, openssl_client_required_tls_v1_1, false, R"(Require a TLSv1.1 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1_1") \ DECLARE(Bool, openssl_client_required_tls_v1_2, false, R"(Require a TLSv1.2 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1_2") \ diff --git a/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml b/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml deleted file mode 100644 index b1dccbb15c66e..0000000000000 --- a/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - 9440 - - - - /etc/clickhouse-server/config.d/client-cert.pem - /etc/clickhouse-server/config.d/client-key.pem - /etc/clickhouse-server/config.d/ca-cert.pem - none - - - diff --git a/tests/integration/test_accept_invalid_certificate/test.py b/tests/integration/test_accept_invalid_certificate/test.py index b66f311b92355..afea8011c4b99 100644 --- a/tests/integration/test_accept_invalid_certificate/test.py +++ b/tests/integration/test_accept_invalid_certificate/test.py @@ -9,7 +9,6 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) MAX_RETRY = 5 -CA_CERT = f"{SCRIPT_DIR}/certs/ca-cert.pem" cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( @@ -36,18 +35,6 @@ ) -node2 = cluster.add_instance( - "node2", - main_configs=[ - "configs/ssl_config_ca_signed.xml", - "certs/client-key.pem", - "certs/client-cert.pem", - "certs/ca-cert.pem", - ], - with_zookeeper=False, -) - - @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: @@ -74,36 +61,6 @@ def started_cluster(): """ -# node2 presents a certificate this CA signs, so the chain is valid and only the name can fail. -config_ca_signed = """ - - - {caConfig} - false - - -""" - -config_ca_signed_sni_override = """ - client - - - {caConfig} - false - - -""" - -config_ca_signed_no_extended_verification = """ - - - {caConfig} - false - false - - -""" - def execute_query_native(node, query, config): fd, config_path = tempfile.mkstemp( @@ -175,29 +132,3 @@ def test_strict_connection_reject(): config_connection_accept.format(ip_address=f"{instance.ip_address}"), ) assert "certificate verify failed" in str(err.value) - - -def test_hostname_mismatch_rejected_by_default(): - with pytest.raises(Exception) as err: - execute_query_native(node2, "SELECT 1", config_ca_signed.format(caConfig=CA_CERT)) - assert "Unacceptable certificate" in str(err.value) - - -def test_hostname_match_accepted(): - assert ( - execute_query_native( - node2, "SELECT 1", config_ca_signed_sni_override.format(caConfig=CA_CERT) - ) - == "1\n" - ) - - -def test_extended_verification_disabled(): - assert ( - execute_query_native( - node2, - "SELECT 1", - config_ca_signed_no_extended_verification.format(caConfig=CA_CERT), - ) - == "1\n" - ) diff --git a/tests/integration/test_acme_tls/configs/config.xml b/tests/integration/test_acme_tls/configs/config.xml index 0977ed12d7114..4594c72e885e3 100644 --- a/tests/integration/test_acme_tls/configs/config.xml +++ b/tests/integration/test_acme_tls/configs/config.xml @@ -14,9 +14,6 @@ - - false AcceptCertificateHandler diff --git a/tests/integration/test_acme_tls/configs/config_multi.xml b/tests/integration/test_acme_tls/configs/config_multi.xml index 9cda6eb0ba878..56c9d415886c4 100644 --- a/tests/integration/test_acme_tls/configs/config_multi.xml +++ b/tests/integration/test_acme_tls/configs/config_multi.xml @@ -14,9 +14,6 @@ - - false AcceptCertificateHandler diff --git a/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml b/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml index e51749f354385..c11c5736d96f8 100644 --- a/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml +++ b/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml @@ -7,9 +7,6 @@ /etc/clickhouse-server/config.d/server.key - - false AcceptCertificateHandler diff --git a/tests/integration/test_keeper_secure_client/configs/generate_certs.sh b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh deleted file mode 100755 index e398289b8cc9b..0000000000000 --- a/tests/integration/test_keeper_secure_client/configs/generate_certs.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Self-signed certificate that Keeper presents on its secure port. It is also its own trust -# anchor (CA:TRUE), so a client can point caConfig at it directly. -# -# The subjectAltName has to list every name a client may connect to: `node1` is the cluster -# member name used in use_secure_keeper.xml, and `localhost` is kept because the raft -# configuration and the existing arms use it. Once a dNSName SAN is present OpenSSL ignores -# the common name, so localhost must be listed explicitly rather than left to CN. -openssl req -newkey rsa:2048 -x509 -days 36500 -nodes -batch \ - -keyout server.key -out server.crt \ - -subj "/CN=localhost" \ - -addext "subjectAltName=DNS:localhost,DNS:node1" \ - -addext "basicConstraints=critical,CA:TRUE" diff --git a/tests/integration/test_keeper_secure_client/configs/server.crt b/tests/integration/test_keeper_secure_client/configs/server.crt index d9ad8941d6158..7ade2d9627338 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.crt +++ b/tests/integration/test_keeper_secure_client/configs/server.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDKDCCAhCgAwIBAgIUBTZCvmMS+9yaX8IRKGkGqbcMBOIwDQYJKoZIhvcNAQEL -BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI2MDgyNzExMzMzOFoYDzIxMjYw -ODAzMTEzMzM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQCyOozYThV2lJBPteVCm7YMVJ/w4hralGidnbkpGZxQ -OnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6EBM0RYv7SQtcoUG2YErKHuASHSSc -DdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxhUFROGgguoIdVOKUtDwib/mq27048 -+uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUGle4rMlAR8oTDBmHlrowoEOjN/mwd -WnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUItvXreLuvdt1wQoCC1bH6h6K3ymsu8 -1xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCFXtunioRbAgMBAAGjcDBuMB0GA1Ud -DgQWBBQP0fRsGOqD4ZxQ1oTe7v9HIzc3RTAfBgNVHSMEGDAWgBQP0fRsGOqD4ZxQ -1oTe7v9HIzc3RTAbBgNVHREEFDASgglsb2NhbGhvc3SCBW5vZGUxMA8GA1UdEwEB -/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGw73J/dlsYphCnXZzVfOWiwFtRX -ij9M3/GjtCfmfh2MriYcNlCGqw+7GvzUUJ44rKjRxWyF+FfVM7DC4O0a6oimvtwE -yhb9mFshGDUpMX6WtpjhFGDtNjbgXrz6FXkt8U5SQayVoVLQjIT6XZSmXEccMJpM -U3lGAqPaeMVYVZqQPcvdO+t+Jfrjjznpa66biD87BvJ4CCWNynWi/ftpK76bk1fj -SbL0RgG1SUBAJikJirL2OW+ADgCyPxP86IgTKdOV92e+00spcfvSlms7UzrChXA7 -Kv0Z/9WUgL3tL87kiP0eG3XaZp83w+WS0OOcaU8uQ0AhofzWXMnQT0Bgq/8= +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_secure_client/configs/server.key b/tests/integration/test_keeper_secure_client/configs/server.key index 390ae5e9995f6..f0fb61ac443fa 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.key +++ b/tests/integration/test_keeper_secure_client/configs/server.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEugIBADANBgkqhkiG9w0BAQEFAASCBKQwggSgAgEAAoIBAQCyOozYThV2lJBP -teVCm7YMVJ/w4hralGidnbkpGZxQOnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6 -EBM0RYv7SQtcoUG2YErKHuASHSScDdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxh -UFROGgguoIdVOKUtDwib/mq27048+uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUG -le4rMlAR8oTDBmHlrowoEOjN/mwdWnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUIt -vXreLuvdt1wQoCC1bH6h6K3ymsu81xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCF -XtunioRbAgMBAAECgf9xetYy44TG+9Dm7+oJfxtqvncX6N8wTU5PDR3YDXZ5nrz+ -q2ZcE2+A2RdE52nDoI2wT0u3Vw1k4K+VSKbxoVZ/qQKS+BnfBXCZQQ9qeSPWOPWY -70CQhZOwjfp2oY8v/3a6DoYp74zPRCutynfjQq/zAEMq/+Yrymaw7E0GLVijehE7 -eLhAFL/1IAjzvr5ikaN2MgSGHwcZ7XNpTN0Lvpz1PH2LkpDmh7PWUKcvu7nV3cQc -x8BkEFFaqvlXq0NfgPNCZ9qL9/Y+VrJjv0Bta7KCO/6nbmJubUGVZbzv0qwOJBVB -/3iki0BoEG2YJzJLuKnJJ0Ugf6usNai1NY9PKtkCgYEA3V+QJzcL0fWrvw6DBP0m -WWv1afZGr2ONjHmz65IvegFpt9qmm2CnRVzFPXet1VJkxK+MNFiPs+h29s1d2G7/ -VdwU5+b2KdQkgG9QKWsRv9bkNUqK48LWvLYkUCNhYW7oxCH4SxBNbSdVDUpXnZJ/ -u6T71YRgY9yYRI6HnHrJr40CgYEAzhtZ7phK4inHokcXWLJBhYy6HwiENzNUlEm9 -dLRTeExzrdb/zA8WCPc+Vb1dSoNa0l/5TFfyoRpZ2RsfX9iGZIR2GiQhC8ftK2XH -ZynQBN+rGNBoarWhPfpyMU8g5pHCIxChOJaMTA5sb52oHE5D03oQXa0jUQDS7ie8 -IIUh9YcCgYB2I8zLkM3nFAO/J6B2Zh80M7A6B2t7qFZfNIf3XQGnS9++dZraav95 -veOpRRhpMtqCOOlK+kRej94FIl5F5h6wFI63oAOZGRUN0dnm8LP9PFJ3ujtbB50Q -AuhoOCl9FPJ10WcvsBiR+x6hxB30Ar3WR1r6fPXv9Ubxt1raAJFn9QKBgCCNg1vC -4/jqnmRMFCznIqAkRPVH1BIf4lj1eJ3pKVHTyWWIkqg5LcLGwiqqmQR+0KZvkCF1 -tpWpYs1+kisCON/BaCHi2PFSZ2w2TBNIMFnEtfJaYFOSUmBGoSiWldy04tPE+aHF -bW1JzeinHvkxh1bsbY9OHEjb93S0A+ui+2UbAoGADWmYVq4ksRSrHMa/9gwqGC5k -bWx791nrTf2UKlpsBA2b7kQ76DLMzgPGzCAGo5condd65y/4oP/Mr2+mEZkDu+Dh -W8jCsFN9RW2XP4ekgJj41hKAyKedP7JkbBb7pznvtqq1oLndIKN00jjrlfIbsKD0 -Reoz9LQnDowqs/fGhh8= +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= -----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml deleted file mode 100644 index 55a744c632f5a..0000000000000 --- a/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - /etc/clickhouse-server/config.d/server.crt - /etc/clickhouse-server/config.d/server.key - none - true - sslv2,sslv3 - - - - /etc/clickhouse-server/config.d/server.crt - false - relaxed - true - sslv2,sslv3 - - RejectCertificateHandler - - - - diff --git a/tests/integration/test_keeper_secure_client/test.py b/tests/integration/test_keeper_secure_client/test.py index fe8f65aa7bde8..1f00e68810287 100644 --- a/tests/integration/test_keeper_secure_client/test.py +++ b/tests/integration/test_keeper_secure_client/test.py @@ -24,16 +24,6 @@ "configs/server.key", ], ) -# Same secure Keeper, but this client actually verifies the certificate it is presented. -node3 = cluster.add_instance( - "node3", - main_configs=[ - "configs/use_secure_keeper.xml", - "configs/ssl_conf_verify.xml", - "configs/server.crt", - "configs/server.key", - ], -) @pytest.fixture(scope="module") @@ -50,9 +40,3 @@ def started_cluster(): def test_connection(started_cluster): # just nothrow node2.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") - - -def test_connection_verifying_certificate(started_cluster): - # The socket connects to the address node1 resolves to, so this only succeeds if - # the certificate is matched against `node1` rather than against that address. - node3.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 118172905b9a9..ca4e6147f928b 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -44,7 +44,6 @@ def started_cluster(): config = """ - {sslHost} strict @@ -63,7 +62,6 @@ def execute_query_native(node, query, user, cert_name, password=None): certificateFile=f"{SCRIPT_DIR}/certs/{cert_name}-cert.pem", privateKeyFile=f"{SCRIPT_DIR}/certs/{cert_name}-key.pem", caConfig=f"{SCRIPT_DIR}/certs/ca-cert.pem", - sslHost=SSL_HOST, ) file = open(config_path, "w") From 17ef135f8aae2b93fe27a8f38b058d8ec09eb7c0 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 13 Oct 2026 09:59:50 +0200 Subject: [PATCH 279/296] Keep passing the Keeper host name to the secure socket Same as on master: it is correct with either `extendedVerification` default and only adds SNI while verification is off. --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 27 +++++++++- .../configs/generate_certs.sh | 14 +++++ .../configs/server.crt | 34 ++++++------ .../configs/server.key | 52 +++++++++---------- .../configs/ssl_conf_verify.xml | 23 ++++++++ .../test_keeper_secure_client/test.py | 16 ++++++ 6 files changed, 122 insertions(+), 44 deletions(-) create mode 100755 tests/integration/test_keeper_secure_client/configs/generate_certs.sh create mode 100644 tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index db582884434c4..039ff73ba69c7 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -372,6 +372,25 @@ void triggerWatchCallback( } } +#if USE_SSL +/// The host part of ":", split lexically at the same character Poco::Net::SocketAddress +/// splits it at, so that a port spelled as a service name still resolves. The result is matched +/// against a certificate, so an IPv6 literal loses its brackets; a shape naming no host is empty. +std::string peerHostName(const std::string & host_and_port) +{ + if (host_and_port.starts_with('/')) + return {}; + + if (host_and_port.starts_with('[')) + { + size_t closing_bracket = host_and_port.find(']'); + return closing_bracket == std::string::npos ? std::string{} : host_and_port.substr(1, closing_bracket - 1); + } + + return host_and_port.substr(0, host_and_port.find(':')); +} +#endif + } template @@ -617,7 +636,13 @@ void ZooKeeper::connect( if (node.secure) { #if USE_SSL - socket = Poco::Net::SecureStreamSocket(); + auto secure_socket = Poco::Net::SecureStreamSocket(); + /// The certificate names the configured host while the socket connects to the + /// address it resolved to, so the name has to be carried explicitly. This is + /// also what puts the host into the SNI extension. + if (const auto peer_host_name = peerHostName(node.host); !peer_host_name.empty()) + secure_socket.setPeerHostName(peer_host_name); + socket = secure_socket; #else throw Poco::Exception( "Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support."); diff --git a/tests/integration/test_keeper_secure_client/configs/generate_certs.sh b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh new file mode 100755 index 0000000000000..e398289b8cc9b --- /dev/null +++ b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Self-signed certificate that Keeper presents on its secure port. It is also its own trust +# anchor (CA:TRUE), so a client can point caConfig at it directly. +# +# The subjectAltName has to list every name a client may connect to: `node1` is the cluster +# member name used in use_secure_keeper.xml, and `localhost` is kept because the raft +# configuration and the existing arms use it. Once a dNSName SAN is present OpenSSL ignores +# the common name, so localhost must be listed explicitly rather than left to CN. +openssl req -newkey rsa:2048 -x509 -days 36500 -nodes -batch \ + -keyout server.key -out server.crt \ + -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,DNS:node1" \ + -addext "basicConstraints=critical,CA:TRUE" diff --git a/tests/integration/test_keeper_secure_client/configs/server.crt b/tests/integration/test_keeper_secure_client/configs/server.crt index 7ade2d9627338..d9ad8941d6158 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.crt +++ b/tests/integration/test_keeper_secure_client/configs/server.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow -FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI -4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T -4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU -7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj -sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg -pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC -kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j -4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr -85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C -L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD -+UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L -P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp -0g== +MIIDKDCCAhCgAwIBAgIUBTZCvmMS+9yaX8IRKGkGqbcMBOIwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI2MDgyNzExMzMzOFoYDzIxMjYw +ODAzMTEzMzM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCyOozYThV2lJBPteVCm7YMVJ/w4hralGidnbkpGZxQ +OnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6EBM0RYv7SQtcoUG2YErKHuASHSSc +DdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxhUFROGgguoIdVOKUtDwib/mq27048 ++uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUGle4rMlAR8oTDBmHlrowoEOjN/mwd +WnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUItvXreLuvdt1wQoCC1bH6h6K3ymsu8 +1xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCFXtunioRbAgMBAAGjcDBuMB0GA1Ud +DgQWBBQP0fRsGOqD4ZxQ1oTe7v9HIzc3RTAfBgNVHSMEGDAWgBQP0fRsGOqD4ZxQ +1oTe7v9HIzc3RTAbBgNVHREEFDASgglsb2NhbGhvc3SCBW5vZGUxMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGw73J/dlsYphCnXZzVfOWiwFtRX +ij9M3/GjtCfmfh2MriYcNlCGqw+7GvzUUJ44rKjRxWyF+FfVM7DC4O0a6oimvtwE +yhb9mFshGDUpMX6WtpjhFGDtNjbgXrz6FXkt8U5SQayVoVLQjIT6XZSmXEccMJpM +U3lGAqPaeMVYVZqQPcvdO+t+Jfrjjznpa66biD87BvJ4CCWNynWi/ftpK76bk1fj +SbL0RgG1SUBAJikJirL2OW+ADgCyPxP86IgTKdOV92e+00spcfvSlms7UzrChXA7 +Kv0Z/9WUgL3tL87kiP0eG3XaZp83w+WS0OOcaU8uQ0AhofzWXMnQT0Bgq/8= -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_secure_client/configs/server.key b/tests/integration/test_keeper_secure_client/configs/server.key index f0fb61ac443fa..390ae5e9995f6 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.key +++ b/tests/integration/test_keeper_secure_client/configs/server.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn -DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L -OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus -TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 -JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F -KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 -ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP -TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f -7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN -MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 -212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu -plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr -/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu -Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa -CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g -1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 -bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq -cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez -on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K -GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 -nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU -2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh -vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa -dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr -vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 -KCBtmIrQLqzMpnBpLNuSY+Q= +MIIEugIBADANBgkqhkiG9w0BAQEFAASCBKQwggSgAgEAAoIBAQCyOozYThV2lJBP +teVCm7YMVJ/w4hralGidnbkpGZxQOnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6 +EBM0RYv7SQtcoUG2YErKHuASHSScDdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxh +UFROGgguoIdVOKUtDwib/mq27048+uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUG +le4rMlAR8oTDBmHlrowoEOjN/mwdWnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUIt +vXreLuvdt1wQoCC1bH6h6K3ymsu81xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCF +XtunioRbAgMBAAECgf9xetYy44TG+9Dm7+oJfxtqvncX6N8wTU5PDR3YDXZ5nrz+ +q2ZcE2+A2RdE52nDoI2wT0u3Vw1k4K+VSKbxoVZ/qQKS+BnfBXCZQQ9qeSPWOPWY +70CQhZOwjfp2oY8v/3a6DoYp74zPRCutynfjQq/zAEMq/+Yrymaw7E0GLVijehE7 +eLhAFL/1IAjzvr5ikaN2MgSGHwcZ7XNpTN0Lvpz1PH2LkpDmh7PWUKcvu7nV3cQc +x8BkEFFaqvlXq0NfgPNCZ9qL9/Y+VrJjv0Bta7KCO/6nbmJubUGVZbzv0qwOJBVB +/3iki0BoEG2YJzJLuKnJJ0Ugf6usNai1NY9PKtkCgYEA3V+QJzcL0fWrvw6DBP0m +WWv1afZGr2ONjHmz65IvegFpt9qmm2CnRVzFPXet1VJkxK+MNFiPs+h29s1d2G7/ +VdwU5+b2KdQkgG9QKWsRv9bkNUqK48LWvLYkUCNhYW7oxCH4SxBNbSdVDUpXnZJ/ +u6T71YRgY9yYRI6HnHrJr40CgYEAzhtZ7phK4inHokcXWLJBhYy6HwiENzNUlEm9 +dLRTeExzrdb/zA8WCPc+Vb1dSoNa0l/5TFfyoRpZ2RsfX9iGZIR2GiQhC8ftK2XH +ZynQBN+rGNBoarWhPfpyMU8g5pHCIxChOJaMTA5sb52oHE5D03oQXa0jUQDS7ie8 +IIUh9YcCgYB2I8zLkM3nFAO/J6B2Zh80M7A6B2t7qFZfNIf3XQGnS9++dZraav95 +veOpRRhpMtqCOOlK+kRej94FIl5F5h6wFI63oAOZGRUN0dnm8LP9PFJ3ujtbB50Q +AuhoOCl9FPJ10WcvsBiR+x6hxB30Ar3WR1r6fPXv9Ubxt1raAJFn9QKBgCCNg1vC +4/jqnmRMFCznIqAkRPVH1BIf4lj1eJ3pKVHTyWWIkqg5LcLGwiqqmQR+0KZvkCF1 +tpWpYs1+kisCON/BaCHi2PFSZ2w2TBNIMFnEtfJaYFOSUmBGoSiWldy04tPE+aHF +bW1JzeinHvkxh1bsbY9OHEjb93S0A+ui+2UbAoGADWmYVq4ksRSrHMa/9gwqGC5k +bWx791nrTf2UKlpsBA2b7kQ76DLMzgPGzCAGo5condd65y/4oP/Mr2+mEZkDu+Dh +W8jCsFN9RW2XP4ekgJj41hKAyKedP7JkbBb7pznvtqq1oLndIKN00jjrlfIbsKD0 +Reoz9LQnDowqs/fGhh8= -----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml new file mode 100644 index 0000000000000..55a744c632f5a --- /dev/null +++ b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml @@ -0,0 +1,23 @@ + + + + /etc/clickhouse-server/config.d/server.crt + /etc/clickhouse-server/config.d/server.key + none + true + sslv2,sslv3 + + + + /etc/clickhouse-server/config.d/server.crt + false + relaxed + true + sslv2,sslv3 + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_keeper_secure_client/test.py b/tests/integration/test_keeper_secure_client/test.py index 1f00e68810287..fe8f65aa7bde8 100644 --- a/tests/integration/test_keeper_secure_client/test.py +++ b/tests/integration/test_keeper_secure_client/test.py @@ -24,6 +24,16 @@ "configs/server.key", ], ) +# Same secure Keeper, but this client actually verifies the certificate it is presented. +node3 = cluster.add_instance( + "node3", + main_configs=[ + "configs/use_secure_keeper.xml", + "configs/ssl_conf_verify.xml", + "configs/server.crt", + "configs/server.key", + ], +) @pytest.fixture(scope="module") @@ -40,3 +50,9 @@ def started_cluster(): def test_connection(started_cluster): # just nothrow node2.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") + + +def test_connection_verifying_certificate(started_cluster): + # The socket connects to the address node1 resolves to, so this only succeeds if + # the certificate is matched against `node1` rather than against that address. + node3.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") From 6c68808da6217c6be56fad5f07fb10c9237af7e6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 09:33:45 +0000 Subject: [PATCH 280/296] Backport #125253 to 26.8: Sort the rows of test_storage_rabbitmq row-format tests before comparing them --- tests/integration/test_storage_rabbitmq/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 9f02e5fbe90e1..6b4f639cc6b4c 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -2796,7 +2796,7 @@ def onReceived(channel, method, properties, body): check_expected_result_polling(num_rows, f"SELECT count() FROM {db}.view") - result = instance.query(f"SELECT * FROM {db}.view") + result = instance.query(f"SELECT * FROM {db}.view ORDER BY key") assert result == "0\t0\n10\t100\n20\t200\n30\t300\n40\t400\n" @@ -2884,7 +2884,7 @@ def onReceived(channel, method, properties, body): for i in range(num_rows): expected += str(i * 10) + "\t" + str(i * 100) + "\n" - result = instance.query(f"SELECT * FROM {db}.view") + result = instance.query(f"SELECT * FROM {db}.view ORDER BY key") assert result == expected From d869be12791e0fd6b88cb594254b7b424b96f297 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 10:12:08 +0000 Subject: [PATCH 281/296] Backport #125165 to 26.8: Do not wait for a running statistics refresh when a MergeTree table is shut down --- src/Storages/MergeTree/MergeTreeData.cpp | 9 +++ src/Storages/MergeTree/MergeTreeData.h | 2 + src/Storages/StorageMergeTree.cpp | 3 + src/Storages/StorageReplicatedMergeTree.cpp | 3 + ...tics_refresh_does_not_block_drop.reference | 4 ++ ..._statistics_refresh_does_not_block_drop.sh | 66 +++++++++++++++++++ 6 files changed, 87 insertions(+) create mode 100644 tests/queries/0_stateless/05361_statistics_refresh_does_not_block_drop.reference create mode 100755 tests/queries/0_stateless/05361_statistics_refresh_does_not_block_drop.sh diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ba93e48c657b6..48713cf6d3093 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3243,9 +3243,13 @@ void MergeTreeData::startStatisticsCache() const auto settings = getSettings(); UInt64 refresh_statistics_seconds = (*settings)[MergeTreeSetting::refresh_statistics_interval].totalSeconds(); if (refresh_stats_task) + { + refresh_stats_cancelled = true; refresh_stats_task->deactivate(); + } if (refresh_statistics_seconds) { + refresh_stats_cancelled = false; LOG_INFO(log, "Start to refresh statistics"); refresh_stats_task = getContext()->getSchedulePool()->createTask( getStorageID(), "MergeTreeData::refreshStatistics", @@ -3399,6 +3403,8 @@ try ConditionSelectivityEstimatorBuilder estimator_builder(getContext()); for (const DataPartPtr & data_part : data_parts) { + if (refresh_stats_cancelled) + return; auto parts_lock = readLockParts(); auto stats = data_part->loadStatistics(); estimator_builder.markDataPart(data_part); @@ -3435,7 +3441,10 @@ MergeTreeData::~MergeTreeData() if (refresh_parts_task) refresh_parts_task->deactivate(); if (refresh_stats_task) + { + refresh_stats_cancelled = true; refresh_stats_task->deactivate(); + } } catch (...) { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 5b199b78659ee..1e48e4e85f6a7 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -2086,6 +2086,8 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr std::mutex refresh_parts_mutex; BackgroundSchedulePoolTaskHolder refresh_stats_task; + /// Set before `refresh_stats_task` is deactivated, so that a running `refreshStatistics` returns between parts. + std::atomic_bool refresh_stats_cancelled = false; mutable std::mutex stats_mutex; ConditionSelectivityEstimatorPtr cached_estimator; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index acc8054103e66..4b05914c42094 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -312,7 +312,10 @@ void StorageMergeTree::shutdown(bool) refresh_parts_task->deactivate(); if (refresh_stats_task) + { + refresh_stats_cancelled = true; refresh_stats_task->deactivate(); + } stopOutdatedAndUnexpectedDataPartsLoadingTask(); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 5637e4431ebe3..561a9eb4d24e6 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6124,7 +6124,10 @@ void StorageReplicatedMergeTree::shutdown(bool) if (refresh_parts_task) refresh_parts_task->deactivate(); if (refresh_stats_task) + { + refresh_stats_cancelled = true; refresh_stats_task->deactivate(); + } flushAndPrepareForShutdown(); diff --git a/tests/queries/0_stateless/05361_statistics_refresh_does_not_block_drop.reference b/tests/queries/0_stateless/05361_statistics_refresh_does_not_block_drop.reference new file mode 100644 index 0000000000000..b35e5a2f571b0 --- /dev/null +++ b/tests/queries/0_stateless/05361_statistics_refresh_does_not_block_drop.reference @@ -0,0 +1,4 @@ +24 +24 +alter 1 +drop 1 diff --git a/tests/queries/0_stateless/05361_statistics_refresh_does_not_block_drop.sh b/tests/queries/0_stateless/05361_statistics_refresh_does_not_block_drop.sh new file mode 100755 index 0000000000000..4f076e60e205a --- /dev/null +++ b/tests/queries/0_stateless/05361_statistics_refresh_does_not_block_drop.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-shared-merge-tree, no-replicated-database +# Tag no-fasttest: requires S3 +# Tag no-shared-merge-tree, no-replicated-database: the tables use their own ad hoc S3 disks + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Each table has its own disk, which serves 2 GET requests per second. A statistics refresh reads each of the 5 +# statistics of a part with its own request, so a refresh of 24 parts takes at least 60 seconds. Neither ALTER of +# `refresh_statistics_interval` nor DROP may wait for a refresh in flight. +# HEAD requests count against that limit too, so the disk skips its access check (done under a server-wide lock) and +# the inserts disable `s3_check_objects_after_upload` (enabled in the test profile). + +function create_table() +{ + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $1 (k UInt64, v1 UInt64 STATISTICS(basic), v2 UInt64 STATISTICS(basic), v3 UInt64 STATISTICS(basic), + v4 UInt64 STATISTICS(basic), v5 UInt64 STATISTICS(basic)) ENGINE = MergeTree ORDER BY k + SETTINGS refresh_statistics_interval = 0, auto_statistics_types = '', + disk = disk(name = '${CLICKHOUSE_TEST_UNIQUE_NAME}_$1', type = object_storage, object_storage_type = s3, + metadata_type = local, endpoint = 'http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}/$1/', + access_key_id = clickhouse, secret_access_key = clickhouse, s3_max_get_rps = 2, s3_max_get_burst = 1, + skip_access_check = 1)" + $CLICKHOUSE_CLIENT -q "SYSTEM STOP MERGES $1" + $CLICKHOUSE_CLIENT -q " + INSERT INTO $1 SELECT number, number, number, number, number, number FROM numbers(24) + SETTINGS max_block_size = 1, min_insert_block_size_rows = 1, min_insert_block_size_bytes = 0, + max_insert_threads = 1, materialize_statistics_on_insert = 1, materialize_statistics_on_insert_max_table_size = 0, + s3_check_objects_after_upload = 0" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = '$1' AND active" +} + +function wait_for_refresh() +{ + for _ in {1..600}; do + [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.background_schedule_pool + WHERE database = currentDatabase() AND table = '$1' AND log_name = 'MergeTreeData::refreshStatistics' + AND executing AND elapsed_ms > 1000") == 1 ]] && return + sleep 0.1 + done + echo "the statistics refresh of $1 is not running" +} + +create_table t_alter +create_table t_drop + +$CLICKHOUSE_CLIENT -q "ALTER TABLE t_alter MODIFY SETTING refresh_statistics_interval = 300" +wait_for_refresh t_alter +$CLICKHOUSE_CLIENT --query_id "${CLICKHOUSE_TEST_UNIQUE_NAME}_alter" -q "ALTER TABLE t_alter MODIFY SETTING refresh_statistics_interval = 299" +# The interrupted refresh published nothing, so the task armed by the ALTER refreshes again. +wait_for_refresh t_alter + +$CLICKHOUSE_CLIENT -q "ALTER TABLE t_drop MODIFY SETTING refresh_statistics_interval = 300" +wait_for_refresh t_drop +$CLICKHOUSE_CLIENT --query_id "${CLICKHOUSE_TEST_UNIQUE_NAME}_drop" -q "DROP TABLE t_drop SYNC" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_alter SYNC" + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT -q " + SELECT substring(query_id, length('${CLICKHOUSE_TEST_UNIQUE_NAME}_') + 1), query_duration_ms < 20000 + FROM system.query_log + WHERE event_date >= yesterday() AND current_database = currentDatabase() AND type = 'QueryFinish' + AND startsWith(query_id, '${CLICKHOUSE_TEST_UNIQUE_NAME}_') + ORDER BY 1" From 1f135100115648b3c0b32e63e60603456edcd489 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 10:17:12 +0000 Subject: [PATCH 282/296] Backport #122770 to 26.8: Fix credential leak when a named argument precedes positional ones in engines and table functions --- src/Parsers/ASTFunction.cpp | 17 +- src/Parsers/FunctionSecretArgumentsFinder.cpp | 404 +++++++++--------- src/Parsers/FunctionSecretArgumentsFinder.h | 27 +- .../FunctionSecretArgumentsFinderAST.h | 2 + src/Parsers/tests/gtest_Parser.cpp | 4 +- ...oker_and_xdbc_credential_masking.reference | 2 +- ...te_cluster_name_password_masking.reference | 1 + ...7_remote_cluster_name_password_masking.sql | 10 +- ...rguments_named_before_positional.reference | 73 ++++ ...ecret_arguments_named_before_positional.sh | 105 +++++ 10 files changed, 405 insertions(+), 240 deletions(-) create mode 100644 tests/queries/0_stateless/05258_secret_arguments_named_before_positional.reference create mode 100755 tests/queries/0_stateless/05258_secret_arguments_named_before_positional.sh diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index ffe6d3a5efd25..f7a44f6ae31cf 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -1032,14 +1032,6 @@ void ASTFunction::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSetting if (!settings.show_secrets) { - /// An argument with a partially masked replacement (e.g. a presigned S3 URL whose - /// credential parameters are hidden but whose host and path are kept). - if (auto replaced = secret_arguments.replaced_arguments.find(i); replaced != secret_arguments.replaced_arguments.end()) - { - ostr << replaced->second; - continue; - } - /// A nested secret map like `headers(..)` / `extra_credentials(..)` has its values /// hidden but its keys kept. Checked before the secret-span branch below because such a /// map can itself fall inside a named span, where it must not be formatted as `key = ...`. @@ -1080,6 +1072,15 @@ void ASTFunction::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSetting continue; } + /// An argument with a partially masked replacement (e.g. a presigned S3 URL whose + /// credential parameters are hidden but whose host and path are kept). Checked after the + /// individual masks, so an argument a fail-closed rule hides whole is not partially shown. + if (auto replaced = secret_arguments.replaced_arguments.find(i); replaced != secret_arguments.replaced_arguments.end()) + { + ostr << replaced->second; + continue; + } + if (secret_arguments.start <= i && i < secret_arguments.start + secret_arguments.count) { if (secret_arguments.are_named) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 8c4855a06d2b5..edbf0a93d06b9 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -77,6 +78,17 @@ namespace return 0; return {}; } + + constexpr std::string_view password_key[] = {"password"}; + + /// `password`, and the TLS credentials passed as the literal contents of a certificate or a key file + /// rather than as a path to it. + constexpr std::string_view mysql_secret_keys[] + = {"password", "ssl_ca_pem", "ssl_cert_pem", "ssl_key_pem", "sslrootcert_pem", "sslcert_pem", "sslkey_pem"}; + + constexpr std::string_view ytsaurus_secret_keys[] = {"oauth_token"}; + + constexpr std::string_view kafka_secret_keys[] = {"kafka_sasl_password"}; } void FunctionSecretArgumentsFinder::maskEveryArgument() @@ -294,109 +306,136 @@ void FunctionSecretArgumentsFinder::maskS3UrlArgument(const std::vector void FunctionSecretArgumentsFinder::findOrdinaryFunctionSecretArguments() { - if ((function->name() == "mysql") || (function->name() == "postgresql")) + const String & name = function->name(); + + /// The arguments of these are expressions, where `=` is a comparison rather than a named argument. + if ((name == "encrypt") || (name == "decrypt") || (name == "aes_encrypt_mysql") || (name == "aes_decrypt_mysql") + || (name == "tryDecrypt")) + { + /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) + findEncryptionFunctionSecretArguments(); + return; + } + if (equalsCaseInsensitive(name, "HMAC")) + { + /// HMAC('mode', 'message', 'key') -> HMAC('mode', 'message', '[HIDDEN]') + findHMACSecretArguments(); + return; + } + if (name == "bigquery") + { + /// Accepts positional arguments after named ones and models their slots itself. + findBigQuerySecretArguments(); + return; + } + + std::function find_secrets; + if ((name == "mysql") || (name == "postgresql")) { /// mysql('host:port', 'database', 'table', 'user', 'password', ...) /// postgresql('host:port', 'database', 'table', 'user', 'password', ...) - /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...) - findMySQLFunctionSecretArguments(); + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 4, .secret_keys = mysql_secret_keys}); }; } - else if (function->name() == "mongodb") + else if (name == "mongodb") { - findMongoDBSecretArguments(); + find_secrets = [this] { findMongoDBSecretArguments(); }; } - else if ((function->name() == "s3") || (function->name() == "cosn") || (function->name() == "oss") || - (function->name() == "deltaLake") || (function->name() == "deltaLakeS3") || (function->name() == "hudi") || - (function->name() == "iceberg") || (function->name() == "gcs") || (function->name() == "icebergS3") || - (function->name() == "paimon") || (function->name() == "paimonS3")) + else if ((name == "s3") || (name == "cosn") || (name == "oss") || (name == "deltaLake") || (name == "deltaLakeS3") + || (name == "hudi") || (name == "iceberg") || (name == "gcs") || (name == "icebergS3") || (name == "paimon") + || (name == "paimonS3")) { /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) - findS3FunctionSecretArguments(/* is_cluster_function= */ false); + find_secrets = [this] { findS3FunctionSecretArguments(/* is_cluster_function= */ false); }; } - else if ((function->name() == "s3Cluster") || (function ->name() == "hudiCluster") || - (function ->name() == "deltaLakeCluster") || (function ->name() == "deltaLakeS3Cluster") || - (function ->name() == "icebergS3Cluster") || (function ->name() == "icebergCluster") || - (function ->name() == "paimonCluster") || (function ->name() == "paimonS3Cluster")) + else if ((name == "s3Cluster") || (name == "hudiCluster") || (name == "deltaLakeCluster") || (name == "deltaLakeS3Cluster") + || (name == "icebergS3Cluster") || (name == "icebergCluster") || (name == "paimonCluster") + || (name == "paimonS3Cluster")) { /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...) - findS3FunctionSecretArguments(/* is_cluster_function= */ true); + find_secrets = [this] { findS3FunctionSecretArguments(/* is_cluster_function= */ true); }; } - else if ((function->name() == "azureBlobStorage") || (function->name() == "deltaLakeAzure") || - (function->name() == "icebergAzure") || (function->name() == "paimonAzure")) + else if ((name == "azureBlobStorage") || (name == "deltaLakeAzure") || (name == "icebergAzure") || (name == "paimonAzure")) { /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) - findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ false); + find_secrets = [this] { findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ false); }; } - else if ((function->name() == "azureBlobStorageCluster") || (function->name() == "icebergAzureCluster") || - (function->name() == "deltaLakeAzureCluster") || (function->name() == "paimonAzureCluster")) + else if ((name == "azureBlobStorageCluster") || (name == "icebergAzureCluster") || (name == "deltaLakeAzureCluster") + || (name == "paimonAzureCluster")) { /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]) - findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ true); + find_secrets = [this] { findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ true); }; } - else if ((function->name() == "remote") || (function->name() == "remoteSecure")) + else if ((name == "remote") || (name == "remoteSecure")) { /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...) - findRemoteFunctionSecretArguments(); + find_secrets = [this] { findRemoteFunctionSecretArguments(); }; } - else if ((function->name() == "encrypt") || (function->name() == "decrypt") || - (function->name() == "aes_encrypt_mysql") || (function->name() == "aes_decrypt_mysql") || - (function->name() == "tryDecrypt")) - { - /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) - findEncryptionFunctionSecretArguments(); - } - else if (equalsCaseInsensitive(function->name(), "HMAC")) - { - /// HMAC('mode', 'message', 'key') -> HMAC('mode', 'message', '[HIDDEN]') - findHMACSecretArguments(); - } - else if (function->name() == "url" || function->name() == "urlCluster") + else if ((name == "url") || (name == "urlCluster")) { /// url('url', ...) keeps the url at slot 0; urlCluster('cluster', 'url', ...) at slot 1. - findURLSecretArguments(function->name() == "urlCluster" ? 1 : 0); - } - else if (function->name() == "redis") - { - findRedisFunctionSecretArguments(); + find_secrets = [this, is_cluster = name == "urlCluster"] { findURLSecretArguments(is_cluster ? 1 : 0); }; } - else if (function->name() == "ytsaurus") + else if (name == "redis") { - findYTsaurusStorageTableEngineSecretArguments(); + /// redis('host:port', 'key', 'structure', 'db_index', 'password', 'pool_size') + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 4, .secret_keys = password_key}); }; } - else if (function->name() == "bigquery") + else if (name == "ytsaurus") { - findBigQuerySecretArguments(); + /// ytsaurus('http_proxy_url', 'cypress_path', 'oauth_token', 'structure') + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 2, .secret_keys = ytsaurus_secret_keys}); }; } - else if ((function->name() == "arrowFlight") || (function->name() == "arrowflight")) + else if ((name == "arrowFlight") || (name == "arrowflight")) { - findArrowFlightSecretArguments(); + /// arrowFlight('host:port', 'dataset', 'username', 'password') + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 3, .secret_keys = password_key}); }; } - else if ((function->name() == "jdbc") || (function->name() == "odbc")) + else if ((name == "jdbc") || (name == "odbc")) { /// jdbc('DSN', schema, table) or jdbc('DSN', table) /// odbc('DSN', schema, table) or odbc('DSN', table) /// The DSN (connection string) may contain credentials. - findXDBCSecretArguments(); + find_secrets = [this] { findXDBCSecretArguments(); }; } + else + { + return; + } + + /// The query is formatted for logging before validation rejects a positional argument after a + /// `key = value` one, and the slot it was meant for is unknowable: hide every such positional. + classifyPositionalArguments(); + find_secrets(); } -void FunctionSecretArgumentsFinder::findMySQLFunctionSecretArguments() +void FunctionSecretArgumentsFinder::findPositionalAndNamedSecretArguments(const PositionalSecretSignature & signature) { - if (isNamedCollectionName(0)) + /// Not gated on the named-collection form: an identifier can also be a positional endpoint + /// (`redis(localhost, ...)`), and a collection rejects positional overrides anyway. + if (signature.positional_secret_slot) { - /// mysql(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - findTLSCredentialsSecretArguments(1); - } - else - { - /// mysql('host:port', 'database', 'table', 'user', 'password', ...) const auto positional = classifyPositionalArguments(); - if (positional.size() > 4) - markSecretArgument(positional[4]); - findSecretNamedArgument("password", 0); - findTLSCredentialsSecretArguments(0); + if (*signature.positional_secret_slot < positional.size()) + markSecretArgument(positional[*signature.positional_secret_slot]); + + /// The explicit form constant-folds its positionals, so an `equals` at the secret slot can be the + /// folded secret with the secret as its left operand (`Redis('host:port', 0, 'password' = 'x')`): + /// hide it whole. An identifier first argument does not prove a named collection (`redis(localhost, ...)`), + /// so there only an identifier left operand (`port = 6379`), which does not fold, is kept as an override. + const size_t slot = *signature.positional_secret_slot; + if (slot < function->arguments->size()) + { + const auto equals_func = function->arguments->at(slot)->getFunction(); + if (equals_func && equals_func->name() == "equals" + && (!function->arguments->at(0)->isIdentifier() || !equals_func->arguments || equals_func->arguments->size() != 2 + || !equals_func->arguments->at(0)->isIdentifier())) + markSecretArgument(slot); + } } + + for (const auto & key : signature.secret_keys) + findSecretNamedArgument(key); + markNamedArgumentsWithUnreadableKeys(0); } std::vector FunctionSecretArgumentsFinder::classifyPositionalArguments(size_t start) @@ -405,7 +444,11 @@ std::vector FunctionSecretArgumentsFinder::classifyPositionalArguments(s bool seen_named = false; for (size_t i = start; i < function->arguments->size(); ++i) { - const auto equals_func = function->arguments->at(i)->getFunction(); + const auto argument = function->arguments->at(i); + if (argument->isSettings()) + continue; + + const auto equals_func = argument->getFunction(); if (equals_func && equals_func->name() == "equals" && equals_func->hasArguments() && equals_func->arguments->size() == 2) { @@ -448,79 +491,47 @@ void FunctionSecretArgumentsFinder::markNamedArgumentsWithUnreadableKeys(size_t } } -void FunctionSecretArgumentsFinder::findTLSCredentialsSecretArguments(size_t start) -{ - for (const auto & key : tls_credentials_secret_keys) - findSecretNamedArgument(key, start); - - markNamedArgumentsWithUnreadableKeys(start); -} - void FunctionSecretArgumentsFinder::findMongoDBSecretArguments() { - String uri; - - if (isNamedCollectionName(0)) - { - /// MongoDB(named_collection, ..., password = 'password', ...) - if (findSecretNamedArgument("password", 1)) - return; - - /// MongoDB(named_collection, ..., uri = 'mongodb://username:password@127.0.0.1:27017', ...) - if (findNamedArgument(&uri, "uri", 1) == -1) - return; - - result.are_named = true; - result.start = 1; - } - else if (function->arguments->size() == 2) - { - tryGetStringFromArgument(0, &uri); - result.are_named = false; - result.start = 0; - } - else - { - // MongoDB('127.0.0.1:27017', 'database', 'collection', 'user, 'password'...) - markSecretArgument(4, false); - return; - } - - chassert(result.count == 0); - maskURIPassword(&uri); - result.count = 1; - result.replacement = std::move(uri); -} - -void FunctionSecretArgumentsFinder::findRedisTableEngineSecretArguments() -{ - /// Redis does not have URL/address argument, - /// only 'host:port' and separate "password" argument. + /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...) + /// MongoDB(named_collection, ..., password = 'password', ...) + findPositionalAndNamedSecretArguments({.positional_secret_slot = 4, .secret_keys = password_key}); + + /// Hides the password of a uri, or the whole value when it is not a plain string literal. + auto mask_uri = [this](size_t index, const AbstractFunction::Argument & value, std::string_view prefix, bool argument_is_named) + { + String uri; + if (!value.tryGetString(&uri, /* allow_identifier= */ false)) + markSecretArgument(index, argument_is_named); + else if (maskURIPassword(&uri)) + result.replaced_arguments[index] = String(prefix) + quoteString(uri); + }; - if (isNamedCollectionName(0)) + /// MongoDB('mongodb://username:password@127.0.0.1:27017/database', 'collection'[, ...]). Not gated + /// on the argument count, which a rejected named argument changes; a `host:port` has no password. + if (function->arguments->size() != 0) { - if (findSecretNamedArgument("password", 1)) - return; - } - else - { - // Redis('host:port', 'db_index', 'password', 'pool_size') - markSecretArgument(2, false); - return; + const auto first = function->arguments->at(0); + const auto first_function = first->getFunction(); + if (first->isIdentifier()) + { + String name; + /// A collection name has no password; a backquoted uri, rejected as an unknown collection, can. + if (first->tryGetString(&name, /* allow_identifier= */ true) && maskURIPassword(&name)) + result.replaced_arguments[0] = backQuoteIfNeed(name); + } + else if (!first_function || first_function->name() != "equals") + { + mask_uri(0, *first, "", /* argument_is_named= */ false); + } } -} -void FunctionSecretArgumentsFinder::findArrowFlightSecretArguments() -{ - if (isNamedCollectionName(0)) - { - /// ArrowFlight(named_collection, ..., password = 'password') - findSecretNamedArgument("password", 1); - } - else + /// MongoDB(named_collection, ..., uri = 'mongodb://username:password@127.0.0.1:27017', ...) + /// Every occurrence: a duplicated or conflicting override is logged before validation rejects it. + for (ssize_t i = findNamedArgument(nullptr, "uri"); i >= 0; i = findNamedArgument(nullptr, "uri", static_cast(i) + 1)) { - /// ArrowFlight('host:port', 'dataset', 'username', 'password') - markSecretArgument(3); + const auto index = static_cast(i); + mask_uri(index, *function->arguments->at(index)->getFunction()->arguments->at(1), "uri = ", /* argument_is_named= */ true); } } @@ -822,13 +833,11 @@ bool FunctionSecretArgumentsFinder::tryGetStringFromArgument(const AbstractFunct void FunctionSecretArgumentsFinder::findRemoteFunctionSecretArguments() { - if (isNamedCollectionName(0)) - { - /// remote(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - /// An identifier is also a cluster name when no such collection exists, and that form keeps the - /// password in a positional slot, so the walk below has to run for it too. - } + /// remote(named_collection, ..., password = 'password', ...), and a `password` written before the + /// positionals. An identifier is also a cluster name when no such collection exists, and that form + /// keeps the password in a positional slot, so the walk below has to run for it too. + findSecretNamedArgument("password"); + markNamedArgumentsWithUnreadableKeys(0); /// We're going to replace 'password' with '[HIDDEN'] for the following signatures: /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key]) @@ -948,22 +957,27 @@ void FunctionSecretArgumentsFinder::findHMACSecretArguments() void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() { const String & engine_name = function->name(); + std::function find_secrets; if (engine_name == "ExternalDistributed") { /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') - findExternalDistributedTableEngineSecretArguments(); + /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...) + find_secrets = [this] + { + findPositionalAndNamedSecretArguments( + {.positional_secret_slot = 5, .secret_keys = password_key}); + }; } else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") || (engine_name == "MaterializedPostgreSQL")) { /// MySQL('host:port', 'database', 'table', 'user', 'password', ...) /// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) /// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) - /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...) - findMySQLFunctionSecretArguments(); + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 4, .secret_keys = mysql_secret_keys}); }; } else if (engine_name == "MongoDB") { - findMongoDBSecretArguments(); + find_secrets = [this] { findMongoDBSecretArguments(); }; } else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") || (engine_name == "GCS") || (engine_name == "DeltaLake") || (engine_name == "DeltaLakeS3") || (engine_name == "Hudi") @@ -972,31 +986,36 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() || (engine_name == "S3Queue")) { /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...) - findS3TableEngineSecretArguments(); + find_secrets = [this] { findS3TableEngineSecretArguments(); }; } else if (engine_name == "URL") { - findURLSecretArguments(); + find_secrets = [this] { findURLSecretArguments(); }; } else if (engine_name == "AzureBlobStorage" || engine_name == "AzureQueue") { - findAzureBlobStorageTableEngineSecretArguments(); + find_secrets = [this] { findAzureBlobStorageTableEngineSecretArguments(); }; } else if (engine_name == "Redis") { - findRedisTableEngineSecretArguments(); + /// Redis('host:port', 'db_index', 'password', 'pool_size') + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 2, .secret_keys = password_key}); }; } else if (engine_name == "YTsaurus") { - findYTsaurusStorageTableEngineSecretArguments(); + /// YTsaurus('http_proxy_url', 'cypress_path', 'oauth_token') + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 2, .secret_keys = ytsaurus_secret_keys}); }; } else if (engine_name == "BigQuery") { + /// Accepts positional arguments after named ones and models their slots itself. findBigQuerySecretArguments(); + return; } else if (engine_name == "ArrowFlight") { - findArrowFlightSecretArguments(); + /// ArrowFlight('host:port', 'dataset', 'username', 'password') + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 3, .secret_keys = password_key}); }; } else if ((engine_name == "Remote") || (engine_name == "RemoteSecure")) { @@ -1004,32 +1023,39 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() /// RemoteSecure(...) - same as Remote(...) /// The arguments are identical to the `remote`/`remoteSecure` table functions, so reuse /// the same finder (it also handles the named-collection form `Remote(named_collection, ...)`). - findRemoteFunctionSecretArguments(); + find_secrets = [this] { findRemoteFunctionSecretArguments(); }; } else if (engine_name == "NATS") { /// NATS(named_collection, nats_password = 'password', nats_credentials = '...', ...) - findNATSTableEngineSecretArguments(); + find_secrets = [this] { findNATSTableEngineSecretArguments(); }; } else if (engine_name == "RabbitMQ") { /// RabbitMQ(named_collection, rabbitmq_address = '...', rabbitmq_password = '...') - findRabbitMQTableEngineSecretArguments(); + find_secrets = [this] { findRabbitMQTableEngineSecretArguments(); }; } else if (engine_name == "Kafka") { /// Kafka(named_collection, kafka_sasl_password = '...'); the legacy positional form carries no /// secret and makes the collection name optional, so a named argument can be the first one. - findSecretNamedArgument("kafka_sasl_password", 0); - markNamedArgumentsWithUnreadableKeys(0); + find_secrets = [this] { findPositionalAndNamedSecretArguments({.secret_keys = kafka_secret_keys}); }; } else if ((engine_name == "JDBC") || (engine_name == "ODBC")) { /// JDBC('DSN', database, table) /// ODBC('DSN', database, table) /// The DSN (connection string) may contain credentials. - findXDBCSecretArguments(); + find_secrets = [this] { findXDBCSecretArguments(); }; + } + else + { + return; } + + /// A positional after a named argument is hidden, see findOrdinaryFunctionSecretArguments. + classifyPositionalArguments(); + find_secrets(); } void FunctionSecretArgumentsFinder::findBrokerTableEngineSecretArguments( @@ -1098,20 +1124,6 @@ void FunctionSecretArgumentsFinder::findRabbitMQTableEngineSecretArguments() findBrokerTableEngineSecretArguments(rabbitmq_secret_keys, "rabbitmq_address"); } -void FunctionSecretArgumentsFinder::findExternalDistributedTableEngineSecretArguments() -{ - if (isNamedCollectionName(1)) - { - /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 2); - } - else - { - /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') - markSecretArgument(5); - } -} - void FunctionSecretArgumentsFinder::findS3TableEngineSecretArguments() { if (isNamedCollectionName(0)) @@ -1184,18 +1196,6 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen markSecretArgument(*credential_arg_idx); } -void FunctionSecretArgumentsFinder::findRedisFunctionSecretArguments() -{ - // redis(host:port, key, structure, db_index, password, pool_size) - markSecretArgument(4); -} - -void FunctionSecretArgumentsFinder::findYTsaurusStorageTableEngineSecretArguments() -{ - // YTsaurus('base_uri', 'yt_path', 'auth_token') - markSecretArgument(2); -} - void FunctionSecretArgumentsFinder::findBigQuerySecretArguments() { /// bigquery('project', 'dataset', 'table'[, 'access_token'][, key = value, ...]) @@ -1284,61 +1284,43 @@ void FunctionSecretArgumentsFinder::findBigQuerySecretArguments() void FunctionSecretArgumentsFinder::findDatabaseEngineSecretArguments() { const String & engine_name = function->name(); - if (engine_name == "MySQL" || - engine_name == "PostgreSQL" || - engine_name == "MaterializedPostgreSQL") + std::function find_secrets; + if (engine_name == "MySQL" || engine_name == "PostgreSQL" || engine_name == "MaterializedPostgreSQL" + || engine_name == "Remote" || engine_name == "RemoteSecure") { /// MySQL('host:port', 'database', 'user', 'password') /// PostgreSQL('host:port', 'database', 'user', 'password') - findMySQLDatabaseSecretArguments(); - } - else if (engine_name == "Remote" || engine_name == "RemoteSecure") - { /// Remote('addresses_expr', 'database', 'user', 'password') - /// RemoteSecure(...) - same as Remote(...) - /// The password is the last positional argument (or `password = ...` in the named-collection - /// form), exactly like the MySQL/PostgreSQL database engines. Note this differs from the - /// `Remote`/`RemoteSecure` *table* engine signature (which also has a table name), so the - /// database engine cannot reuse `findRemoteFunctionSecretArguments`. - findMySQLDatabaseSecretArguments(); + /// Unlike the `Remote` table engine there is no table name, so `findRemoteFunctionSecretArguments` + /// does not apply. + find_secrets = [this] { findPositionalAndNamedSecretArguments({.positional_secret_slot = 3, .secret_keys = mysql_secret_keys}); }; } else if (engine_name == "S3") { /// S3('url', 'access_key_id', 'secret_access_key') - findS3DatabaseSecretArguments(); + find_secrets = [this] { findS3DatabaseSecretArguments(); }; } else if (engine_name == "DataLakeCatalog") { - findDataLakeCatalogSecretArguments(); + find_secrets = [this] { findDataLakeCatalogSecretArguments(); }; } else if (engine_name == "Backup") { - findBackupDatabaseSecretArguments(); + find_secrets = [this] { findBackupDatabaseSecretArguments(); }; } else if (engine_name == "URL") { /// URL('base_url') - findURLSecretArguments(); - } -} - -void FunctionSecretArgumentsFinder::findMySQLDatabaseSecretArguments() -{ - if (isNamedCollectionName(0)) - { - /// MySQL(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - findTLSCredentialsSecretArguments(1); + find_secrets = [this] { findURLSecretArguments(); }; } else { - /// MySQL('host:port', 'database', 'user', 'password') - const auto positional = classifyPositionalArguments(); - if (positional.size() > 3) - markSecretArgument(positional[3]); - findSecretNamedArgument("password", 0); - findTLSCredentialsSecretArguments(0); + return; } + + /// A positional after a named argument is hidden, see findOrdinaryFunctionSecretArguments. + classifyPositionalArguments(); + find_secrets(); } void FunctionSecretArgumentsFinder::findS3DatabaseSecretArguments() diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 2f742efa084a8..b72c87816017c 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -29,6 +29,9 @@ class AbstractFunction /// The exact literal text of any scalar literal (`1`, `true`, `1.5`), with strings quoted. /// Lets a reconstructor keep non-string values like `use_environment_credentials = 1` visible. virtual bool tryGetLiteralText(String * res) const = 0; + /// A `SETTINGS` clause among the arguments (`remote(..., SETTINGS ...)`): not a positional + /// argument, and it hides its own secret values when formatted. + virtual bool isSettings() const { return false; } }; class Arguments { @@ -120,11 +123,6 @@ class FunctionSecretArgumentsFinder = {"secret_access_key", "session_token", "google_adc_client_secret", "google_adc_refresh_token", "external_id", "role_session_name"}; - /// Named arguments carrying TLS credentials as the literal contents of a certificate or a key file, - /// rather than as a path to it. They are secret and have to be hidden the same way a password is. - static constexpr std::string_view tls_credentials_secret_keys[] - = {"ssl_ca_pem", "ssl_cert_pem", "ssl_key_pem", "sslrootcert_pem", "sslcert_pem", "sslkey_pem"}; - /// Named arguments carrying NATS credentials. They are the setting names, because the `NATS` engine /// takes its arguments as overrides of a named collection (`NATS(collection, nats_token = '...')`). /// `nats_server_list` is a destination and can carry URI userinfo credentials, so hide it whole. @@ -177,12 +175,19 @@ class FunctionSecretArgumentsFinder /// path visible. The field set mirrors `BackupInfo::removeCredentialsFromS3URL`. void maskS3UrlArgument(const std::vector & positional, size_t url_slot); + /// The shape shared by most external-storage engines: an explicit positional form with the secret at + /// one slot, or a named-collection form whose secrets are `key = value` overrides. + struct PositionalSecretSignature + { + /// Slot of the secret among the positional arguments of the explicit form; none when that form has no secret. + std::optional positional_secret_slot = {}; + /// Secret `key = value` overrides, hidden in both forms. + std::span secret_keys = {}; + }; + void findPositionalAndNamedSecretArguments(const PositionalSecretSignature & signature); + void findOrdinaryFunctionSecretArguments(); - void findMySQLFunctionSecretArguments(); - void findTLSCredentialsSecretArguments(size_t start); void findMongoDBSecretArguments(); - void findRedisTableEngineSecretArguments(); - void findArrowFlightSecretArguments(); void findXDBCSecretArguments(); void findS3FunctionSecretArguments(bool is_cluster_function); void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function); @@ -226,18 +231,14 @@ class FunctionSecretArgumentsFinder void findEncryptionFunctionSecretArguments(); void findHMACSecretArguments(); void findTableEngineSecretArguments(); - void findExternalDistributedTableEngineSecretArguments(); void findS3TableEngineSecretArguments(); void findAzureBlobStorageTableEngineSecretArguments(); - void findRedisFunctionSecretArguments(); - void findYTsaurusStorageTableEngineSecretArguments(); void findBigQuerySecretArguments(); void findBrokerTableEngineSecretArguments( std::span secret_keys, std::string_view address_key); void findNATSTableEngineSecretArguments(); void findRabbitMQTableEngineSecretArguments(); void findDatabaseEngineSecretArguments(); - void findMySQLDatabaseSecretArguments(); void findS3DatabaseSecretArguments(); void findDataLakeCatalogSecretArguments(); void findBackupDatabaseSecretArguments(); diff --git a/src/Parsers/FunctionSecretArgumentsFinderAST.h b/src/Parsers/FunctionSecretArgumentsFinderAST.h index 754411535c8dc..6d0c5c0503424 100644 --- a/src/Parsers/FunctionSecretArgumentsFinderAST.h +++ b/src/Parsers/FunctionSecretArgumentsFinderAST.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -24,6 +25,7 @@ class FunctionAST : public AbstractFunction return nullptr; } bool isIdentifier() const override { return argument->as(); } + bool isSettings() const override { return argument->as(); } bool tryGetString(String * res, bool allow_identifier) const override { if (const auto * literal = argument->as()) diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index 634465edcde62..2039740746589 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -517,8 +517,8 @@ TEST(ParserCreateQuery, MaskKafkaTableEngineCredentials) EXPECT_EQ(first_arg_masked.find("plain_first_password"), String::npos); EXPECT_NE(first_arg_masked.find("kafka_sasl_password = '[HIDDEN]'"), String::npos); - /// The positional argument beside it is not a secret and stays visible. - EXPECT_NE(first_arg_masked.find("'clickhouse'"), String::npos); + /// A positional argument after a named one has an unknowable slot, so it is hidden. + EXPECT_EQ(first_arg_masked.find("'clickhouse'"), String::npos); /// The `SETTINGS` clause form is masked by `Kafka::SETTINGS_TO_HIDE` and must agree. const String settings_query = diff --git a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference index 126d8989eee7a..7e2d83c82b3c4 100644 --- a/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference +++ b/tests/queries/0_stateless/05233_broker_and_xdbc_credential_masking.reference @@ -33,5 +33,5 @@ f2_rabbitmq_engine_password CREATE TABLE t05233 (`x` UInt8) ENGINE = RabbitMQ(nc f3_rabbitmq_engine_control CREATE TABLE t05233 (x UInt8) ENGINE = RabbitMQ(nc05233, rabbitmq_address = 'amqp://h:5672/v') f4_kafka_engine_password CREATE TABLE t05233 (`x` UInt8) ENGINE = Kafka(nc05233, kafka_sasl_password = '[HIDDEN]') f5_kafka_positional_control CREATE TABLE db05233absent.t05233 (x UInt8) ENGINE = Kafka('broker05233:9092', 'topic05233', 'group05233', 'JSONEachRow') -f6_kafka_first_argument_secret CREATE TABLE t05233 (`x` UInt8) ENGINE = Kafka(kafka_sasl_password = '[HIDDEN]', 'clickhouse') +f6_kafka_first_argument_secret CREATE TABLE t05233 (`x` UInt8) ENGINE = Kafka(kafka_sasl_password = '[HIDDEN]', '[HIDDEN]') 0 diff --git a/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.reference b/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.reference index 3b8835252a031..5a1cd53856725 100644 --- a/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.reference +++ b/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.reference @@ -4,6 +4,7 @@ 1 1 1 SELECT 1 FROM remote(\'\', \'db\', \'t\', \'usr\', \'[HIDDEN]\') SELECT 1 FROM remote(\'\', \'db\', \'t\', \'usr\', \'[HIDDEN]\', rand()) +SELECT 1 FROM remote(\'\', \'db\', \'t\', \'usr\', password = \'[HIDDEN]\') SELECT 1 FROM remote(\'\', db.t, \'usr\', \'[HIDDEN]\') SELECT 1 FROM remote(\'\', numbers(10), \'usr\', \'[HIDDEN]\') SELECT 1 FROM remote(cluster_that_does_not_exist_05237, system, one, \'usr\', \'[HIDDEN]\') diff --git a/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.sql b/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.sql index c3deec41db3c5..bf275950eca1e 100644 --- a/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.sql +++ b/tests/queries/0_stateless/05237_remote_cluster_name_password_masking.sql @@ -18,9 +18,9 @@ SELECT 1 FROM remote('', db.t, 'usr', 'SECRETPW6'); -- { serverError BAD_ARGUMEN SELECT 1 FROM remote('', numbers(10), 'usr', 'SECRETPW7'); -- { serverError BAD_ARGUMENTS } SELECT 1 FROM remote('', 'db', 't', 'usr', 'SECRETPW8', rand()); -- { serverError BAD_ARGUMENTS } --- Not credentials, and they have to stay readable: a user name with no password after it, a --- sharding key spelled as an equality whose left operand happens to be called `password`, and a --- SETTINGS clause. +-- Not credentials, and they have to stay readable: a user name with no password after it and a +-- SETTINGS clause. A `password = ...` argument is hidden even where it could be a sharding key: it +-- reads the same as a named password written in the wrong place. SELECT 1 FROM remote(test_shard_localhost, system, one, 'usr'); SELECT 1 FROM remote('', 'db', 't', 'usr', password = 'NOT_A_CREDENTIAL'); -- { serverError BAD_ARGUMENTS } SELECT 1 FROM remote('', 'db', 't', 'usr', rand(), SETTINGS skip_unavailable_shards = 1); -- { serverError BAD_ARGUMENTS } @@ -48,8 +48,8 @@ ORDER BY query; SELECT uniqExact(query) = 11, countIf(query LIKE '%SECRETPW%') = 0, - uniqExactIf(query, query LIKE '%[HIDDEN]%') = 8, - uniqExactIf(query, query LIKE '%NOT_A_CREDENTIAL%') = 1, + uniqExactIf(query, query LIKE '%[HIDDEN]%') = 9, + uniqExactIf(query, query LIKE '%NOT_A_CREDENTIAL%') = 0, uniqExactIf(query, query LIKE '%skip_unavailable_shards%') = 1 FROM system.query_log WHERE current_database = currentDatabase() diff --git a/tests/queries/0_stateless/05258_secret_arguments_named_before_positional.reference b/tests/queries/0_stateless/05258_secret_arguments_named_before_positional.reference new file mode 100644 index 0000000000000..3a9f13873441a --- /dev/null +++ b/tests/queries/0_stateless/05258_secret_arguments_named_before_positional.reference @@ -0,0 +1,73 @@ +SELECT * FROM mongodb(foo = 1, '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +SELECT * FROM redis(foo = 1, '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +SELECT * FROM remote('127.0.0.1', foo = 1, '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +CREATE TABLE tm1 (`x` Int32) ENGINE = ArrowFlight(foo = 1, '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +CREATE TABLE tm3 (`x` Int32) ENGINE = ExternalDistributed(foo = 1, '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +CREATE TABLE tm5 (`x` Int32) ENGINE = YTsaurus(foo = 1, '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +CREATE TABLE tm7 (`x` Int32) ENGINE = Redis(foo = 1, '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') PRIMARY KEY x +SELECT * FROM mysql(foo = 1, '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +SELECT * FROM mongodb('127.0.0.1:27017', 'db', 'coll', 'user', '[HIDDEN]') +SELECT * FROM redis('127.0.0.1:6379', 'k', 'k String', 0, '[HIDDEN]') +SELECT * FROM remote('127.0.0.1', 'db', 't', 'user', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = ArrowFlight('127.0.0.1:5006', 'ds', 'user', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = ExternalDistributed('MySQL', '127.0.0.1:3306', 'db', 't', 'user', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = YTsaurus('http://127.0.0.1', '//p', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = Redis('127.0.0.1:6379', 0, '[HIDDEN]', 16) PRIMARY KEY x +SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', 'user', '[HIDDEN]') +SELECT * FROM arrowFlight('127.0.0.1:5006', 'ds', 'user', '[HIDDEN]') +SELECT * FROM ytsaurus('http://127.0.0.1', '//p', '[HIDDEN]', 'x Int32') +CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'user', '[HIDDEN]') +SELECT * FROM mysql(creds, concat('pass', 'word') = '[HIDDEN]') +SELECT * FROM mysql(creds, password = '[HIDDEN]', '[HIDDEN]') +SELECT * FROM postgresql(creds, concat('pass', 'word') = '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = MySQL(creds, concat('pass', 'word') = '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = PostgreSQL(creds, password = '[HIDDEN]', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = MaterializedPostgreSQL(creds, concat('pass', 'word') = '[HIDDEN]') +CREATE DATABASE d ENGINE = PostgreSQL(creds, concat('pass', 'word') = '[HIDDEN]') +CREATE DATABASE d ENGINE = Remote(creds, password = '[HIDDEN]', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB(creds, concat('pass', 'word') = '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB(creds, password = '[HIDDEN]', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB(creds, uri = 'mongodb://user:[HIDDEN]@127.0.0.1:27017/db') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB(creds, collection = 'c', uri = 'mongodb://user:[HIDDEN]@127.0.0.1:27017/db') +SELECT * FROM mongodb('mongodb://user:[HIDDEN]@127.0.0.1:27017/db', 'coll') +CREATE TABLE t (`x` Int32) ENGINE = Redis(creds, concat('pass', 'word') = '[HIDDEN]') PRIMARY KEY x +CREATE TABLE t (`x` Int32) ENGINE = Redis(creds, password = '[HIDDEN]', '[HIDDEN]') PRIMARY KEY x +SELECT * FROM redis(creds, concat('pass', 'word') = '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = ArrowFlight(creds, concat('pass', 'word') = '[HIDDEN]') +SELECT * FROM arrowFlight(creds, password = '[HIDDEN]', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = YTsaurus(creds, concat('oauth', '_token') = '[HIDDEN]') +SELECT * FROM ytsaurus(creds, oauth_token = '[HIDDEN]', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = ExternalDistributed('MySQL', creds, concat('pass', 'word') = '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = ExternalDistributed('MySQL', creds, password = '[HIDDEN]', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = Kafka(creds, concat('kafka_sasl', '_password') = '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = Kafka(creds, kafka_sasl_password = '[HIDDEN]', '[HIDDEN]') +SELECT * FROM urlCluster(foo = 1, '[HIDDEN]') +SELECT * FROM remote('127.0.0.1', 'db', 't', 'user', '[HIDDEN]', number = 1, SETTINGS skip_unavailable_shards = 1) +SELECT * FROM mysql(creds, password = '[HIDDEN]', SETTINGS connect_timeout = 1) +CREATE TABLE t (`x` Int32) ENGINE = Redis('127.0.0.1:6379', 0, '[HIDDEN]') PRIMARY KEY x +SELECT * FROM redis('127.0.0.1:6379', 'k', 'k String', 0, '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = ArrowFlight('127.0.0.1:5006', 'ds', 'user', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = Redis(creds, host = 'h', port = 6379) PRIMARY KEY x +SELECT * FROM redis(localhost, 'k', 'k String', 0, '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = ArrowFlight(localhost, 'ds', 'user', '[HIDDEN]') +SELECT * FROM mysql(localhost, 'db', 't', 'u', '[HIDDEN]') +SELECT * FROM mysql(creds, host = 'h', port = 1, db = 'd', `table` = 't') +CREATE TABLE t (`dummy` UInt8) ENGINE = Remote(creds, concat('pass', 'word') = '[HIDDEN]') +SELECT * FROM remote(password = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +SELECT * FROM remoteSecure(creds, concat('pass', 'word') = '[HIDDEN]') +SELECT * FROM redis(localhost, 'k', 'k String', 0, '[HIDDEN]') +SELECT * FROM ytsaurus(proxy, '//p', '[HIDDEN]', 'x Int32') +CREATE TABLE t (`x` Int32) ENGINE = YTsaurus(proxy, '//p', '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB(creds, password = '[HIDDEN]', uri = 'mongodb://user:[HIDDEN]@127.0.0.1/db') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB(creds, uri = 'mongodb://user:[HIDDEN]@127.0.0.1/db', uri = 'mongodb://user:[HIDDEN]@127.0.0.1/db') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB(creds, uri = '[HIDDEN]') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB('mongodb://user:[HIDDEN]@127.0.0.1/db', 'c', '_id') +SELECT * FROM mongodb('mongodb://user:[HIDDEN]@127.0.0.1/db', 'c', 'x Int32') +SELECT * FROM mongodb('mongodb://user:[HIDDEN]@127.0.0.1/db', 'c', 'x Int32', '_id') +SELECT * FROM mongodb('mongodb://user:[HIDDEN]@127.0.0.1/db', 'c', password = '[HIDDEN]') +SELECT * FROM mongodb('mongodb://user:[HIDDEN]@127.0.0.1/db', 'c', 'x Int32', '_id', '[HIDDEN]') +SELECT * FROM mongodb() +CREATE TABLE t (`x` Int32) ENGINE = MongoDB(`mongodb://user:[HIDDEN]@127.0.0.1/db`, 'c') +SELECT * FROM mongodb(`mongodb://user:[HIDDEN]@127.0.0.1/db`, 'c', 'x Int32') +CREATE TABLE t (`x` Int32) ENGINE = MongoDB('mongodb://user:[HIDDEN]@127.0.0.1/db?label=it\'s', 'c') +0 diff --git a/tests/queries/0_stateless/05258_secret_arguments_named_before_positional.sh b/tests/queries/0_stateless/05258_secret_arguments_named_before_positional.sh new file mode 100755 index 0000000000000..2a67e2e43b00b --- /dev/null +++ b/tests/queries/0_stateless/05258_secret_arguments_named_before_positional.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A `key = value` argument before the positional ones shifts them: every positional after it is hidden. +mixed_queries=( + "SELECT * FROM mongodb(foo = 1, '127.0.0.1:27017', 'db', 'coll', 'user', 'leakmongo')" + "SELECT * FROM redis(foo = 1, '127.0.0.1:6379', 'k', 'k String', 0, 'leakredisf')" + "SELECT * FROM remote('127.0.0.1', foo = 1, 'db', 't', 'user', 'leakremote')" + "CREATE TABLE tm1 (x Int32) ENGINE = ArrowFlight(foo = 1, '127.0.0.1:5006', 'ds', 'user', 'leakaf')" + "CREATE TABLE tm3 (x Int32) ENGINE = ExternalDistributed(foo = 1, 'MySQL', '127.0.0.1:3306', 'db', 't', 'user', 'leakextdist')" + "CREATE TABLE tm5 (x Int32) ENGINE = YTsaurus(foo = 1, 'http://127.0.0.1', '//p', 'leakyt')" + "CREATE TABLE tm7 (x Int32) ENGINE = Redis(foo = 1, '127.0.0.1:6379', 0, 'leakredeng', 16) PRIMARY KEY x" + "SELECT * FROM mysql(foo = 1, '127.0.0.1:3306', 'db', 't', 'user', 'leakmysql')" +) + +other_queries=( + # The all-positional forms hide only the password. + "SELECT * FROM mongodb('127.0.0.1:27017', 'db', 'coll', 'user', 'pw')" + "SELECT * FROM redis('127.0.0.1:6379', 'k', 'k String', 0, 'pw')" + "SELECT * FROM remote('127.0.0.1', 'db', 't', 'user', 'pw')" + "CREATE TABLE t (x Int32) ENGINE = ArrowFlight('127.0.0.1:5006', 'ds', 'user', 'pw')" + "CREATE TABLE t (x Int32) ENGINE = ExternalDistributed('MySQL', '127.0.0.1:3306', 'db', 't', 'user', 'pw')" + "CREATE TABLE t (x Int32) ENGINE = YTsaurus('http://127.0.0.1', '//p', 'pw')" + "CREATE TABLE t (x Int32) ENGINE = Redis('127.0.0.1:6379', 0, 'pw', 16) PRIMARY KEY x" + "SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', 'user', 'pw')" + "SELECT * FROM arrowFlight('127.0.0.1:5006', 'ds', 'user', 'pw')" + "SELECT * FROM ytsaurus('http://127.0.0.1', '//p', 'pw', 'x Int32')" + "CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'user', 'pw')" + # A named collection with a key that is an expression hides its value; a positional after a named override is hidden. + "SELECT * FROM mysql(creds, concat('pass', 'word') = 'S1')" + "SELECT * FROM mysql(creds, password = 'S2', 'after')" + "SELECT * FROM postgresql(creds, concat('pass', 'word') = 'S1')" + "CREATE TABLE t (x Int32) ENGINE = MySQL(creds, concat('pass', 'word') = 'S1')" + "CREATE TABLE t (x Int32) ENGINE = PostgreSQL(creds, password = 'S2', 'after')" + "CREATE TABLE t (x Int32) ENGINE = MaterializedPostgreSQL(creds, concat('pass', 'word') = 'S1')" + "CREATE DATABASE d ENGINE = PostgreSQL(creds, concat('pass', 'word') = 'S1')" + "CREATE DATABASE d ENGINE = Remote(creds, password = 'S2', 'after')" + "CREATE TABLE t (x Int32) ENGINE = MongoDB(creds, concat('pass', 'word') = 'S1')" + "CREATE TABLE t (x Int32) ENGINE = MongoDB(creds, password = 'S2', 'after')" + "CREATE TABLE t (x Int32) ENGINE = MongoDB(creds, uri = 'mongodb://user:S3@127.0.0.1:27017/db')" + "CREATE TABLE t (x Int32) ENGINE = MongoDB(creds, collection = 'c', uri = 'mongodb://user:S4@127.0.0.1:27017/db')" + "SELECT * FROM mongodb('mongodb://user:S3@127.0.0.1:27017/db', 'coll')" + "CREATE TABLE t (x Int32) ENGINE = Redis(creds, concat('pass', 'word') = 'S1') PRIMARY KEY x" + "CREATE TABLE t (x Int32) ENGINE = Redis(creds, password = 'S2', 'after') PRIMARY KEY x" + "SELECT * FROM redis(creds, concat('pass', 'word') = 'S1')" + "CREATE TABLE t (x Int32) ENGINE = ArrowFlight(creds, concat('pass', 'word') = 'S1')" + "SELECT * FROM arrowFlight(creds, password = 'S2', 'after')" + "CREATE TABLE t (x Int32) ENGINE = YTsaurus(creds, concat('oauth', '_token') = 'S1')" + "SELECT * FROM ytsaurus(creds, oauth_token = 'S2', 'after')" + "CREATE TABLE t (x Int32) ENGINE = ExternalDistributed('MySQL', creds, concat('pass', 'word') = 'S1')" + "CREATE TABLE t (x Int32) ENGINE = ExternalDistributed('MySQL', creds, password = 'S2', 'after')" + "CREATE TABLE t (x Int32) ENGINE = Kafka(creds, concat('kafka_sasl', '_password') = 'S1')" + "CREATE TABLE t (x Int32) ENGINE = Kafka(creds, kafka_sasl_password = 'S2', 'after')" + # A url after a named argument is hidden whole, not only its password. + "SELECT * FROM urlCluster(foo = 1, 'https://user:S5@example.com/path')" + # A `SETTINGS` clause is not a positional argument and stays readable. + "SELECT * FROM remote('127.0.0.1', 'db', 't', 'user', 'pw', number = 1, SETTINGS skip_unavailable_shards = 1)" + "SELECT * FROM mysql(creds, password = 'S2', SETTINGS connect_timeout = 1)" + # A folded `equals` at the secret slot can hold the secret as its left operand: it is hidden whole. + "CREATE TABLE t (x Int32) ENGINE = Redis('127.0.0.1:6379', 0, 'S10' = 'x') PRIMARY KEY x" + "SELECT * FROM redis('127.0.0.1:6379', 'k', 'k String', 0, 'S10' = 'x')" + "CREATE TABLE t (x Int32) ENGINE = ArrowFlight('127.0.0.1:5006', 'ds', 'user', 'S10' = 'x')" + "CREATE TABLE t (x Int32) ENGINE = Redis(creds, host = 'h', port = 6379) PRIMARY KEY x" + "SELECT * FROM redis(localhost, 'k', 'k String', 0, 'S11' = 'x')" + "CREATE TABLE t (x Int32) ENGINE = ArrowFlight(localhost, 'ds', 'user', 'S11' = 'x')" + "SELECT * FROM mysql(localhost, 'db', 't', 'u', 'S11' = 'x')" + "SELECT * FROM mysql(creds, host = 'h', port = 1, db = 'd', table = 't')" + "CREATE TABLE t (dummy UInt8) ENGINE = Remote(creds, concat('pass', 'word') = 'S12')" + "SELECT * FROM remote(password = 'S12', '127.0.0.1:9000', 'system', 'one', 'u')" + "SELECT * FROM remoteSecure(creds, concat('pass', 'word') = 'S12')" + # An identifier can be a positional endpoint rather than a named collection. + "SELECT * FROM redis(localhost, 'k', 'k String', 0, 'S6')" + "SELECT * FROM ytsaurus(proxy, '//p', 'S6', 'x Int32')" + "CREATE TABLE t (x Int32) ENGINE = YTsaurus(proxy, '//p', 'S6')" + # Every `uri` override is masked, also beside a `password` one. + "CREATE TABLE t (x Int32) ENGINE = MongoDB(creds, password = 'S7', uri = 'mongodb://user:S7@127.0.0.1/db')" + "CREATE TABLE t (x Int32) ENGINE = MongoDB(creds, uri = 'mongodb://user:S8@127.0.0.1/db', uri = 'mongodb://user:S9@127.0.0.1/db')" + "CREATE TABLE t (x Int32) ENGINE = MongoDB(creds, uri = concat('mongodb://user:', 'S10@127.0.0.1'))" + "CREATE TABLE t (x Int32) ENGINE = MongoDB('mongodb://user:S11@127.0.0.1/db', 'c', '_id')" + "SELECT * FROM mongodb('mongodb://user:S11@127.0.0.1/db', 'c', 'x Int32')" + "SELECT * FROM mongodb('mongodb://user:S11@127.0.0.1/db', 'c', 'x Int32', '_id')" + "SELECT * FROM mongodb('mongodb://user:S12@127.0.0.1/db', 'c', password = 'S12')" + "SELECT * FROM mongodb('mongodb://user:S14@127.0.0.1/db', 'c', 'x Int32', '_id', password = 'S14')" + "SELECT * FROM mongodb()" + "CREATE TABLE t (x Int32) ENGINE = MongoDB(\`mongodb://user:S15@127.0.0.1/db\`, 'c')" + "SELECT * FROM mongodb(\`mongodb://user:S15@127.0.0.1/db\`, 'c', 'x Int32')" + "CREATE TABLE t (x Int32) ENGINE = MongoDB('mongodb://user:S13@127.0.0.1/db?label=it\\'s', 'c')" +) + +for query in "${mixed_queries[@]}" "${other_queries[@]}"; do + $CLICKHOUSE_FORMAT --oneline <<< "$query" +done + +# The rejected statements are logged with the same masking. +for query in "${mixed_queries[@]}"; do + $CLICKHOUSE_CLIENT -q "$query" >/dev/null 2>&1 +done +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS tm1; DROP TABLE IF EXISTS tm3; DROP TABLE IF EXISTS tm5; DROP TABLE IF EXISTS tm7" +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT -q " + SELECT count() FROM system.query_log + WHERE current_database = currentDatabase() AND type != 'QueryStart' AND query LIKE '%leak%' AND query NOT LIKE '%query_log%'" From 7134d459284a8120e23ffb2b8aeaeef5a834984b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 10:19:53 +0000 Subject: [PATCH 283/296] Backport #115424 to 26.8: Add ability to select jemalloc 4K/16K page size on aarch64 (default 64K) --- contrib/jemalloc-cmake/CMakeLists.txt | 20 +++++++++++++++++++ .../internal/jemalloc_internal_defs.h.in | 4 ++-- .../internal/jemalloc_internal_defs.h.in | 4 ++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 00d8349091850..34eeb17b00322 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -198,6 +198,26 @@ else () message (FATAL_ERROR "internal jemalloc: This arch is not supported") endif () +if (OS_LINUX AND ARCH_AARCH64) + # aarch64 kernels are built with 4KiB, 16KiB or 64KiB pages. jemalloc's page size is fixed at + # compile time and must not be smaller than the kernel page size, otherwise jemalloc fails to + # boot. 64KiB runs everywhere, at the cost of a higher memory footprint on smaller-page kernels + # (page-sized slabs, page-granular purging); pick a smaller size when the target kernel is known. + set (JEMALLOC_AARCH64_PAGE_SIZE_KIB 64 CACHE STRING "jemalloc page size on aarch64, in KiB (4, 16 or 64). Must not be less than the kernel page size.") + if (JEMALLOC_AARCH64_PAGE_SIZE_KIB EQUAL 4) + set (JEMALLOC_LG_PAGE 12) + elseif (JEMALLOC_AARCH64_PAGE_SIZE_KIB EQUAL 16) + set (JEMALLOC_LG_PAGE 14) + elseif (JEMALLOC_AARCH64_PAGE_SIZE_KIB EQUAL 64) + set (JEMALLOC_LG_PAGE 16) + else () + message (FATAL_ERROR "Invalid JEMALLOC_AARCH64_PAGE_SIZE_KIB=${JEMALLOC_AARCH64_PAGE_SIZE_KIB}, must be 4, 16 or 64") + endif () + # A PMD-level THP maps (page size / 8) entries of one page each: 2MiB/32MiB/512MiB for 4/16/64 KiB pages. + math (EXPR JEMALLOC_LG_HUGEPAGE "2 * ${JEMALLOC_LG_PAGE} - 3") + message (STATUS "jemalloc aarch64 page size: ${JEMALLOC_AARCH64_PAGE_SIZE_KIB}KiB (LG_PAGE=${JEMALLOC_LG_PAGE}, LG_HUGEPAGE=${JEMALLOC_LG_HUGEPAGE})") +endif () + configure_file(${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h.in ${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h) target_include_directories(_jemalloc SYSTEM PRIVATE diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 9e495d40d94aa..9a98fbd3810a0 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -204,7 +204,7 @@ /* #undef LG_QUANTUM */ /* One page is 2^LG_PAGE bytes. */ -#define LG_PAGE 16 +#define LG_PAGE @JEMALLOC_LG_PAGE@ /* Maximum number of regions in a slab. */ /* #undef CONFIG_LG_SLAB_MAXREGS */ @@ -214,7 +214,7 @@ * system does not explicitly support huge pages; system calls that require * explicit huge page support are separately configured. */ -#define LG_HUGEPAGE 29 +#define LG_HUGEPAGE @JEMALLOC_LG_HUGEPAGE@ /* * If defined, adjacent virtual memory mappings with identical attributes diff --git a/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index 3a8ef54f40eab..0ac9fb9c4526d 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -200,7 +200,7 @@ /* #undef LG_QUANTUM */ /* One page is 2^LG_PAGE bytes. */ -#define LG_PAGE 16 +#define LG_PAGE @JEMALLOC_LG_PAGE@ /* Maximum number of regions in a slab. */ /* #undef CONFIG_LG_SLAB_MAXREGS */ @@ -210,7 +210,7 @@ * system does not explicitly support huge pages; system calls that require * explicit huge page support are separately configured. */ -#define LG_HUGEPAGE 29 +#define LG_HUGEPAGE @JEMALLOC_LG_HUGEPAGE@ /* * If defined, adjacent virtual memory mappings with identical attributes From d5edf7aa5158fea09f2e0b80eaca05ada159993c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 14:58:10 +0000 Subject: [PATCH 284/296] Backport #125076 to 26.8: Fix StorageKafka2: memory leak with partitions shuffling --- src/Storages/Kafka/KafkaConsumer2.cpp | 59 +++++++++++++++++++++++++++ src/Storages/Kafka/KafkaConsumer2.h | 5 +++ 2 files changed, 64 insertions(+) diff --git a/src/Storages/Kafka/KafkaConsumer2.cpp b/src/Storages/Kafka/KafkaConsumer2.cpp index 9cad254f98ab8..3320c8b5b86d6 100644 --- a/src/Storages/Kafka/KafkaConsumer2.cpp +++ b/src/Storages/Kafka/KafkaConsumer2.cpp @@ -36,6 +36,7 @@ using namespace std::chrono_literals; static constexpr auto EVENT_POLL_TIMEOUT = 50ms; static constexpr auto DRAIN_TIMEOUT_MS = 5000ms; static constexpr auto GET_KAFKA_METADATA_TIMEOUT_MS = 1000ms; +static constexpr size_t DRAIN_REMOVED_PARTITION_BATCH_SIZE = 1024; bool KafkaConsumer2::TopicPartition::operator<(const TopicPartition & other) const { @@ -148,6 +149,51 @@ void KafkaConsumer2::cleanQueuesAndMessages() void KafkaConsumer2::initializeQueues(const cppkafka::TopicPartitionList & topic_partitions) { + std::unordered_set new_partitions; + for (const auto & topic_partition : topic_partitions) + new_partitions.emplace(TopicPartition{topic_partition.get_topic(), topic_partition.get_partition()}); + + /// After a failed call `queues` may not match the consumer's assignment, so take the assignment from librdkafka. + cppkafka::TopicPartitionList removed_partitions; + for (const auto & topic_partition : consumer->get_assignment()) + if (!new_partitions.contains(TopicPartition{topic_partition.get_topic(), topic_partition.get_partition()})) + removed_partitions.emplace_back(topic_partition.get_topic(), topic_partition.get_partition()); + + if (!removed_partitions.empty()) + { + /// Removing a partition from the assignment only stops its fetcher: the messages librdkafka has already + /// prefetched stay in the partition queue, which is owned by librdkafka and is not freed when we drop our handle. + /// Pausing makes these messages outdated, and reading the queue drops outdated messages and frees them. + /// Pausing alone is not enough: librdkafka purges the queue on pause, but the purge stops at the version barrier + /// that the pause itself puts at the head of the queue. + /// Remember the partitions before pausing them, so that they are resumed when they come back even if the pause + /// succeeds only for some of them and throws, or draining throws. + for (const auto & topic_partition : removed_partitions) + paused_partitions.emplace(TopicPartition{topic_partition.get_topic(), topic_partition.get_partition()}); + consumer->pause_partitions(removed_partitions); + + for (const auto & topic_partition : removed_partitions) + { + TopicPartition key{topic_partition.get_topic(), topic_partition.get_partition()}; + /// The handle is lost if the previous call failed after the assignment. + if (!queues.contains(key)) + queues.emplace(key, consumer->get_partition_queue(topic_partition)); + const auto & queue = queues.at(key); + LOG_DEBUG( + log, + "Draining {} queued entries of partition {}:{} that is removed from the assignment", + queue.get_length(), + key.topic, + key.partition_id); + /// After the pause every queued message is outdated, so `consume_batch` frees them without returning them, + /// and normally the first call empties the queue. A paused partition gets no new messages, so the queue only + /// shrinks and its current length bounds the number of batches. + for (size_t batches_left = queue.get_length() / DRAIN_REMOVED_PARTITION_BATCH_SIZE + 1; batches_left > 0; --batches_left) + if (queue.consume_batch(DRAIN_REMOVED_PARTITION_BATCH_SIZE, 0ms).empty()) + break; + } + } + cleanQueuesAndMessages(); queues.reserve(topic_partitions.size()); @@ -161,6 +207,19 @@ void KafkaConsumer2::initializeQueues(const cppkafka::TopicPartitionList & topic consumer->get_partition_queue(topic_partition)); consumer->assign(topic_partitions); + + /// Unassigning clears only librdkafka's own pause flag, so a partition we paused stays paused when it comes back. + cppkafka::TopicPartitionList partitions_to_resume; + for (const auto & topic_partition : new_partitions) + if (paused_partitions.contains(topic_partition)) + partitions_to_resume.emplace_back(topic_partition.topic, topic_partition.partition_id); + if (!partitions_to_resume.empty()) + { + consumer->resume_partitions(partitions_to_resume); + /// Forget the partitions only after the resume succeeded, so that if it throws, the next call retries it. + for (const auto & topic_partition : partitions_to_resume) + paused_partitions.erase(TopicPartition{topic_partition.get_topic(), topic_partition.get_partition()}); + } } catch (...) { diff --git a/src/Storages/Kafka/KafkaConsumer2.h b/src/Storages/Kafka/KafkaConsumer2.h index aa15d0d994fcd..963ab9e565390 100644 --- a/src/Storages/Kafka/KafkaConsumer2.h +++ b/src/Storages/Kafka/KafkaConsumer2.h @@ -10,6 +10,8 @@ #include #include +#include + namespace CurrentMetrics { extern const Metric KafkaConsumers; @@ -173,6 +175,9 @@ class KafkaConsumer2 : public IKafkaExceptionInfoSink, public std::enable_shared // order is important, need to be destructed before consumer std::unordered_map queues; + /// Partitions paused by `initializeQueues` when they were removed from the assignment. They have to be resumed + /// if they are assigned again, because librdkafka clears only its own pause flag on unassign. + std::unordered_set paused_partitions; const Names topics; bool polledDataUnusable(const TopicPartition & topic_partition) const; From dfd6105cf3f7b3dfb118506cc4697260495f89e4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 15:01:36 +0000 Subject: [PATCH 285/296] Backport #121010 to 26.8: Fix missing materialized-CTE gate when two CTEs of a set subquery read one shared CTE --- src/Planner/Planner.cpp | 3 + ...ared_by_two_ctes_in_set_subquery.reference | 6 ++ ...cte_shared_by_two_ctes_in_set_subquery.sql | 72 +++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 tests/queries/0_stateless/05230_materialized_cte_shared_by_two_ctes_in_set_subquery.reference create mode 100644 tests/queries/0_stateless/05230_materialized_cte_shared_by_two_ctes_in_set_subquery.sql diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 246a63ac50012..3ddba61f8dcee 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -2121,6 +2121,9 @@ void addBuildSubqueriesForMaterializedCTEsIfNeeded( materialized_cte->cte_name); auto cte_options = select_query_options.subquery(); + /// Gates for a materialized CTE belong to the plan that collected it, where a deeper + /// level yields a higher and therefore dominating gate; this CTE's plan is not that plan. + cte_options.forceMaterializeCTE(false); Planner cte_planner( cte_subquery, cte_options, diff --git a/tests/queries/0_stateless/05230_materialized_cte_shared_by_two_ctes_in_set_subquery.reference b/tests/queries/0_stateless/05230_materialized_cte_shared_by_two_ctes_in_set_subquery.reference new file mode 100644 index 0000000000000..c976a600587d1 --- /dev/null +++ b/tests/queries/0_stateless/05230_materialized_cte_shared_by_two_ctes_in_set_subquery.reference @@ -0,0 +1,6 @@ +4 +3 +3 +0 +2 +1 diff --git a/tests/queries/0_stateless/05230_materialized_cte_shared_by_two_ctes_in_set_subquery.sql b/tests/queries/0_stateless/05230_materialized_cte_shared_by_two_ctes_in_set_subquery.sql new file mode 100644 index 0000000000000..cb12f530b24a2 --- /dev/null +++ b/tests/queries/0_stateless/05230_materialized_cte_shared_by_two_ctes_in_set_subquery.sql @@ -0,0 +1,72 @@ +-- A set subquery declaring a MATERIALIZED CTE that two OTHER MATERIALIZED CTEs of that subquery read +-- (a UNION ALL whose branches each chain two CTEs, or a single-branch diamond) lost the materialization +-- gate for the shared CTE, so a reader ran before its writer: "Reading from materialized CTE 'c' before +-- its materialization completed - DelayedPortsProcessor gate is missing in the query plan". + +SET enable_analyzer = 1; +SET enable_materialized_cte = 1; +SET enable_lightweight_update = 1; +-- Pin: the table must be read with more than one stream, or no reader of the side pipeline is ever +-- scheduled and the bug is invisible. +SET max_threads = 3; + +DROP TABLE IF EXISTS t_05230; + +CREATE TABLE t_05230 (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +-- Pin: one part per INSERT, kept separate, for the same reason as max_threads above. +SYSTEM STOP MERGES t_05230; +INSERT INTO t_05230 VALUES (1, 200); +INSERT INTO t_05230 VALUES (2, 1048578); +INSERT INTO t_05230 VALUES (3, 7); +INSERT INTO t_05230 VALUES (4, 0); +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_05230' AND active; + +-- Pin: the CTEs are materialized rather than inlined, so the cases below are not vacuous. A CTE read +-- only once is inlined, which is why each one is read twice. The UNION ALL count is 3 because the two +-- branches' identical `c` merge into one CTE while their differing `d` bodies stay two. +SELECT countIf(explain LIKE '%MaterializingCTE (Materializing CTE:%') FROM ( + EXPLAIN SELECT count() FROM t_05230 WHERE v IN ( + WITH c AS MATERIALIZED (SELECT number AS x FROM numbers(3)), + d AS MATERIALIZED (SELECT x + 1048577 AS y FROM c) + SELECT DISTINCT a.y FROM d AS a, d AS b + UNION ALL + WITH c AS MATERIALIZED (SELECT number AS x FROM numbers(3)), + d AS MATERIALIZED (SELECT x + 200 AS y FROM c) + SELECT a.y FROM d AS a, d AS b)); + +SELECT countIf(explain LIKE '%MaterializingCTE (Materializing CTE:%') FROM ( + EXPLAIN SELECT count() FROM t_05230 WHERE v IN ( + WITH c AS MATERIALIZED (SELECT number AS x FROM numbers(3)), + d AS MATERIALIZED (SELECT x + 200 AS y FROM c), + e AS MATERIALIZED (SELECT x + 1048577 AS y FROM c) + SELECT d1.y FROM d AS d1, d AS d2, e AS e1, e AS e2)); + +-- Negative control for the pin above: without the MATERIALIZED keyword there are no writer steps at +-- all, so the pin measures materialization rather than always holding. +SELECT countIf(explain LIKE '%MaterializingCTE (Materializing CTE:%') FROM ( + EXPLAIN SELECT count() FROM t_05230 WHERE v IN ( + WITH c AS (SELECT number AS x FROM numbers(3)), + d AS (SELECT x + 1048577 AS y FROM c) + SELECT DISTINCT a.y FROM d AS a, d AS b)); + +-- The reported shape, as a SELECT: the subquery yields {1048577..1048579} + {200..202}, matching v = 200 +-- and v = 1048578 only. +SELECT count() FROM t_05230 WHERE v IN ( + WITH c AS MATERIALIZED (SELECT number AS x FROM numbers(3)), + d AS MATERIALIZED (SELECT x + 1048577 AS y FROM c) + SELECT DISTINCT a.y FROM d AS a, d AS b + UNION ALL + WITH c AS MATERIALIZED (SELECT number AS x FROM numbers(3)), + d AS MATERIALIZED (SELECT x + 200 AS y FROM c) + SELECT a.y FROM d AS a, d AS b); + +-- The same defect without any UNION: two distinct CTEs reading one shared CTE. The subquery yields +-- {200..202}, matching v = 200 only. +SELECT count() FROM t_05230 WHERE v IN ( + WITH c AS MATERIALIZED (SELECT number AS x FROM numbers(3)), + d AS MATERIALIZED (SELECT x + 200 AS y FROM c), + e AS MATERIALIZED (SELECT x + 1048577 AS y FROM c) + SELECT d1.y FROM d AS d1, d AS d2, e AS e1, e AS e2); + +DROP TABLE t_05230; From 39fc661410bca4bc8611ef5f9b1871dea6d033e9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 13 Oct 2026 16:33:23 +0000 Subject: [PATCH 286/296] Update autogenerated version to 26.8.26.12 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index ac9b215343d89..372b3aaad469a 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54538) +SET(VERSION_REVISION 54539) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 26) -SET(VERSION_GITHASH fe4c23110e19371e9275dc2a8d535d7208b4e53e) -SET(VERSION_DESCRIBE v26.8.26.1-lts) -SET(VERSION_STRING 26.8.26.1) +SET(VERSION_PATCH 27) +SET(VERSION_GITHASH a9f78f125b269c5cb9013dc793330e45a0bc316b) +SET(VERSION_DESCRIBE v26.8.27.1-lts) +SET(VERSION_STRING 26.8.27.1) # end of autochange From 543056992bdd05769f75a5299ed316a098b1d84e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 03:48:26 +0000 Subject: [PATCH 287/296] Backport #125515 to 26.8: Keep the user's settings in the embedded client of the web terminal and SSH --- src/Server/ClientEmbedded/ClientEmbedded.cpp | 7 ++- tests/integration/test_webterminal/test.py | 57 ++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/Server/ClientEmbedded/ClientEmbedded.cpp b/src/Server/ClientEmbedded/ClientEmbedded.cpp index 2e847ba8d779c..53ab1407849ba 100644 --- a/src/Server/ClientEmbedded/ClientEmbedded.cpp +++ b/src/Server/ClientEmbedded/ClientEmbedded.cpp @@ -179,8 +179,13 @@ try addOptionsToTheClientConfiguration(options); /// Apply settings specified as command line arguments (read environment variables). + /// Apply only the changed settings and check them against the constraints: replacing the whole + /// `Settings` would drop what the session context got from the user's profiles and roles + /// (e.g. `readonly`, resource limits, `additional_table_filters`). global_context = session->sessionContext(); - global_context->setSettings(*cmd_settings); + SettingsChanges cmd_settings_changes = cmd_settings->changes(); + global_context->checkSettingsConstraints(cmd_settings_changes, SettingSource::QUERY); + global_context->applySettingsChanges(cmd_settings_changes); is_interactive = stdin_is_a_tty; /// If a query is passed via SSH - just append it to the list of queries to execute: diff --git a/tests/integration/test_webterminal/test.py b/tests/integration/test_webterminal/test.py index bd6588c28ebce..8141d45ba4e14 100644 --- a/tests/integration/test_webterminal/test.py +++ b/tests/integration/test_webterminal/test.py @@ -568,3 +568,60 @@ def test_tab_completion_respects_session_user(): instance.query("DROP USER IF EXISTS completer") instance.query("DROP TABLE IF EXISTS default.visible_completion_target") instance.query("DROP TABLE IF EXISTS default.hidden_completion_target") + + +def test_session_keeps_user_settings(): + """The embedded client must keep the settings of the authenticated user. + + It used to replace the session settings with default values, dropping + everything that comes from the user's profiles, roles and + `CREATE USER ... SETTINGS`, so a `readonly` user got write access and + resource limits were removed. Settings sent along with a query must also + be checked against the constraints. + """ + instance.query( + "CREATE USER restricted_terminal_user IDENTIFIED WITH plaintext_password BY 'restricted_password' " + "SETTINGS readonly = 1, max_memory_usage = 5000000 CONST" + ) + + try: + sock = _open_ws(instance.ip_address, 8123) + try: + _ws_send_text( + sock, + json.dumps( + { + "type": "auth", + "user": "restricted_terminal_user", + "password": "restricted_password", + } + ), + ) + + output = _read_pty_until(sock, timeout=20, marker=":) ") + assert ":) " in output, f"no prompt from the embedded client: {output!r}" + + _ws_send_binary( + sock, + b"SELECT concat('settings_', getSetting('readonly')::String, '_', getSetting('max_memory_usage')::String) FORMAT TSV\r", + ) + output = _read_pty_until(sock, timeout=10, marker="settings_1_5000000\r\n") + assert "settings_1_5000000\r\n" in output, ( + f"the session settings of the user were not preserved: {output!r}" + ) + + # Wait for the next prompt: while a query runs, the client reads the + # keyboard (to toggle the progress table) and would swallow the next query. + if ":) " not in output.split("settings_1_5000000\r\n", 1)[1]: + output = _read_pty_until(sock, timeout=10, marker=":) ") + assert ":) " in output, f"no prompt after the first query: {output!r}" + + _ws_send_binary(sock, b"SELECT 'bypassed' SETTINGS readonly = 0 FORMAT TSV\r") + output = _read_pty_until(sock, timeout=10, marker="READONLY") + assert "READONLY" in output and "bypassed\r\n" not in output, ( + f"changing `readonly` in the query was not rejected: {output!r}" + ) + finally: + sock.close() + finally: + instance.query("DROP USER IF EXISTS restricted_terminal_user") From ed59709adfb0d26b8edb40b62a6538ec5b3ae83b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 08:12:43 +0000 Subject: [PATCH 288/296] Backport #117351 to 26.8: Check `allow_statistics` only when validating a mutation, not when running it --- src/Interpreters/InterpreterAlterQuery.cpp | 24 +++++ src/Interpreters/MutationsInterpreter.cpp | 13 ++- .../__init__.py | 0 .../config/disable_statistics.xml | 12 +++ .../test.py | 99 +++++++++++++++++++ ...erialize_statistics_session_gate.reference | 6 ++ ...75_materialize_statistics_session_gate.sql | 37 +++++++ 7 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_statistics_mutation_session_gate/__init__.py create mode 100644 tests/integration/test_statistics_mutation_session_gate/config/disable_statistics.xml create mode 100644 tests/integration/test_statistics_mutation_session_gate/test.py create mode 100644 tests/queries/0_stateless/05175_materialize_statistics_session_gate.reference create mode 100644 tests/queries/0_stateless/05175_materialize_statistics_session_gate.sql diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index d9faa8bfa31a5..016cda373bac0 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -64,6 +64,7 @@ namespace Setting extern const SettingsSeconds lock_acquire_timeout; extern const SettingsAlterUpdateMode alter_update_mode; extern const SettingsBool enable_lightweight_update; + extern const SettingsBool allow_statistics; extern const SettingsBool validate_mutation_query; extern const SettingsTimezone session_timezone; extern const SettingsUInt64 max_parser_depth; @@ -85,6 +86,7 @@ namespace ErrorCodes extern const int UNKNOWN_DATABASE; extern const int QUERY_IS_PROHIBITED; extern const int SUPPORT_IS_DISABLED; + extern const int INCORRECT_QUERY; } namespace @@ -310,6 +312,27 @@ std::optional tryRewriteToLightweightUpdate(CommandSegments & segments, return res; } +/// `allow_statistics` gates the statistics DDL - whether statistics may be declared on a table - +/// so only the submitting session's value is meaningful for it, and the gate must be applied +/// exactly once, here at submission time. +/// +/// `ALTER TABLE ... ADD/DROP/MODIFY STATISTICS` carries an `AlterCommand` and is gated by +/// `AlterCommands::validate`, but `MATERIALIZE STATISTICS` parses into a `MutationCommands` +/// segment only, so it is checked here, next to the other submission-time mutation validation. +/// `MutationsInterpreter` does not check it: it also runs in the background with a context that +/// reads the server-default profile instead of the submitting session. +void checkStatisticsMutationsAreAllowed(const MutationCommands & commands, const Settings & settings) +{ + if (settings[Setting::allow_statistics]) + return; + + for (const auto & command : commands) + { + if (command.type == MutationCommand::MATERIALIZE_STATISTICS || command.type == MutationCommand::DROP_STATISTICS) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Alter table with statistics is disabled. Turn on allow_statistics"); + } +} + BlockIO runCommandSegments(CommandSegments & segments, const StoragePtr & table, const ContextPtr & context) { BlockIO res; @@ -345,6 +368,7 @@ BlockIO runCommandSegments(CommandSegments & segments, const StoragePtr & table, { auto metadata_snapshot = table->getInMemoryMetadataPtr(context, true); table->checkMutationIsPossible(*mutation_commands, settings); + checkStatisticsMutationsAreAllowed(*mutation_commands, settings); /// Replicated-storage non-determinism check must always run, even when /// `validate_mutation_query=0` — bypassing it would let nondeterministic mutations /// diverge replicas. The heavier query-shape validation that constructs a full diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index f286a7af274b7..3be2fee7b8c85 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -81,7 +81,6 @@ namespace Setting extern const SettingsUInt64 max_rows_to_transfer; extern const SettingsOverflowMode transfer_overflow_mode; extern const SettingsBool use_concurrency_control; - extern const SettingsBool allow_statistics; extern const SettingsBool validate_mutation_query; extern const SettingsSetOperationMode union_default_mode; extern const SettingsSetOperationMode intersect_default_mode; @@ -1342,8 +1341,12 @@ void MutationsInterpreter::prepare(bool dry_run) } else if (command.type == MutationCommand::MATERIALIZE_STATISTICS) { - if (!context->getSettingsRef()[Setting::allow_statistics]) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Alter table with statistics is disabled. Turn on allow_statistics"); + /// `allow_statistics` gates the DDL: whether statistics may be declared on a table. Only + /// the submitting session's value is meaningful for that, so the gate lives at submission + /// time - `checkStatisticsMutationsAreAllowed` in `InterpreterAlterQuery`. Checking it here + /// asked the background context that executes the mutation, which reads the server-default + /// profile rather than the session that submitted the statement, so a mutation accepted + /// with a session opt-in failed forever and wedged the table's mutation queue. mutation_kind.set(MutationKind::MUTATE_INDEX_STATISTICS_PROJECTION); /// if we execute `ALTER TABLE ... MATERIALIZE STATISTICS ALL`, we materalize all the statistics in this table. @@ -1398,8 +1401,8 @@ void MutationsInterpreter::prepare(bool dry_run) } else if (command.type == MutationCommand::DROP_STATISTICS) { - if (!context->getSettingsRef()[Setting::allow_statistics]) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Alter table with statistics is disabled. Turn on allow_statistics"); + /// See MATERIALIZE_STATISTICS above: gated at submission time, not under the background + /// context that executes the mutation. mutation_kind.set(MutationKind::MUTATE_INDEX_STATISTICS_PROJECTION); diff --git a/tests/integration/test_statistics_mutation_session_gate/__init__.py b/tests/integration/test_statistics_mutation_session_gate/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_statistics_mutation_session_gate/config/disable_statistics.xml b/tests/integration/test_statistics_mutation_session_gate/config/disable_statistics.xml new file mode 100644 index 0000000000000..1d8d744300526 --- /dev/null +++ b/tests/integration/test_statistics_mutation_session_gate/config/disable_statistics.xml @@ -0,0 +1,12 @@ + + + + + 0 + + + diff --git a/tests/integration/test_statistics_mutation_session_gate/test.py b/tests/integration/test_statistics_mutation_session_gate/test.py new file mode 100644 index 0000000000000..ae270af5cea80 --- /dev/null +++ b/tests/integration/test_statistics_mutation_session_gate/test.py @@ -0,0 +1,99 @@ +# https://github.com/ClickHouse/ClickHouse/issues/115448 +# +# `allow_statistics` gates the statistics DDL. It used to be checked twice: once when the statement +# was validated, with the submitting session's value, and once more while the mutation ran - where +# the background context reads the server-default profile rather than that session. With +# `allow_statistics = 0` in the profile and `allow_statistics = 1` in the statement, the mutation was +# therefore accepted and then failed on every retry, sitting in `system.mutations` with +# `is_done = 0` forever and blocking every later mutation on the table. +# +# The server profile is what a stateless test cannot set, which is why this half of the reproducer +# lives here; `05175_materialize_statistics_session_gate` pins the submission-time rejection. + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", + user_configs=["config/disable_statistics.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def unfinished_mutations(table): + return node.query( + f"SELECT countIf(is_done = 0) FROM system.mutations WHERE database = 'default' AND table = '{table}'" + ).strip() + + +def test_statistics_mutations_with_session_opt_in(started_cluster): + node.query("DROP TABLE IF EXISTS t_session_gate SYNC") + node.query( + """ + CREATE TABLE t_session_gate (k UInt64, a Int64) ENGINE = MergeTree ORDER BY k + """ + ) + node.query("INSERT INTO t_session_gate SELECT number, number FROM numbers(100)") + + # `ADD STATISTICS` has always honoured the session opt-in. + node.query( + "ALTER TABLE t_session_gate ADD STATISTICS a TYPE tdigest", + settings={"allow_statistics": 1, "mutations_sync": 2}, + ) + + # `MATERIALIZE STATISTICS` used to be accepted here and then fail forever in the background. + node.query( + "ALTER TABLE t_session_gate MATERIALIZE STATISTICS a", + settings={"allow_statistics": 1, "mutations_sync": 2}, + ) + assert unfinished_mutations("t_session_gate") == "0" + + node.query( + "ALTER TABLE t_session_gate DROP STATISTICS a", + settings={"allow_statistics": 1, "mutations_sync": 2}, + ) + assert unfinished_mutations("t_session_gate") == "0" + + # The table is not wedged: a later, unrelated mutation still completes. + node.query( + "ALTER TABLE t_session_gate UPDATE a = a + 1 WHERE k < 10", + settings={"mutations_sync": 2}, + ) + assert unfinished_mutations("t_session_gate") == "0" + assert node.query("SELECT count() FROM t_session_gate").strip() == "100" + + node.query("DROP TABLE t_session_gate SYNC") + + +def test_statistics_mutations_without_session_opt_in(started_cluster): + node.query("DROP TABLE IF EXISTS t_session_gate_off SYNC") + node.query( + """ + CREATE TABLE t_session_gate_off (k UInt64, a Int64) ENGINE = MergeTree ORDER BY k + """ + ) + + # Without the opt-in the profile's value still refuses the statements up front, and nothing + # reaches the mutation queue. + for statement in ( + "ALTER TABLE t_session_gate_off ADD STATISTICS a TYPE tdigest", + "ALTER TABLE t_session_gate_off MATERIALIZE STATISTICS a", + "ALTER TABLE t_session_gate_off DROP STATISTICS a", + ): + assert "Alter table with statistics is disabled" in node.query_and_get_error( + statement + ) + + assert unfinished_mutations("t_session_gate_off") == "0" + node.query("DROP TABLE t_session_gate_off SYNC") diff --git a/tests/queries/0_stateless/05175_materialize_statistics_session_gate.reference b/tests/queries/0_stateless/05175_materialize_statistics_session_gate.reference new file mode 100644 index 0000000000000..d033580b60410 --- /dev/null +++ b/tests/queries/0_stateless/05175_materialize_statistics_session_gate.reference @@ -0,0 +1,6 @@ +refused at submission +0 +accepted and finishes +0 +100 +0 diff --git a/tests/queries/0_stateless/05175_materialize_statistics_session_gate.sql b/tests/queries/0_stateless/05175_materialize_statistics_session_gate.sql new file mode 100644 index 0000000000000..328b69a5bc38e --- /dev/null +++ b/tests/queries/0_stateless/05175_materialize_statistics_session_gate.sql @@ -0,0 +1,37 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/115448 +-- `allow_statistics` gates the DDL - whether statistics may be declared - and only the submitting +-- session's value is meaningful for that. It used to be checked again while the mutation ran, under +-- the background context that reads the server-default profile, so a `MATERIALIZE STATISTICS` +-- accepted with a session opt-in failed forever and wedged the table's mutation queue. The full +-- reproducer needs a server profile with `allow_statistics = 0`, which a stateless test cannot set; +-- it lives in `tests/integration/test_statistics_mutation_session_gate`. What is pinned here is that +-- submission-time validation still refuses the statement, and that the statement works with the +-- setting on. + +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_materialize_statistics_gate; +CREATE TABLE t_materialize_statistics_gate (k UInt64, a Int64) ENGINE = MergeTree ORDER BY k; +INSERT INTO t_materialize_statistics_gate SELECT number, number FROM numbers(100); + +SELECT 'refused at submission'; +ALTER TABLE t_materialize_statistics_gate ADD STATISTICS a TYPE tdigest SETTINGS allow_statistics = 0; -- { serverError INCORRECT_QUERY } +ALTER TABLE t_materialize_statistics_gate MATERIALIZE STATISTICS a SETTINGS allow_statistics = 0; -- { serverError INCORRECT_QUERY } +ALTER TABLE t_materialize_statistics_gate DROP STATISTICS a SETTINGS allow_statistics = 0; -- { serverError INCORRECT_QUERY } + +-- Nothing of the above reached the mutation queue. +SELECT count() FROM system.mutations +WHERE database = currentDatabase() AND table = 't_materialize_statistics_gate'; + +SELECT 'accepted and finishes'; +ALTER TABLE t_materialize_statistics_gate ADD STATISTICS a TYPE tdigest SETTINGS allow_statistics = 1; +ALTER TABLE t_materialize_statistics_gate MATERIALIZE STATISTICS a SETTINGS allow_statistics = 1; +SELECT countIf(is_done = 0) FROM system.mutations +WHERE database = currentDatabase() AND table = 't_materialize_statistics_gate'; +SELECT count() FROM t_materialize_statistics_gate; + +ALTER TABLE t_materialize_statistics_gate DROP STATISTICS a SETTINGS allow_statistics = 1; +SELECT countIf(is_done = 0) FROM system.mutations +WHERE database = currentDatabase() AND table = 't_materialize_statistics_gate'; + +DROP TABLE t_materialize_statistics_gate; From ce41d20e10289a9aa9f391c97ec9e107824c50a5 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:53:16 +0200 Subject: [PATCH 289/296] Fix build of Iceberg `DROP PARTITION` after external paths support `AlterDropPartitionExecutor` (https://github.com/Altinity/ClickHouse/pull/2533) called `getManifestFileEntriesHandle` and `generateManifestList` without the `ExternalStorageCache` argument added by https://github.com/Altinity/ClickHouse/pull/2551. Pass the cache of `IcebergMetadata` to the executor. --- .../DataLakes/Iceberg/AlterDropPartitionExecutor.cpp | 9 ++++++--- .../DataLakes/Iceberg/AlterDropPartitionExecutor.h | 5 ++++- .../ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp | 3 ++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp index af38b06ed41aa..43e790061e2bd 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp @@ -218,7 +218,8 @@ AlterDropPartitionExecutor::AlterDropPartitionExecutor( const PersistentTableComponents & components_, const DataLakeStorageSettings & data_lake_settings_, String write_format_, - LoggerPtr log_) + LoggerPtr log_, + ExternalStorageCache & external_storages_) : command(command_) , metadata(metadata_) , context(context_) @@ -227,6 +228,7 @@ AlterDropPartitionExecutor::AlterDropPartitionExecutor( , data_lake_settings(data_lake_settings_) , write_format(std::move(write_format_)) , log(std::move(log_)) + , external_storages(external_storages_) { } @@ -365,7 +367,7 @@ AlterDropPartitionExecutor::discoverTargetFilePaths(const SnapshotState & state, for (const auto & manifest_key : state.snapshot->manifest_list_entries) { - auto handle = getManifestFileEntriesHandle(object_storage, components, context, log, manifest_key, state.schema_id); + auto handle = getManifestFileEntriesHandle(object_storage, components, context, log, manifest_key, state.schema_id, external_storages); collect(handle.getFilesWithoutDeleted(FileContentType::DATA)); collect(handle.getFilesWithoutDeleted(FileContentType::POSITION_DELETE)); @@ -460,7 +462,7 @@ AlterDropPartitionExecutor::buildDropPlan(const SnapshotState & state, const Tar for (const auto & manifest_key : state.snapshot->manifest_list_entries) { - auto handle = getManifestFileEntriesHandle(object_storage, components, context, log, manifest_key, state.schema_id); + auto handle = getManifestFileEntriesHandle(object_storage, components, context, log, manifest_key, state.schema_id, external_storages); size_t entries_to_keep = 0; size_t entries_to_remove = 0; @@ -575,6 +577,7 @@ AlterDropPartitionExecutor::ManifestListWriteResult AlterDropPartitionExecutor:: components.path_resolver, state.metadata_object, object_storage, + external_storages, context, /*manifest_entry_names=*/{}, new_snapshot, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.h index de47a35ab9ec4..8557c7faf0e2b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.h @@ -30,6 +30,7 @@ namespace DB class FileNamesGenerator; class IcebergMetadata; +struct ExternalStorageCache; namespace Iceberg { @@ -45,7 +46,8 @@ class AlterDropPartitionExecutor const PersistentTableComponents & components_, const DataLakeStorageSettings & data_lake_settings_, String write_format_, - LoggerPtr log_); + LoggerPtr log_, + ExternalStorageCache & external_storages_); void run(); @@ -110,6 +112,7 @@ class AlterDropPartitionExecutor const DataLakeStorageSettings & data_lake_settings; String write_format; LoggerPtr log; + ExternalStorageCache & external_storages; }; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 593befd3c69e5..1504317340dd8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -976,7 +976,8 @@ void IcebergMetadata::alterPartitionDropImpl(const PartitionCommand & command, C persistent_components, data_lake_settings, write_format, - log); + log, + *external_storages); executor.run(); } From 6ebd5008dbb6ad6a70f9ed383ae2e4beb5baf929 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:53:16 +0200 Subject: [PATCH 290/296] Fix `Native` binary type encoding for `remote` with a table function A single-shard `remote` with a nested table function is sent to the remote server as an initial query, so that server applies `output_format_native_encode_types_in_binary_format` and `input_format_native_decode_types_in_binary_format` to the `Native` blocks of the wire, while the server-side `Connection` read and wrote them with type names. Make `Connection` without format settings from the caller follow the same rule as the server (see `getNativeWireFormatSettings`). Fixes `05317_native_binary_types_between_servers`: https://altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2572&sha=98fe94fb9ce108eda1b9bca34cb71b325c1058e4&name_0=PR&name_1=Fast+test&name_2=Tests https://github.com/Altinity/ClickHouse/pull/2572 --- src/Client/Connection.cpp | 27 +++++++++++++++++++++++---- src/Client/Connection.h | 5 +++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 997b8fbc10bcb..71df187ab8cfe 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -73,8 +73,11 @@ namespace DB { namespace Setting { + extern const SettingsUInt64 input_format_binary_max_type_complexity; + extern const SettingsBool input_format_native_decode_types_in_binary_format; extern const SettingsString network_compression_method; extern const SettingsInt64 network_zstd_compression_level; + extern const SettingsBool output_format_native_encode_types_in_binary_format; } namespace FailPoints @@ -1080,6 +1083,22 @@ void Connection::sendQuery( query_id = query_id_; + /// The server applies the binary type encoding settings to the `Native` blocks of every query except a secondary one + /// (see `getNativeWireFormatSettings`), and a server sends initial queries to other servers too. + query_wire_format_settings.reset(); + if (settings && !(client_info && client_info->query_kind == ClientInfo::QueryKind::SECONDARY_QUERY)) + { + const bool server_encodes = (*settings)[Setting::output_format_native_encode_types_in_binary_format]; + const bool server_decodes = (*settings)[Setting::input_format_native_decode_types_in_binary_format]; + if (server_encodes || server_decodes) + { + query_wire_format_settings.emplace(); + query_wire_format_settings->native.decode_types_in_binary_format = server_encodes; + query_wire_format_settings->native.encode_types_in_binary_format = server_decodes; + query_wire_format_settings->binary.max_binary_type_complexity = (*settings)[Setting::input_format_binary_max_type_complexity]; + } + } + /// Avoid reusing connections that had been left in the intermediate state /// (i.e. not all packets had been sent). bool completed = false; @@ -1260,7 +1279,7 @@ void Connection::sendData(const Block & block, const String & name, bool scalar) else maybe_compressed_out = out; - block_out = std::make_unique(*maybe_compressed_out, server_revision, std::make_shared(block.cloneEmpty()), format_settings); + block_out = std::make_unique(*maybe_compressed_out, server_revision, std::make_shared(block.cloneEmpty()), getNativeFormatSettings()); } if (scalar) @@ -1729,7 +1748,7 @@ void Connection::initBlockInput() initMaybeCompressedInput(); /// The server may send marshalled result blocks; their consumers convert them back. block_in = std::make_unique( - *maybe_compressed_in, server_revision, format_settings, ISerialization::KindSet::all()); + *maybe_compressed_in, server_revision, getNativeFormatSettings(), ISerialization::KindSet::all()); } } @@ -1746,7 +1765,7 @@ void Connection::initBlockLogsInput() } /// Have to return superset of SystemLogsQueue::getSampleBlock() columns - block_logs_in = std::make_unique(*logs_buf, server_revision, format_settings); + block_logs_in = std::make_unique(*logs_buf, server_revision, getNativeFormatSettings()); } } @@ -1762,7 +1781,7 @@ void Connection::initBlockProfileEventsInput() profile_events_buf = maybe_compressed_in.get(); } - block_profile_events_in = std::make_unique(*profile_events_buf, server_revision, format_settings); + block_profile_events_in = std::make_unique(*profile_events_buf, server_revision, getNativeFormatSettings()); } } diff --git a/src/Client/Connection.h b/src/Client/Connection.h index ac5d4f57e84c4..5ff0b775a11b4 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -343,6 +343,11 @@ class Connection : public IServerConnection AsyncCallback async_callback = {}; std::optional format_settings; + /// The binary type encoding of the `Native` blocks of the current query, as the server applies it. + /// Used when the caller does not set `format_settings`. + std::optional query_wire_format_settings; + + std::optional getNativeFormatSettings() const { return format_settings ? format_settings : query_wire_format_settings; } void connect(const ConnectionTimeouts & timeouts); From 7d824c68ef8b29246a336b7411cd49e9d12e31b8 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:53:16 +0200 Subject: [PATCH 291/296] Adjust `05153_text_index_mergeTreeTextIndex_remote` for `remote` with a table function In Antalya a single-shard `remote` with a nested table function runs an initial query on the remote server as the user of the connection, so `mergeTreeTextIndex` checks the access of that user instead of refusing a secondary query. https://altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2572&sha=98fe94fb9ce108eda1b9bca34cb71b325c1058e4&name_0=PR&name_1=Fast+test&name_2=Tests https://github.com/Altinity/ClickHouse/pull/2572 --- .../05153_text_index_mergeTreeTextIndex_remote.reference | 2 +- .../0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference index 80d7d44bd4660..f2f563255d594 100644 --- a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference @@ -3,7 +3,7 @@ ACCESS_DENIED ACCESS_DENIED ACCESS_DENIED ACCESS_DENIED -ACCESS_DENIED +2 2 2 2 diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh index 0bd207645ea78..a3171dd851331 100755 --- a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh @@ -50,8 +50,8 @@ run_remote_as_user $CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name" -# Over an ordinary connection the shard runs the query as the user of the connection, which the function refuses -# unless that is the initiating user; with the local shortcut it runs as the user itself. +# Over an ordinary connection `remote` with a table function on a single shard runs an initial query there as the user +# of the connection, which checks the access of that user; with the local shortcut it runs as the user itself. run_remote_as_user $CLICKHOUSE_CLIENT -q " From 56241dce698c7c1bfe053501188d2d83c241aeb8 Mon Sep 17 00:00:00 2001 From: blau-ai Date: Fri, 16 Oct 2026 07:33:17 +0000 Subject: [PATCH 292/296] Fix Iceberg table-engine secret-args crash and stale strip-nothing stats test The backport of #122770 refactored findTableEngineSecretArguments() to dispatch through a std::function find_secrets, but the Iceberg branch kept calling findIcebergTableEngineSecretArguments() directly without assigning find_secrets and without returning. Execution then fell through to the trailing find_secrets() call with an empty std::function, so masking any CREATE TABLE ... ENGINE=Iceberg(...) (via hasSecretParts() during query logging) threw std::bad_function_call. Assign the lambda like every sibling branch so classifyPositionalArguments() still runs before it. This crashed test_storage_iceberg_disks in integration tests. The upstream 26.8.27 merge moved Iceberg data-file column sizes from an in-memory byteSize() estimate in update() to the real on-disk sizes reported by the output format at finalize() (addColumnSizesOnDisk()). The gtest still expected update() alone to populate column sizes, so getColumnSizes() was empty. Populate sizes via addColumnSizesOnDisk() to match the new design (addColumnSizesOnDisk uses +=, so restoring byteSize() in update() would double-count). Co-Authored-By: Claude Opus 4.8 Signed-off-by: blau-ai --- src/Parsers/FunctionSecretArgumentsFinder.cpp | 2 +- .../tests/gtest_iceberg_strip_nothing.cpp | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 62fa0ea206154..fac2fb9016e45 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -1041,7 +1041,7 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() } else if (engine_name == "Iceberg") { - findIcebergTableEngineSecretArguments(); + find_secrets = [this] { findIcebergTableEngineSecretArguments(); }; } else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") || (engine_name == "GCS") || (engine_name == "DeltaLake") || (engine_name == "DeltaLakeS3") || (engine_name == "Hudi") diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_strip_nothing.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_strip_nothing.cpp index 42a1617b8d4f0..06e84be767286 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_strip_nothing.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_strip_nothing.cpp @@ -1,6 +1,8 @@ #include #include +#include +#include #include #include #include @@ -165,6 +167,19 @@ std::vector fieldIdsOf(const std::vector> & stats) return ids; } +/// Column sizes are populated from the data file's on-disk column sizes (reported by the output +/// format at finalize, see MultipleFileWriter), not from `update`. Mirror that here so the size +/// getters have something to return; the values are irrelevant, only the covered field ids matter. +void addColumnSizes(DataFileStatistics & stats) +{ + Block sample_block({ + ColumnWithTypeAndName(type("Int64"), "c1"), + ColumnWithTypeAndName(type("Nullable(String)"), "c2"), + ColumnWithTypeAndName(type("Nullable(Nothing)"), "c3"), + }); + stats.addColumnSizesOnDisk({{"c1", 1}, {"c2", 1}, {"c3", 1}}, sample_block); +} + } TEST(IcebergStripNothing, ExcludedColumnsHaveNoStatistics) @@ -172,6 +187,7 @@ TEST(IcebergStripNothing, ExcludedColumnsHaveNoStatistics) DataFileStatistics stats(statisticsSchema()); stats.excludeColumns({false, false, true}); stats.update(statisticsChunk()); + addColumnSizes(stats); const std::vector written = {1, 2}; EXPECT_EQ(fieldIdsOf(stats.getColumnSizes()), written); @@ -191,6 +207,7 @@ TEST(IcebergStripNothing, StatisticsWithoutExclusionCoverEveryColumn) { DataFileStatistics stats(statisticsSchema()); stats.update(statisticsChunk()); + addColumnSizes(stats); const std::vector all = {1, 2, 3}; EXPECT_EQ(fieldIdsOf(stats.getColumnSizes()), all); From fed75c61e77f04336c60803f28105086afffd633 Mon Sep 17 00:00:00 2001 From: blau-ai Date: Fri, 16 Oct 2026 13:23:06 +0000 Subject: [PATCH 293/296] Fix Iceberg write crash on excluded columns in addColumnSizesOnDisk The 26.8.27 bump combined two Iceberg changes that each predate the other: upstream #121392 added addColumnSizesOnDisk(), which iterates every schema field and throws LOGICAL_ERROR if a field reports no on-disk size; Altinity #2363 added the `unknown`-type write path, which strips columns that are not physically written (e.g. a column of only the Iceberg `unknown` type) and marks their schema positions via excludeColumns(). The getters (getColumnSizes / getNullCounts / getLowerBounds / getUpperBounds) all skip excluded positions, but addColumnSizesOnDisk() did not, so finalize() aborted with "Written data file does not report the on-disk size of column " and took the server down. Skip excluded positions there too, matching the getters. This crashed test_storage_iceberg_no_spark/test_unknown_type.py (the write tests, plus the cascade of Connection-refused failures after the server died). For normal tables excludeColumns() is never called, so isExcluded() is always false and behavior is unchanged. Validated by CI on this PR, not locally (no build in this environment). Signed-off-by: blau-ai Co-Authored-By: Claude Opus 4.8 --- .../ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp index 263322746879a..88494c3233055 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp @@ -85,6 +85,11 @@ void DataFileStatistics::addColumnSizesOnDisk(const std::unordered_map Date: Fri, 16 Oct 2026 13:23:16 +0000 Subject: [PATCH 294/296] Mask Iceberg storage_type engine/function args without over-hiding Backport #122770 added a generic fail-closed pass, classifyPositionalArguments(), run before each engine/function's own finder: it hides every positional argument that appears after the first `key = value` argument. The Altinity-only `storage_type = ...` argument of the Iceberg engine and table function is exactly such a leading named argument, so for Iceberg(storage_type = 's3', , , ) the generic pass marked , AND as secret before the Iceberg finder (which skips storage_type and delegates to the S3/Azure finder) could classify them, producing Iceberg(storage_type = 's3', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') instead of hiding only the secret. Bypass the generic pass for Iceberg by returning right after its finder, exactly as BigQuery already does; the Iceberg finder skips storage_type and delegates to findS3*/findAzure*, which run their own positional-after-named fail-close (classifyS3Arguments), so the security property is preserved. The database-engine Iceberg path uses a different finder (findDataLakeCatalogSecretArguments) and is left unchanged. Fixes test_mask_sensitive_info/test.py::test_create_table and ::test_table_functions. Validated by CI on this PR, not locally (no build in this environment). Signed-off-by: blau-ai Co-Authored-By: Claude Opus 4.8 --- src/Parsers/FunctionSecretArgumentsFinder.cpp | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index fac2fb9016e45..5f1f8e3165110 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -334,6 +334,15 @@ void FunctionSecretArgumentsFinder::findOrdinaryFunctionSecretArguments() findBigQuerySecretArguments(); return; } + if ((name == "iceberg") || (name == "icebergCluster")) + { + /// `storage_type = ...` is a named argument that legitimately precedes the positional ones; the + /// Iceberg finder skips it and delegates to the S3/Azure finder, which classifies the remaining + /// positionals itself. Bypass the generic positional-after-named masking below, which would see + /// `storage_type` as a leading named argument and wrongly hide every positional after it. + findIcebergFunctionSecretArguments(/* is_cluster_function= */ name == "icebergCluster"); + return; + } std::function find_secrets; if ((name == "mysql") || (name == "postgresql")) @@ -346,14 +355,6 @@ void FunctionSecretArgumentsFinder::findOrdinaryFunctionSecretArguments() { find_secrets = [this] { findMongoDBSecretArguments(); }; } - else if (name == "iceberg") - { - find_secrets = [this] { findIcebergFunctionSecretArguments(/* is_cluster_function= */ false); }; - } - else if (name == "icebergCluster") - { - find_secrets = [this] { findIcebergFunctionSecretArguments(/* is_cluster_function= */ true); }; - } else if ((name == "s3") || (name == "cosn") || (name == "oss") || (name == "deltaLake") || (name == "deltaLakeS3") || (name == "hudi") || (name == "gcs") || (name == "icebergS3") || (name == "paimon") || (name == "paimonS3")) @@ -1041,7 +1042,12 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() } else if (engine_name == "Iceberg") { - find_secrets = [this] { findIcebergTableEngineSecretArguments(); }; + /// `storage_type = ...` is a named argument that legitimately precedes the positional ones; the + /// Iceberg finder skips it and delegates to the S3/Azure finder, which classifies the remaining + /// positionals itself. Bypass the generic positional-after-named masking below, which would see + /// `storage_type` as a leading named argument and wrongly hide every positional after it. + findIcebergTableEngineSecretArguments(); + return; } else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") || (engine_name == "GCS") || (engine_name == "DeltaLake") || (engine_name == "DeltaLakeS3") || (engine_name == "Hudi") From 4c289ce012de1f1c921d452ce0ef5ab6e5506b40 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:23:20 +0200 Subject: [PATCH 295/296] Fix `04512`/`04513`/`04514` geo pruning tests for stale Iceberg manifest bounds Iceberg writes now keep the bounds of every column whose bounds can be serialized, instead of dropping all bounds when one column (here the `Point` column) can't be serialized. The tests inserted a placeholder row `(0, ...)` and then replaced its data file with a fixture holding ids 1..4, so the manifest declared `id` bounds of `[0, 0]` and `allow_experimental_iceberg_read_optimization` substituted the constant `0` for `id` without reading the file. Insert the fixture's own rows instead, so the manifest statistics stay true for the fixture that replaces the data file. https://github.com/Altinity/ClickHouse/pull/2572#issuecomment-6216474204 --- ...04512_parquet_geo_pruning_iceberg_renamed_column.sh | 10 ++++++++-- ..._parquet_geo_pruning_iceberg_renamed_bbox_column.sh | 10 ++++++++-- .../04514_parquet_geo_pruning_geostats_fallback.sh | 10 ++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/04512_parquet_geo_pruning_iceberg_renamed_column.sh b/tests/queries/0_stateless/04512_parquet_geo_pruning_iceberg_renamed_column.sh index 3f323797abdc6..b8474f7d866fb 100755 --- a/tests/queries/0_stateless/04512_parquet_geo_pruning_iceberg_renamed_column.sh +++ b/tests/queries/0_stateless/04512_parquet_geo_pruning_iceberg_renamed_column.sh @@ -41,7 +41,13 @@ ${CLICKHOUSE_CLIENT} --query " (id Int32, geometry Point, bbox_xmin Float64, bbox_ymin Float64, bbox_xmax Float64, bbox_ymax Float64) ENGINE = IcebergLocal('${ICEBERG_PATH}', 'Parquet'); - INSERT INTO ${TEST_TABLE} VALUES (0, (0, 0), 0, 0, 0, 0); + -- The same rows as the fixture that replaces this data file below, so the manifest + -- statistics of the data file (e.g. the \`id\` bounds) stay true for the fixture. + INSERT INTO ${TEST_TABLE} VALUES + (1, (-98, 31), -98, 31, -98, 31), + (2, (-97.5, 31.5), -97.5, 31.5, -97.5, 31.5), + (3, (-98, 34), -98, 34, -98, 34), + (4, (-97.5, 34.5), -97.5, 34.5, -97.5, 34.5); " # Bump the schema by renaming the geometry column. Existing data files still @@ -52,7 +58,7 @@ ${CLICKHOUSE_CLIENT} --query " ALTER TABLE ${TEST_TABLE} RENAME COLUMN geometry TO geom_renamed; " -# Replace the just-inserted (throwaway) data file with a hand-crafted fixture +# Replace the just-inserted data file with a hand-crafted fixture # that has the same field_ids, 2 row groups (south Texas ids 1,2; north Texas # ids 3,4), and real `covering.bbox` GeoParquet metadata copied from # `04059_geo_spatial_pruning.parquet`. diff --git a/tests/queries/0_stateless/04513_parquet_geo_pruning_iceberg_renamed_bbox_column.sh b/tests/queries/0_stateless/04513_parquet_geo_pruning_iceberg_renamed_bbox_column.sh index f4a376fa6ea61..4ddadbfb8fd21 100755 --- a/tests/queries/0_stateless/04513_parquet_geo_pruning_iceberg_renamed_bbox_column.sh +++ b/tests/queries/0_stateless/04513_parquet_geo_pruning_iceberg_renamed_bbox_column.sh @@ -36,7 +36,13 @@ ${CLICKHOUSE_CLIENT} --query " (id Int32, geometry Point, bbox_xmin Float64, bbox_ymin Float64, bbox_xmax Float64, bbox_ymax Float64) ENGINE = IcebergLocal('${ICEBERG_PATH}', 'Parquet'); - INSERT INTO ${TEST_TABLE} VALUES (0, (0, 0), 0, 0, 0, 0); + -- The same rows as the fixture that replaces this data file below, so the manifest + -- statistics of the data file (e.g. the \`id\` bounds) stay true for the fixture. + INSERT INTO ${TEST_TABLE} VALUES + (1, (-98, 31), -98, 31, -98, 31), + (2, (-97.5, 31.5), -97.5, 31.5, -97.5, 31.5), + (3, (-98, 34), -98, 34, -98, 34), + (4, (-97.5, 34.5), -97.5, 34.5, -97.5, 34.5); " # Bump the schema by renaming one of the `covering.bbox` sub-columns. Existing @@ -47,7 +53,7 @@ ${CLICKHOUSE_CLIENT} --query " ALTER TABLE ${TEST_TABLE} RENAME COLUMN bbox_xmin TO bbox_xmin_renamed; " -# Replace the just-inserted (throwaway) data file with the same hand-crafted +# Replace the just-inserted data file with the same hand-crafted # fixture used by 04512: 2 row groups (south Texas ids 1,2; north Texas ids # 3,4), real `covering.bbox` GeoParquet metadata pointing at its own raw # `bbox_xmin`/etc columns. diff --git a/tests/queries/0_stateless/04514_parquet_geo_pruning_geostats_fallback.sh b/tests/queries/0_stateless/04514_parquet_geo_pruning_geostats_fallback.sh index 4a9ce34d1b5b4..4fcc849d4d5ed 100755 --- a/tests/queries/0_stateless/04514_parquet_geo_pruning_geostats_fallback.sh +++ b/tests/queries/0_stateless/04514_parquet_geo_pruning_geostats_fallback.sh @@ -36,7 +36,13 @@ ${CLICKHOUSE_CLIENT} --query " (id Int32, geometry Point, bbox_xmin Float64, bbox_ymin Float64, bbox_xmax Float64, bbox_ymax Float64) ENGINE = IcebergLocal('${ICEBERG_PATH}', 'Parquet'); - INSERT INTO ${TEST_TABLE} VALUES (0, (0, 0), 0, 0, 0, 0); + -- The same rows as the fixture that replaces this data file below, so the manifest + -- statistics of the data file (e.g. the \`id\` bounds) stay true for the fixture. + INSERT INTO ${TEST_TABLE} VALUES + (1, (-98, 31), -98, 31, -98, 31), + (2, (-97.5, 31.5), -97.5, 31.5, -97.5, 31.5), + (3, (-98, 34), -98, 34, -98, 34), + (4, (-97.5, 34.5), -97.5, 34.5, -97.5, 34.5); " # Bump the schema by renaming the geometry column. Existing data files still carry `geometry` @@ -47,7 +53,7 @@ ${CLICKHOUSE_CLIENT} --query " ALTER TABLE ${TEST_TABLE} RENAME COLUMN geometry TO geom_renamed; " -# Replace the just-inserted (throwaway) data file with a hand-crafted fixture: 2 row groups +# Replace the just-inserted data file with a hand-crafted fixture: 2 row groups # (south Texas ids 1,2; north Texas ids 3,4), real `geospatial_statistics.bbox` metadata on the # geometry column and no `covering.bbox` at all. DATA_FILE=$(find "${ICEBERG_PATH}/data" -name '*.parquet' | head -n 1) From a12c7e0bf9f4d587ee5c3a340c2e02304ad3ce81 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Sun, 18 Oct 2026 16:32:49 +0200 Subject: [PATCH 296/296] Fix duplicated `ExternalStorageCache` parameter in `AlterDropPartitionExecutor` after merge The merge of `antalya-26.8` kept both the bump-branch trailing `external_storages_` constructor parameter and the one added after `object_storage_`, producing a duplicate parameter and member. Keep the `antalya-26.8` layout. https://altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2572&sha=7d6e1e5fde8e4467bf7590a1c4da27d72368a50f&name_0=PR&name_1=Build%20%28amd_debug%29 https://github.com/Altinity/ClickHouse/pull/2572 --- .../DataLakes/Iceberg/AlterDropPartitionExecutor.cpp | 4 +--- .../DataLakes/Iceberg/AlterDropPartitionExecutor.h | 4 +--- .../ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp index a7f7f83927bfc..faf3b30e2a176 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp @@ -219,8 +219,7 @@ AlterDropPartitionExecutor::AlterDropPartitionExecutor( const PersistentTableComponents & components_, const DataLakeStorageSettings & data_lake_settings_, String write_format_, - LoggerPtr log_, - ExternalStorageCache & external_storages_) + LoggerPtr log_) : command(command_) , metadata(metadata_) , context(context_) @@ -230,7 +229,6 @@ AlterDropPartitionExecutor::AlterDropPartitionExecutor( , data_lake_settings(data_lake_settings_) , write_format(std::move(write_format_)) , log(std::move(log_)) - , external_storages(external_storages_) { } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.h index ee9249ac4978c..a462b94962a57 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.h @@ -47,8 +47,7 @@ class AlterDropPartitionExecutor const PersistentTableComponents & components_, const DataLakeStorageSettings & data_lake_settings_, String write_format_, - LoggerPtr log_, - ExternalStorageCache & external_storages_); + LoggerPtr log_); void run(); @@ -114,7 +113,6 @@ class AlterDropPartitionExecutor const DataLakeStorageSettings & data_lake_settings; String write_format; LoggerPtr log; - ExternalStorageCache & external_storages; }; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 7bc5245a5355a..7bf01dfbd5791 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -977,8 +977,7 @@ void IcebergMetadata::alterPartitionDropImpl(const PartitionCommand & command, C persistent_components, data_lake_settings, write_format, - log, - *external_storages); + log); executor.run(); }