diff --git a/README.md b/README.md index a3f29a23..7a96e924 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,7 @@ still might break. * String * LowCardinality(String) or LowCardinality(FixedString(N)) * Tuple +* Nested, read and written as Array(Tuple(...)) * UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 * UInt128, Int128 * UUID diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index 2fb0d39f..eb4c67ae 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -166,7 +166,8 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti return CreateTerminalColumn(ast); } - case TypeAst::Tuple: { + case TypeAst::Tuple: + case TypeAst::Nested: { std::vector columns; std::vector names; @@ -185,10 +186,12 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti } } - if (any_named) { - return std::make_shared(columns, std::move(names)); + auto tuple = any_named ? std::make_shared(columns, std::move(names)) + : std::make_shared(columns); + if (ast.meta == TypeAst::Nested) { + return std::make_shared(tuple); } - return std::make_shared(columns); + return tuple; } case TypeAst::Enum: { diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index d305a398..84b3df4b 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -146,6 +146,10 @@ static TypeAst::Meta GetTypeMeta(const StringView& name) { return TypeAst::Map; } + if (name == "Nested") { + return TypeAst::Nested; + } + return TypeAst::Terminal; } diff --git a/clickhouse/types/type_parser.h b/clickhouse/types/type_parser.h index cdef18ee..dc6db8cb 100644 --- a/clickhouse/types/type_parser.h +++ b/clickhouse/types/type_parser.h @@ -22,7 +22,8 @@ struct TypeAst { Enum, LowCardinality, SimpleAggregateFunction, - Map + Map, + Nested }; /// Type's category. diff --git a/docs/index.mdx b/docs/index.mdx index 4460ebc8..eaa2a0fa 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -164,6 +164,7 @@ client.Select( - `Nullable(T)` - `Array(T)` - `Tuple` +- `Nested`, read and written as `Array(Tuple(...))` - `Map` - `IPv4`, `IPv6` - `Point`, `Ring`, `Polygon`, `MultiPolygon` diff --git a/ut/CreateColumnByType_ut.cpp b/ut/CreateColumnByType_ut.cpp index 0f1fbc4f..4abfaf60 100644 --- a/ut/CreateColumnByType_ut.cpp +++ b/ut/CreateColumnByType_ut.cpp @@ -9,12 +9,33 @@ #include +#include +#include +#include #include +#if !defined(_WIN32) +#include +#endif + namespace { using namespace clickhouse; } +#if !defined(_WIN32) +namespace { +long MaxResidentKiB() { + struct rusage usage {}; + getrusage(RUSAGE_SELF, &usage); +#if defined(__APPLE__) + return usage.ru_maxrss / 1024; +#else + return usage.ru_maxrss; +#endif +} +} +#endif + TEST(CreateColumnByType, CreateSimpleAggregateFunction) { auto col = CreateColumnByType("SimpleAggregateFunction(funt, Int32)"); @@ -280,3 +301,47 @@ INSTANTIATE_TEST_SUITE_P(Nested, CreateColumnByTypeWithName, ::testing::Values( "Array(Nullable(LowCardinality(FixedString(10000))))", "Array(Enum8('ONE' = 1, 'TWO' = 2))" )); + +class CreateColumnByNestedType : public ::testing::TestWithParam> {}; + +TEST_P(CreateColumnByNestedType, CreateColumnByType) { + const auto & [type_name, expected_name] = GetParam(); + const auto col = CreateColumnByType(type_name); + ASSERT_NE(nullptr, col) << "CreateColumnByType returned nullptr for " << type_name; + EXPECT_EQ(expected_name, col->GetType().GetName()); +} + +INSTANTIATE_TEST_SUITE_P(Nested, CreateColumnByNestedType, ::testing::Values( + std::make_tuple("Nested(key String, value UInt8)", "Array(Tuple(key String, value UInt8))"), + std::make_tuple("Array(Nested(key String, value UInt8))", "Array(Array(Tuple(key String, value UInt8)))"), + std::make_tuple("Map(String, Nested(key String, value UInt8))", "Map(String, Array(Tuple(key String, value UInt8)))"), + std::make_tuple("Nested(a Nested(b String))", "Array(Tuple(a Array(Tuple(b String))))") +)); + +TEST(CreateColumnByTypeDeathTest, DeepNestedDoesNotCopyAst) { +#if defined(_WIN32) + GTEST_SKIP() << "peak RSS is read with getrusage"; +#else + const size_t depth = 200; + const size_t width = 8000; + std::string type_name; + for (size_t i = 0; i < depth; ++i) { + type_name += "Nested(n "; + } + type_name += "Nested(f0 UInt8"; + for (size_t i = 1; i < width; ++i) { + type_name += ", f" + std::to_string(i) + " UInt8"; + } + type_name += ")"; + type_name.append(depth, ')'); + + const auto create_within_peak_budget = [&type_name] { + const long before = MaxResidentKiB(); + const bool created = CreateColumnByType(type_name) != nullptr; + const long growth = MaxResidentKiB() - before; + std::cerr << "peak RSS grew by " << growth << " KiB"; + std::exit(created && growth < 16 * 1024 ? EXIT_SUCCESS : EXIT_FAILURE); + }; + EXPECT_EXIT(create_within_peak_budget(), ::testing::ExitedWithCode(EXIT_SUCCESS), ""); +#endif +} diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 79ee9ca8..aca0ac21 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1823,6 +1823,117 @@ const auto LocalHostEndpoint = ClientOptions() .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); +namespace { + +using KeyValuePairs = std::vector>; + +KeyValuePairs KeyValuePairsOf(const ColumnRef& column) { + auto tuples = column->AsStrict(); + auto keys = (*tuples)[0]->AsStrict(); + auto values = (*tuples)[1]->AsStrict(); + KeyValuePairs pairs; + for (size_t i = 0; i < tuples->Size(); ++i) { + pairs.emplace_back(std::string(keys->At(i)), values->At(i)); + } + return pairs; +} + +std::vector ArraysOfKeyValuePairsOf(const ColumnRef& column) { + auto arrays = column->AsStrict(); + std::vector arrays_of_pairs; + for (size_t i = 0; i < arrays->Size(); ++i) { + arrays_of_pairs.push_back(KeyValuePairsOf(arrays->GetAsColumn(i))); + } + return arrays_of_pairs; +} + +} + +TEST_P(ClientCase, ArrayOfNested_SelectServerTypedColumn) { + client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested"); + client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested " + "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))"); + client_->Execute("INSERT INTO test_clickhouse_cpp_array_of_nested VALUES " + "(1, [[('a', 1), ('b', 2)], []]), (2, []), (3, [[('c', 3)]])"); + + std::vector> rows; + std::string type_name; + client_->Select("SELECT id, arrays_of_pairs FROM test_clickhouse_cpp_array_of_nested ORDER BY id", [&](const Block& block) { + if (block.GetRowCount() == 0) { + return; + } + type_name = block[1]->GetType().GetName(); + auto arrays_of_pairs = block[1]->AsStrict(); + for (size_t row = 0; row < block.GetRowCount(); ++row) { + rows.push_back(ArraysOfKeyValuePairsOf(arrays_of_pairs->GetAsColumn(row))); + } + }); + + EXPECT_EQ("Array(Array(Tuple(key String, value UInt8)))", type_name); + const std::vector> expected = { + {{{"a", 1}, {"b", 2}}, {}}, + {}, + {{{"c", 3}}}, + }; + EXPECT_EQ(expected, rows); +} + +TEST_P(ClientCase, ArrayOfNested_ReinsertSelectedRows) { + client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested_source"); + client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested_copy"); + client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested_source " + "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))"); + client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested_copy " + "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))"); + client_->Execute("INSERT INTO test_clickhouse_cpp_array_of_nested_source VALUES (1, [[('a', 1), ('b', 2)], []]), (2, [])"); + + std::vector selected_blocks; + client_->Select("SELECT id, arrays_of_pairs FROM test_clickhouse_cpp_array_of_nested_source ORDER BY id", [&](const Block& block) { + if (block.GetRowCount() > 0) { + selected_blocks.push_back(block); + } + }); + for (const auto& block : selected_blocks) { + client_->Insert("test_clickhouse_cpp_array_of_nested_copy", block); + } + + std::vector rendered_rows; + client_->Select("SELECT toString(arrays_of_pairs) FROM test_clickhouse_cpp_array_of_nested_copy ORDER BY id", + [&](const Block& block) { + for (size_t row = 0; row < block.GetRowCount(); ++row) { + rendered_rows.emplace_back(block[0]->AsStrict()->At(row)); + } + }); + + const std::vector expected = {"[[('a',1),('b',2)],[]]", "[]"}; + EXPECT_EQ(expected, rendered_rows); +} + +TEST_P(ClientCase, Nested_UnflattenedColumn) { + client_->Execute("SET flatten_nested = 0"); + client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_nested_unflattened"); + client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_nested_unflattened " + "(id UInt8, pairs Nested(key String, value UInt8))"); + client_->Execute("INSERT INTO test_clickhouse_cpp_nested_unflattened VALUES (1, [('a', 1), ('b', 2)]), (2, [])"); + + std::vector rows; + std::string type_name; + client_->Select("SELECT id, pairs FROM test_clickhouse_cpp_nested_unflattened ORDER BY id", [&](const Block& block) { + if (block.GetRowCount() == 0) { + return; + } + type_name = block[1]->GetType().GetName(); + auto pairs = block[1]->AsStrict(); + for (size_t row = 0; row < block.GetRowCount(); ++row) { + rows.push_back(KeyValuePairsOf(pairs->GetAsColumn(row))); + } + }); + + EXPECT_EQ("Array(Tuple(key String, value UInt8))", type_name); + const std::vector expected = {{{"a", 1}, {"b", 2}}, {}}; + EXPECT_EQ(expected, rows); +} + INSTANTIATE_TEST_SUITE_P( Client, ClientCase, ::testing::Values( diff --git a/ut/type_parser_ut.cpp b/ut/type_parser_ut.cpp index 76bedd7a..d59986b8 100644 --- a/ut/type_parser_ut.cpp +++ b/ut/type_parser_ut.cpp @@ -552,3 +552,29 @@ TEST(TypeParser, AggregateFunction) { EXPECT_FALSE(TypeParser("AggregateFunction(argMax, LowCardinality(Nullable(FixedString(4))), DateTime(3, 'UTC'))").Parse(&ast)); } } + +TEST(TypeParserCase, ParseNested) { + TypeAst ast; + ASSERT_TRUE(TypeParser("Nested(key String, value UInt8)").Parse(&ast)); + ASSERT_EQ(ast.meta, TypeAst::Nested); + ASSERT_EQ(ast.name, "Nested"); + ASSERT_EQ(ast.elements.size(), 2u); + + ASSERT_EQ(ast.elements[0].element_name, "key"); + ASSERT_EQ(ast.elements[0].name, "String"); + ASSERT_EQ(ast.elements[0].code, Type::String); + + ASSERT_EQ(ast.elements[1].element_name, "value"); + ASSERT_EQ(ast.elements[1].name, "UInt8"); + ASSERT_EQ(ast.elements[1].code, Type::UInt8); +} + +TEST(TypeParserCase, ParseNested_FieldNamedNested) { + TypeAst ast; + ASSERT_TRUE(TypeParser("Nested(Nested String)").Parse(&ast)); + ASSERT_EQ(ast.meta, TypeAst::Nested); + ASSERT_EQ(ast.elements.size(), 1u); + ASSERT_EQ(ast.elements[0].meta, TypeAst::Terminal); + ASSERT_EQ(ast.elements[0].element_name, "Nested"); + ASSERT_EQ(ast.elements[0].name, "String"); +}