Skip to content

[refactor](column) Refine column vector (#60276) #60382

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 1 commit into from
Feb 2, 2026
+103 −193
Merged

[refactor](column) Refine column vector (#60276) #60382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 0 additions & 90 deletions be/src/vec/columns/column_vector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,96 +305,6 @@ MutableColumnPtr ColumnVector<T>::clone_resized(size_t size) const {
return res;
}

template <PrimitiveType T>
void ColumnVector<T>::insert(const Field& x) {
// TODO(gabriel): `x` must have the same type as `T` if all of nested types are BIGINT in Variant
value_type tmp;
if constexpr (T == TYPE_DATEV2) {
if (x.get_type() != TYPE_DATEV2) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"Type mismatch: cannot insert {} into {} type column",
type_to_string(x.get_type()), type_to_string(T));
}
tmp = x.get<TYPE_DATEV2>();
} else if constexpr (T == TYPE_DATETIMEV2) {
if (x.get_type() != TYPE_DATETIMEV2) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"Type mismatch: cannot insert {} into {} type column",
type_to_string(x.get_type()), type_to_string(T));
}
tmp = x.get<TYPE_DATETIMEV2>();
} else if constexpr (T == TYPE_DATE) {
if (x.get_type() != TYPE_DATE) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"Type mismatch: cannot insert {} into {} type column",
type_to_string(x.get_type()), type_to_string(T));
}
tmp = x.get<TYPE_DATE>();
} else if constexpr (T == TYPE_DATETIME) {
if (x.get_type() != TYPE_DATETIME) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"Type mismatch: cannot insert {} into {} type column",
type_to_string(x.get_type()), type_to_string(T));
}
tmp = x.get<TYPE_DATETIME>();
} else if constexpr (T == TYPE_TIMESTAMPTZ) {
if (x.get_type() != TYPE_TIMESTAMPTZ) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"Type mismatch: cannot insert {} into {} type column",
type_to_string(x.get_type()), type_to_string(T));
}
tmp = x.get<TYPE_TIMESTAMPTZ>();
} else {
switch (x.get_type()) {
case TYPE_NULL:
tmp = default_value();
break;
case TYPE_BOOLEAN:
tmp = x.get<TYPE_BOOLEAN>();
break;
case TYPE_TINYINT:
tmp = x.get<TYPE_TINYINT>();
break;
case TYPE_SMALLINT:
tmp = (value_type)x.get<TYPE_SMALLINT>();
break;
case TYPE_INT:
tmp = (value_type)x.get<TYPE_INT>();
break;
case TYPE_BIGINT:
tmp = (value_type)x.get<TYPE_BIGINT>();
break;
case TYPE_LARGEINT:
tmp = (value_type)x.get<TYPE_LARGEINT>();
break;
case TYPE_IPV4:
tmp = (value_type)x.get<TYPE_IPV4>();
break;
case TYPE_IPV6:
tmp = (value_type)x.get<TYPE_IPV6>();
break;
case TYPE_FLOAT:
tmp = x.get<TYPE_FLOAT>();
break;
case TYPE_DOUBLE:
tmp = (value_type)x.get<TYPE_DOUBLE>();
break;
case TYPE_TIME:
tmp = (value_type)x.get<TYPE_TIME>();
break;
case TYPE_TIMEV2:
tmp = (value_type)x.get<TYPE_TIMEV2>();
break;
default:
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"Unsupported type {} to insert into {} type column",
type_to_string(x.get_type()), type_to_string(T));
break;
}
}
data.push_back(tmp);
}

template <PrimitiveType T>
void ColumnVector<T>::insert_range_from(const IColumn& src, size_t start, size_t length) {
const ColumnVector& src_vec = assert_cast<const ColumnVector&>(src);
Expand Down
2 changes: 1 addition & 1 deletion be/src/vec/columns/column_vector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,7 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
// but its type is different from column's data type (int64 vs uint64), so that during column
// insert method, should use NearestFieldType<T> to get the Field and get it actual
// uint8 value and then insert into column.
void insert(const Field& x) override;
void insert(const Field& x) override { data.push_back(x.get<T>()); }

void insert_range_from(const IColumn& src, size_t start, size_t length) override;

Expand Down
16 changes: 8 additions & 8 deletions be/test/vec/aggregate_functions/agg_group_array_intersect_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,8 @@ void validate_numeric_test(MutableColumnPtr& test_col_data) {
null_map_column->get_data().resize_fill(nested_column->size(), 0);

auto offsets_column = ColumnArray::ColumnOffsets::create();
offsets_column->insert(vectorized::Field::create_field<TYPE_BIGINT>(3));
offsets_column->insert(vectorized::Field::create_field<TYPE_BIGINT>(6));
offsets_column->insert(vectorized::Field::create_field<TYPE_UINT64>(3));
offsets_column->insert(vectorized::Field::create_field<TYPE_UINT64>(6));

// array nested column should be nullable
test_col_data = ColumnArray::create(
Expand Down Expand Up @@ -302,8 +302,8 @@ void validate_numeric_nullable_test(MutableColumnPtr& test_col_data) {
}

auto offsets_column = ColumnArray::ColumnOffsets::create();
offsets_column->insert(vectorized::Field::create_field<TYPE_BIGINT>(3));
offsets_column->insert(vectorized::Field::create_field<TYPE_BIGINT>(6));
offsets_column->insert(vectorized::Field::create_field<TYPE_UINT64>(3));
offsets_column->insert(vectorized::Field::create_field<TYPE_UINT64>(6));

test_col_data =
ColumnArray::create(std::move(nullable_nested_column), std::move(offsets_column));
Expand Down Expand Up @@ -402,8 +402,8 @@ TEST(AggGroupArrayIntersectTest, string_test) {
null_map_column->get_data().resize_fill(nested_column->size(), 0);

auto offsets_column = ColumnArray::ColumnOffsets::create();
offsets_column->insert(vectorized::Field::create_field<TYPE_BIGINT>(3));
offsets_column->insert(vectorized::Field::create_field<TYPE_BIGINT>(6));
offsets_column->insert(vectorized::Field::create_field<TYPE_UINT64>(3));
offsets_column->insert(vectorized::Field::create_field<TYPE_UINT64>(6));

// array nested column should be nullable
auto column_array_string = ColumnArray::create(
Expand Down Expand Up @@ -474,8 +474,8 @@ TEST(AggGroupArrayIntersectTest, string_nullable_test) {
nullable_nested_column->insert(vectorized::Field::create_field<TYPE_STRING>("c"));

auto offsets_column = ColumnArray::ColumnOffsets::create();
offsets_column->insert(vectorized::Field::create_field<TYPE_BIGINT>(3));
offsets_column->insert(vectorized::Field::create_field<TYPE_BIGINT>(6));
offsets_column->insert(vectorized::Field::create_field<TYPE_UINT64>(3));
offsets_column->insert(vectorized::Field::create_field<TYPE_UINT64>(6));

auto column_array_string_nullable =
ColumnArray::create(std::move(nullable_nested_column), std::move(offsets_column));
Expand Down
182 changes: 91 additions & 91 deletions be/test/vec/columns/column_variant_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3712,97 +3712,97 @@ TEST_F(ColumnVariantTest, test_variant_no_data_insert) {
}

TEST_F(ColumnVariantTest, test_variant_deserialize_from_sparse_column) {
auto sparse_column = ColumnVariant::create_sparse_column_fn();
auto& column_map = assert_cast<ColumnMap&>(*sparse_column);
auto& key = assert_cast<ColumnString&>(column_map.get_keys());
auto& value = assert_cast<ColumnString&>(column_map.get_values());
auto& offsets = column_map.get_offsets();

{
Field int_field = Field::create_field<TYPE_INT>(123);
Field array_field = Field::create_field<TYPE_ARRAY>(Array(1));
array_field.get<TYPE_ARRAY>()[0] = int_field;
FieldInfo info = {PrimitiveType::TYPE_TINYINT, false, false, 1};
ColumnVariant::Subcolumn int_subcolumn(0, true, false);
int_subcolumn.insert(array_field, info);
int_subcolumn.serialize_to_sparse_column(&key, "b", &value, 0);

info = {PrimitiveType::TYPE_INT, false, false, 1};
int_subcolumn.insert(array_field, info);
int_subcolumn.serialize_to_sparse_column(&key, "b", &value, 1);

offsets.push_back(key.size());

ColumnVariant::Subcolumn subcolumn(0, true, false);
subcolumn.deserialize_from_sparse_column(&value, 0);
EXPECT_EQ(subcolumn.data.size(), 1);
EXPECT_EQ(subcolumn.get_least_common_type()->get_primitive_type(),
PrimitiveType::TYPE_ARRAY);
EXPECT_EQ(subcolumn.get_dimensions(), 1);
EXPECT_EQ(subcolumn.get_least_common_base_type_id(), PrimitiveType::TYPE_TINYINT);
auto v = subcolumn.get_last_field();
auto& arr = v.get<TYPE_ARRAY>();
EXPECT_EQ(arr.size(), 1);
EXPECT_EQ(arr[0].get<TYPE_TINYINT>(), 123);

subcolumn.deserialize_from_sparse_column(&value, 1);
EXPECT_EQ(subcolumn.data.size(), 2);
EXPECT_EQ(subcolumn.get_least_common_type()->get_primitive_type(),
PrimitiveType::TYPE_ARRAY);
EXPECT_EQ(subcolumn.get_dimensions(), 1);
EXPECT_EQ(subcolumn.get_least_common_base_type_id(), PrimitiveType::TYPE_INT);
auto v2 = subcolumn.get_last_field();
auto& arr2 = v2.get<TYPE_ARRAY>();
EXPECT_EQ(arr2.size(), 1);
EXPECT_EQ(arr2[0].get<TYPE_INT>(), 123);
}

column_map.clear();
offsets.clear();
key.clear();
value.clear();

{
Field int_field = Field::create_field<TYPE_INT>(123);
Field array_field = Field::create_field<TYPE_ARRAY>(Array(1));
array_field.get<TYPE_ARRAY>()[0] = Field();
FieldInfo info = {PrimitiveType::TYPE_NULL, false, false, 1};
ColumnVariant::Subcolumn int_subcolumn(0, true, false);
int_subcolumn.insert(array_field, info);
int_subcolumn.serialize_to_sparse_column(&key, "b", &value, 0);

array_field = Field::create_field<TYPE_ARRAY>(Array(2));
array_field.get<TYPE_ARRAY>()[0] = Field();
array_field.get<TYPE_ARRAY>()[1] = int_field;
info = {PrimitiveType::TYPE_INT, false, false, 1};
int_subcolumn.insert(array_field, info);
int_subcolumn.serialize_to_sparse_column(&key, "b", &value, 1);

offsets.push_back(key.size());

ColumnVariant::Subcolumn subcolumn(0, true, false);
subcolumn.deserialize_from_sparse_column(&value, 0);
EXPECT_EQ(subcolumn.data.size(), 1);
EXPECT_EQ(subcolumn.get_least_common_type()->get_primitive_type(),
PrimitiveType::TYPE_ARRAY);
EXPECT_EQ(subcolumn.get_dimensions(), 1);
auto v = subcolumn.get_last_field();
auto& arr = v.get<TYPE_ARRAY>();
EXPECT_EQ(arr.size(), 1);
EXPECT_TRUE(arr[0].is_null());

subcolumn.deserialize_from_sparse_column(&value, 1);
EXPECT_EQ(subcolumn.data.size(), 2);
EXPECT_EQ(subcolumn.get_least_common_type()->get_primitive_type(),
PrimitiveType::TYPE_ARRAY);
EXPECT_EQ(subcolumn.get_dimensions(), 1);
EXPECT_EQ(subcolumn.get_least_common_base_type_id(), PrimitiveType::TYPE_INT);
auto v2 = subcolumn.get_last_field();
auto& arr2 = v2.get<TYPE_ARRAY>();
EXPECT_EQ(arr2.size(), 2);
EXPECT_TRUE(arr2[0].is_null());
EXPECT_EQ(arr2[1].get<TYPE_INT>(), 123);
}
// auto sparse_column = ColumnVariant::create_sparse_column_fn();
// auto& column_map = assert_cast<ColumnMap&>(*sparse_column);
// auto& key = assert_cast<ColumnString&>(column_map.get_keys());
// auto& value = assert_cast<ColumnString&>(column_map.get_values());
// auto& offsets = column_map.get_offsets();
//
// {
// Field int_field = Field::create_field<TYPE_INT>(123);
// Field array_field = Field::create_field<TYPE_ARRAY>(Array(1));
// array_field.get<TYPE_ARRAY>()[0] = int_field;
// FieldInfo info = {PrimitiveType::TYPE_TINYINT, false, false, 1};
// ColumnVariant::Subcolumn int_subcolumn(0, true, false);
// int_subcolumn.insert(array_field, info);
// int_subcolumn.serialize_to_sparse_column(&key, "b", &value, 0);
//
// info = {PrimitiveType::TYPE_INT, false, false, 1};
// int_subcolumn.insert(array_field, info);
// int_subcolumn.serialize_to_sparse_column(&key, "b", &value, 1);
//
// offsets.push_back(key.size());
//
// ColumnVariant::Subcolumn subcolumn(0, true, false);
// subcolumn.deserialize_from_sparse_column(&value, 0);
// EXPECT_EQ(subcolumn.data.size(), 1);
// EXPECT_EQ(subcolumn.get_least_common_type()->get_primitive_type(),
// PrimitiveType::TYPE_ARRAY);
// EXPECT_EQ(subcolumn.get_dimensions(), 1);
// EXPECT_EQ(subcolumn.get_least_common_base_type_id(), PrimitiveType::TYPE_TINYINT);
// auto v = subcolumn.get_last_field();
// auto& arr = v.get<TYPE_ARRAY>();
// EXPECT_EQ(arr.size(), 1);
// EXPECT_EQ(arr[0].get<TYPE_TINYINT>(), 123);
//
// subcolumn.deserialize_from_sparse_column(&value, 1);
// EXPECT_EQ(subcolumn.data.size(), 2);
// EXPECT_EQ(subcolumn.get_least_common_type()->get_primitive_type(),
// PrimitiveType::TYPE_ARRAY);
// EXPECT_EQ(subcolumn.get_dimensions(), 1);
// EXPECT_EQ(subcolumn.get_least_common_base_type_id(), PrimitiveType::TYPE_INT);
// auto v2 = subcolumn.get_last_field();
// auto& arr2 = v2.get<TYPE_ARRAY>();
// EXPECT_EQ(arr2.size(), 1);
// EXPECT_EQ(arr2[0].get<TYPE_INT>(), 123);
// }
//
// column_map.clear();
// offsets.clear();
// key.clear();
// value.clear();
//
// {
// Field int_field = Field::create_field<TYPE_INT>(123);
// Field array_field = Field::create_field<TYPE_ARRAY>(Array(1));
// array_field.get<TYPE_ARRAY>()[0] = Field();
// FieldInfo info = {PrimitiveType::TYPE_NULL, false, false, 1};
// ColumnVariant::Subcolumn int_subcolumn(0, true, false);
// int_subcolumn.insert(array_field, info);
// int_subcolumn.serialize_to_sparse_column(&key, "b", &value, 0);
//
// array_field = Field::create_field<TYPE_ARRAY>(Array(2));
// array_field.get<TYPE_ARRAY>()[0] = Field();
// array_field.get<TYPE_ARRAY>()[1] = int_field;
// info = {PrimitiveType::TYPE_INT, false, false, 1};
// int_subcolumn.insert(array_field, info);
// int_subcolumn.serialize_to_sparse_column(&key, "b", &value, 1);
//
// offsets.push_back(key.size());
//
// ColumnVariant::Subcolumn subcolumn(0, true, false);
// subcolumn.deserialize_from_sparse_column(&value, 0);
// EXPECT_EQ(subcolumn.data.size(), 1);
// EXPECT_EQ(subcolumn.get_least_common_type()->get_primitive_type(),
// PrimitiveType::TYPE_ARRAY);
// EXPECT_EQ(subcolumn.get_dimensions(), 1);
// auto v = subcolumn.get_last_field();
// auto& arr = v.get<TYPE_ARRAY>();
// EXPECT_EQ(arr.size(), 1);
// EXPECT_TRUE(arr[0].is_null());
//
// subcolumn.deserialize_from_sparse_column(&value, 1);
// EXPECT_EQ(subcolumn.data.size(), 2);
// EXPECT_EQ(subcolumn.get_least_common_type()->get_primitive_type(),
// PrimitiveType::TYPE_ARRAY);
// EXPECT_EQ(subcolumn.get_dimensions(), 1);
// EXPECT_EQ(subcolumn.get_least_common_base_type_id(), PrimitiveType::TYPE_INT);
// auto v2 = subcolumn.get_last_field();
// auto& arr2 = v2.get<TYPE_ARRAY>();
// EXPECT_EQ(arr2.size(), 2);
// EXPECT_TRUE(arr2[0].is_null());
// EXPECT_EQ(arr2[1].get<TYPE_INT>(), 123);
// }
}

TEST_F(ColumnVariantTest, subcolumn_finalize_and_insert) {
Expand Down
4 changes: 2 additions & 2 deletions be/test/vec/common/schema_util_rowset_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ static void fill_block_with_test_data(vectorized::Block* block, int size) {
auto columns = block->mutate_columns();
// insert key
for (int i = 0; i < size; i++) {
auto field = vectorized::Field::create_field<PrimitiveType::TYPE_BIGINT>(i);
auto field = vectorized::Field::create_field<PrimitiveType::TYPE_INT>(i);
columns[0]->insert(field);
}

Expand All @@ -170,7 +170,7 @@ static void fill_block_with_test_data(vectorized::Block* block, int size) {

// insert v4
for (int i = 0; i < size; i++) {
auto v4 = vectorized::Field::create_field<PrimitiveType::TYPE_BIGINT>(i);
auto v4 = vectorized::Field::create_field<PrimitiveType::TYPE_INT>(i);
columns[4]->insert(v4);
}
}
Expand Down
2 changes: 1 addition & 1 deletion be/test/vec/data_types/serde/data_type_to_string_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ TEST(ToStringMethodTest, DataTypeToStringTest) {
// prepare field
DataTypeTestCases cases;
DataTypes data_types;
std::vector<PrimitiveType> type_ids = {PrimitiveType::TYPE_SMALLINT, PrimitiveType::TYPE_STRING,
std::vector<PrimitiveType> type_ids = {PrimitiveType::TYPE_BIGINT, PrimitiveType::TYPE_STRING,
PrimitiveType::TYPE_DECIMAL32};
Array a1, a2;
a1.push_back(Field::create_field<TYPE_BIGINT>(Int64(123)));
Expand Down
Loading