diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 164b0355dea5b8..2b52ea9ed64842 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -960,6 +960,11 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest& tablet_meta_info.time_series_compaction_level_threshold); need_to_save = true; } + if (tablet_meta_info.__isset.vertical_compaction_num_columns_per_group) { + tablet->tablet_meta()->set_vertical_compaction_num_columns_per_group( + tablet_meta_info.vertical_compaction_num_columns_per_group); + need_to_save = true; + } if (tablet_meta_info.__isset.replica_id) { tablet->tablet_meta()->set_replica_id(tablet_meta_info.replica_id); } diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 948e27522d6771..014c16d180e2d7 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -1600,6 +1600,14 @@ Status CloudTablet::sync_meta() { _tablet_meta->mutable_tablet_schema()->set_disable_auto_compaction( new_disable_auto_compaction); } + // Sync vertical_compaction_num_columns_per_group + auto new_vertical_compaction_num_columns_per_group = + tablet_meta->vertical_compaction_num_columns_per_group(); + if (_tablet_meta->vertical_compaction_num_columns_per_group() != + new_vertical_compaction_num_columns_per_group) { + _tablet_meta->set_vertical_compaction_num_columns_per_group( + new_vertical_compaction_num_columns_per_group); + } return Status::OK(); } diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index 1f9d087ee45d10..9869b876ab754f 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -635,6 +635,8 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in) out->set_time_series_compaction_empty_rowsets_threshold( in.time_series_compaction_empty_rowsets_threshold()); out->set_time_series_compaction_level_threshold(in.time_series_compaction_level_threshold()); + out->set_vertical_compaction_num_columns_per_group( + in.vertical_compaction_num_columns_per_group()); out->set_index_id(in.index_id()); out->set_is_in_memory(in.is_in_memory()); out->set_is_persistent(in.is_persistent()); @@ -712,6 +714,8 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) { out->set_time_series_compaction_empty_rowsets_threshold( in.time_series_compaction_empty_rowsets_threshold()); out->set_time_series_compaction_level_threshold(in.time_series_compaction_level_threshold()); + out->set_vertical_compaction_num_columns_per_group( + in.vertical_compaction_num_columns_per_group()); out->set_index_id(in.index_id()); out->set_is_in_memory(in.is_in_memory()); out->set_is_persistent(in.is_persistent()); @@ -796,6 +800,8 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in) out->set_time_series_compaction_empty_rowsets_threshold( in.time_series_compaction_empty_rowsets_threshold()); out->set_time_series_compaction_level_threshold(in.time_series_compaction_level_threshold()); + out->set_vertical_compaction_num_columns_per_group( + in.vertical_compaction_num_columns_per_group()); out->set_index_id(in.index_id()); out->set_is_in_memory(in.is_in_memory()); out->set_is_persistent(in.is_persistent()); @@ -873,6 +879,8 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) { out->set_time_series_compaction_empty_rowsets_threshold( in.time_series_compaction_empty_rowsets_threshold()); out->set_time_series_compaction_level_threshold(in.time_series_compaction_level_threshold()); + out->set_vertical_compaction_num_columns_per_group( + in.vertical_compaction_num_columns_per_group()); out->set_index_id(in.index_id()); out->set_is_in_memory(in.is_in_memory()); out->set_is_persistent(in.is_persistent()); diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index b9c15f01d0481f..70747d7d5d8ce7 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -167,7 +167,8 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, // unique_key should consider sequence&delete column void Merger::vertical_split_columns(const TabletSchema& tablet_schema, std::vector>* column_groups, - std::vector* key_group_cluster_key_idxes) { + std::vector* key_group_cluster_key_idxes, + int32_t num_columns_per_group) { size_t num_key_cols = tablet_schema.num_key_columns(); size_t total_cols = tablet_schema.num_columns(); std::vector key_columns; @@ -227,8 +228,7 @@ void Merger::vertical_split_columns(const TabletSchema& tablet_schema, continue; } - if (!value_columns.empty() && - value_columns.size() % config::vertical_compaction_num_columns_per_group == 0) { + if (!value_columns.empty() && value_columns.size() % num_columns_per_group == 0) { column_groups->push_back(value_columns); value_columns.clear(); } @@ -479,7 +479,33 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr tablet, ReaderType reader_t LOG(INFO) << "Start to do vertical compaction, tablet_id: " << tablet->tablet_id(); std::vector> column_groups; std::vector key_group_cluster_key_idxes; - vertical_split_columns(tablet_schema, &column_groups, &key_group_cluster_key_idxes); + // If BE config vertical_compaction_num_columns_per_group has been modified from + // its default value (5), use the BE config; otherwise use the tablet meta value. + constexpr int32_t default_num_columns_per_group = 5; + int32_t num_columns_per_group = + config::vertical_compaction_num_columns_per_group != default_num_columns_per_group + ? config::vertical_compaction_num_columns_per_group + : tablet->tablet_meta()->vertical_compaction_num_columns_per_group(); + + DBUG_EXECUTE_IF("Merger.vertical_merge_rowsets.check_num_columns_per_group", { + auto expected_value = DebugPoints::instance()->get_debug_param_or_default( + "Merger.vertical_merge_rowsets.check_num_columns_per_group", "expected_value", -1); + auto expected_tablet_id = DebugPoints::instance()->get_debug_param_or_default( + "Merger.vertical_merge_rowsets.check_num_columns_per_group", "tablet_id", -1); + if (expected_tablet_id != -1 && expected_tablet_id == tablet->tablet_id()) { + if (expected_value != -1 && expected_value != num_columns_per_group) { + LOG(FATAL) << "DEBUG_POINT CHECK FAILED: expected num_columns_per_group=" + << expected_value << " but got " << num_columns_per_group + << " for tablet_id=" << tablet->tablet_id(); + } else { + LOG(INFO) << "DEBUG_POINT CHECK PASSED: num_columns_per_group=" + << num_columns_per_group << ", tablet_id=" << tablet->tablet_id(); + } + } + }); + + vertical_split_columns(tablet_schema, &column_groups, &key_group_cluster_key_idxes, + num_columns_per_group); // Calculate total rows for density calculation after compaction int64_t total_rows = 0; diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h index 733c2de11f0e77..4c5306d816633a 100644 --- a/be/src/olap/merger.h +++ b/be/src/olap/merger.h @@ -73,7 +73,8 @@ class Merger { // for vertical compaction static void vertical_split_columns(const TabletSchema& tablet_schema, std::vector>* column_groups, - std::vector* key_group_cluster_key_idxes); + std::vector* key_group_cluster_key_idxes, + int32_t num_columns_per_group); static Status vertical_compact_one_group( BaseTabletSPtr tablet, ReaderType reader_type, const TabletSchema& tablet_schema, bool is_key, const std::vector& column_group, diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp index c5478499bc0b45..aa0ede3d093d11 100644 --- a/be/src/olap/rowset/segcompaction.cpp +++ b/be/src/olap/rowset/segcompaction.cpp @@ -35,6 +35,7 @@ #include "absl/strings/substitute.h" #include "beta_rowset_writer.h" #include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" #include "common/logging.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" @@ -281,8 +282,15 @@ Status SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt std::vector> column_groups; std::vector key_group_cluster_key_idxes; - Merger::vertical_split_columns(*ctx.tablet_schema, &column_groups, - &key_group_cluster_key_idxes); + // If BE config vertical_compaction_num_columns_per_group has been modified from + // its default value (5), use the BE config; otherwise use the tablet meta value. + constexpr int32_t default_num_columns_per_group = 5; + int32_t num_columns_per_group = + config::vertical_compaction_num_columns_per_group != default_num_columns_per_group + ? config::vertical_compaction_num_columns_per_group + : tablet->tablet_meta()->vertical_compaction_num_columns_per_group(); + Merger::vertical_split_columns(*ctx.tablet_schema, &column_groups, &key_group_cluster_key_idxes, + num_columns_per_group); vectorized::RowSourcesBuffer row_sources_buf(tablet->tablet_id(), tablet->tablet_path(), ReaderType::READER_SEGMENT_COMPACTION); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 6dce390fb4ab17..b94138e4782569 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -125,7 +125,10 @@ TabletMetaSharedPtr TabletMeta::create( request.time_series_compaction_time_threshold_seconds, request.time_series_compaction_empty_rowsets_threshold, request.time_series_compaction_level_threshold, inverted_index_file_storage_format, - request.tde_algorithm, storage_format); + request.tde_algorithm, storage_format, + request.__isset.vertical_compaction_num_columns_per_group + ? request.vertical_compaction_num_columns_per_group + : 5); } TabletMeta::~TabletMeta() { @@ -154,7 +157,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id int64_t time_series_compaction_level_threshold, TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format, TEncryptionAlgorithm::type tde_algorithm, - TStorageFormat::type storage_format) + TStorageFormat::type storage_format, + int32_t vertical_compaction_num_columns_per_group) : _tablet_uid(0, 0), _schema(new TabletSchema), _delete_bitmap(new DeleteBitmap(tablet_id)), @@ -187,6 +191,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id time_series_compaction_empty_rowsets_threshold); tablet_meta_pb.set_time_series_compaction_level_threshold( time_series_compaction_level_threshold); + tablet_meta_pb.set_vertical_compaction_num_columns_per_group( + vertical_compaction_num_columns_per_group); TabletSchemaPB* schema = tablet_meta_pb.mutable_schema(); schema->set_num_short_key_columns(tablet_schema.short_key_column_count); schema->set_num_rows_per_row_block(config::default_num_rows_per_column_file_block); @@ -454,7 +460,9 @@ TabletMeta::TabletMeta(const TabletMeta& b) b._time_series_compaction_time_threshold_seconds), _time_series_compaction_empty_rowsets_threshold( b._time_series_compaction_empty_rowsets_threshold), - _time_series_compaction_level_threshold(b._time_series_compaction_level_threshold) {}; + _time_series_compaction_level_threshold(b._time_series_compaction_level_threshold), + _vertical_compaction_num_columns_per_group( + b._vertical_compaction_num_columns_per_group) {}; void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, ColumnPB* column) { @@ -847,6 +855,8 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { tablet_meta_pb.time_series_compaction_empty_rowsets_threshold(); _time_series_compaction_level_threshold = tablet_meta_pb.time_series_compaction_level_threshold(); + _vertical_compaction_num_columns_per_group = + tablet_meta_pb.vertical_compaction_num_columns_per_group(); if (tablet_meta_pb.has_encryption_algorithm()) { _encryption_algorithm = tablet_meta_pb.encryption_algorithm(); @@ -942,6 +952,8 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_ time_series_compaction_empty_rowsets_threshold()); tablet_meta_pb->set_time_series_compaction_level_threshold( time_series_compaction_level_threshold()); + tablet_meta_pb->set_vertical_compaction_num_columns_per_group( + vertical_compaction_num_columns_per_group()); tablet_meta_pb->set_encryption_algorithm(_encryption_algorithm); } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 8d4d6b697198b9..9fcdbd66ae9151 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -118,7 +118,8 @@ class TabletMeta : public MetadataAdder { TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format = TInvertedIndexFileStorageFormat::V2, TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT, - TStorageFormat::type storage_format = TStorageFormat::V2); + TStorageFormat::type storage_format = TStorageFormat::V2, + int32_t vertical_compaction_num_columns_per_group = 5); // If need add a filed in TableMeta, filed init copy in copy construct function TabletMeta(const TabletMeta& tablet_meta); TabletMeta(TabletMeta&& tablet_meta) = delete; @@ -296,6 +297,13 @@ class TabletMeta : public MetadataAdder { return _time_series_compaction_level_threshold; } + void set_vertical_compaction_num_columns_per_group(int32_t num) { + _vertical_compaction_num_columns_per_group = num; + } + int32_t vertical_compaction_num_columns_per_group() const { + return _vertical_compaction_num_columns_per_group; + } + int64_t ttl_seconds() const { std::shared_lock rlock(_meta_lock); return _ttl_seconds; @@ -366,6 +374,7 @@ class TabletMeta : public MetadataAdder { int64_t _time_series_compaction_time_threshold_seconds = 0; int64_t _time_series_compaction_empty_rowsets_threshold = 0; int64_t _time_series_compaction_level_threshold = 0; + int32_t _vertical_compaction_num_columns_per_group = 5; int64_t _avg_rs_meta_serialize_size = 0; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 90ebd4076e1432..23927ba99b3c99 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1245,6 +1245,9 @@ void MetaServiceImpl::update_tablet(::google::protobuf::RpcController* controlle } } } + } else if (tablet_meta_info.has_vertical_compaction_num_columns_per_group()) { + tablet_meta.set_vertical_compaction_num_columns_per_group( + tablet_meta_info.vertical_compaction_num_columns_per_group()); } int64_t table_id = tablet_meta.table_id(); int64_t index_id = tablet_meta.index_id(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index ddbd2f0b69d449..37d6a5e1bca972 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -234,7 +234,8 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { tbl.rowStorePageSize(), tbl.variantEnableFlattenNested(), null, tbl.storagePageSize(), tbl.getTDEAlgorithmPB(), - tbl.storageDictPageSize(), true); + tbl.storageDictPageSize(), true, + tbl.getVerticalCompactionNumColumnsPerGroup()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets requestBuilder.setDbId(dbId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index d819d8ac7b5249..63e2575abb6847 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -257,7 +257,8 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.rowStorePageSize(), tbl.variantEnableFlattenNested(), clusterKeyUids, tbl.storagePageSize(), tbl.getTDEAlgorithmPB(), - tbl.storageDictPageSize(), true); + tbl.storageDictPageSize(), true, + tbl.getVerticalCompactionNumColumnsPerGroup()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets requestBuilder.setDbId(dbId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index aa46b08b4f6d52..a62e099b1bc0bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -272,7 +272,8 @@ protected void createRollupReplica() throws AlterCancelException { tbl.rowStorePageSize(), tbl.variantEnableFlattenNested(), tbl.storagePageSize(), tbl.getTDEAlgorithm(), - tbl.storageDictPageSize()); + tbl.storageDictPageSize(), + tbl.getVerticalCompactionNumColumnsPerGroup()); createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash); if (this.storageFormat != null) { createReplicaTask.setStorageFormat(this.storageFormat); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 94609e94ca0439..fadd689c04aa6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -649,6 +649,11 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol ColumnType.checkSupportSchemaChangeForComplexType(col.getType(), modColumn.getType(), true); lightSchemaChange = olapTable.getEnableLightSchemaChange(); } + // variant property-only change (e.g. variant_doc_materialization_min_rows) + if (columnPos == null && col.getDataType() == PrimitiveType.VARIANT + && modColumn.getDataType() == PrimitiveType.VARIANT) { + lightSchemaChange = olapTable.getEnableLightSchemaChange(); + } if (col.isClusterKey()) { throw new DdlException("Can not modify cluster key column: " + col.getName()); } @@ -2364,6 +2369,7 @@ public void updateTableProperties(Database db, String tableName, Map timeSeriesCompactionConfig, int enableSingleCompaction, int skipWriteIndexOnLoad, - int disableAutoCompaction) throws UserException { + int disableAutoCompaction, + int verticalCompactionNumColumnsPerGroup) throws UserException { // be id -> Map>> beIdToTabletIdWithHash = Maps.newHashMap(); OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); @@ -2603,7 +2617,7 @@ public void updatePartitionProperties(Database db, String tableName, String part UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(), isInMemory, storagePolicyId, binlogConfig, countDownLatch, compactionPolicy, timeSeriesCompactionConfig, enableSingleCompaction, skipWriteIndexOnLoad, - disableAutoCompaction); + disableAutoCompaction, verticalCompactionNumColumnsPerGroup); batchTask.addTask(task); } if (!FeConstants.runningUnitTest) { @@ -3486,7 +3500,7 @@ public boolean updateBinlogConfig(Database db, OlapTable olapTable, List notAllowedProps = properties.keySet().stream().filter(s -> !allowedProps.contains(s)) @@ -348,6 +349,18 @@ public void updateTableProperties(Database db, String tableName, Map clusterKeyUids, long storagePageSize, EncryptionAlgorithmPB encryptionAlgorithm, long storageDictPageSize, - boolean createInitialRowset) throws DdlException { + boolean createInitialRowset, + int verticalCompactionNumColumnsPerGroup) throws DdlException { OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder(); builder.setTableId(tableId); builder.setIndexId(indexId); @@ -258,6 +260,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, builder.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds); builder.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold); builder.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold); + builder.setVerticalCompactionNumColumnsPerGroup(verticalCompactionNumColumnsPerGroup); OlapFile.TabletSchemaCloudPB.Builder schemaBuilder = OlapFile.TabletSchemaCloudPB.newBuilder(); schemaBuilder.setSchemaVersion(schemaVersion); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 7cdb79b266bec8..93887a392d0665 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -157,6 +157,9 @@ public class PropertyAnalyzer { public static final String PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION = "enable_single_replica_compaction"; + public static final String PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP = + "vertical_compaction_num_columns_per_group"; + public static final String PROPERTIES_STORE_ROW_COLUMN = "store_row_column"; // deprecated public static final String PROPERTIES_ROW_STORE_COLUMNS = "row_store_columns"; @@ -251,6 +254,7 @@ public class PropertyAnalyzer { public static final long TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS_DEFAULT_VALUE = 3600; public static final long TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE = 5; public static final long TIME_SERIES_COMPACTION_LEVEL_THRESHOLD_DEFAULT_VALUE = 1; + public static final int VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE = 5; public static final String PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT = "variant_max_subcolumns_count"; @@ -2101,4 +2105,27 @@ public static TEncryptionAlgorithm analyzeTDEAlgorithm(Map prope } throw new AnalysisException("Invalid tde algorithm: " + name + ", only support AES256 and SM4 currently"); } + + public static Integer analyzeVerticalCompactionNumColumnsPerGroup(Map properties) + throws AnalysisException { + if (properties == null || properties.isEmpty()) { + return VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE; + } + String value = properties.get(PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP); + if (null == value) { + return VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE; + } + properties.remove(PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP); + try { + int num = Integer.parseInt(value); + if (num < 1 || num > 50) { + throw new AnalysisException(PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP + + " must be between 1 and 50"); + } + return num; + } catch (NumberFormatException e) { + throw new AnalysisException(PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP + + " must be a valid integer"); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 360c06f162f4c9..e8b06427e087d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2123,7 +2123,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa objectPool, tbl.rowStorePageSize(), tbl.variantEnableFlattenNested(), tbl.storagePageSize(), tbl.getTDEAlgorithm(), - tbl.storageDictPageSize()); + tbl.storageDictPageSize(), + tbl.getVerticalCompactionNumColumnsPerGroup()); task.setStorageFormat(tbl.getStorageFormat()); task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); @@ -2587,6 +2588,17 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th } olapTable.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold); + // set vertical compaction num columns per group + int verticalCompactionNumColumnsPerGroup + = PropertyAnalyzer.VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE; + try { + verticalCompactionNumColumnsPerGroup = PropertyAnalyzer + .analyzeVerticalCompactionNumColumnsPerGroup(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + olapTable.setVerticalCompactionNumColumnsPerGroup(verticalCompactionNumColumnsPerGroup); + boolean variantEnableFlattenNested = false; try { variantEnableFlattenNested = PropertyAnalyzer.analyzeVariantFlattenNested(properties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 74b651e10b07f5..1473d597338579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -1087,7 +1087,8 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta olapTable.rowStorePageSize(), olapTable.variantEnableFlattenNested(), olapTable.storagePageSize(), olapTable.getTDEAlgorithm(), - olapTable.storageDictPageSize()); + olapTable.storageDictPageSize(), + olapTable.getVerticalCompactionNumColumnsPerGroup()); createReplicaTask.setIsRecoverTask(true); createReplicaTask.setInvertedIndexFileStorageFormat(olapTable .getInvertedIndexFileStorageFormat()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java index 6376d24a46aa86..1e74a816543d43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java @@ -249,6 +249,23 @@ public void validate(ConnectContext ctx) throws UserException { } this.needTableStable = false; this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP)) { + int numColumnsPerGroup; + String numColumnsPerGroupStr = properties + .get(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP); + try { + numColumnsPerGroup = Integer.parseInt(numColumnsPerGroupStr); + if (numColumnsPerGroup < 1 || numColumnsPerGroup > 50) { + throw new AnalysisException( + "vertical_compaction_num_columns_per_group must be between 1 and 50: " + + numColumnsPerGroupStr); + } + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid vertical_compaction_num_columns_per_group format: " + + numColumnsPerGroupStr); + } + this.needTableStable = false; + this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) { if (properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")) { throw new AnalysisException( diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 5dfe410dbd7dc6..6ffae4c6660979 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -124,6 +124,8 @@ public class CreateReplicaTask extends AgentTask { private long timeSeriesCompactionLevelThreshold; + private int verticalCompactionNumColumnsPerGroup; + private boolean storeRowColumn; private BinlogConfig binlogConfig; @@ -163,7 +165,8 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition long rowStorePageSize, boolean variantEnableFlattenNested, long storagePageSize, TEncryptionAlgorithm tdeAlgorithm, - long storageDictPageSize) { + long storageDictPageSize, + int verticalCompactionNumColumnsPerGroup) { super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); this.replicaId = replicaId; @@ -206,6 +209,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition this.timeSeriesCompactionTimeThresholdSeconds = timeSeriesCompactionTimeThresholdSeconds; this.timeSeriesCompactionEmptyRowsetsThreshold = timeSeriesCompactionEmptyRowsetsThreshold; this.timeSeriesCompactionLevelThreshold = timeSeriesCompactionLevelThreshold; + this.verticalCompactionNumColumnsPerGroup = verticalCompactionNumColumnsPerGroup; this.storeRowColumn = storeRowColumn; this.binlogConfig = binlogConfig; this.objectPool = objectPool; @@ -423,6 +427,7 @@ public TCreateTabletReq toThrift() { createTabletReq.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds); createTabletReq.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold); createTabletReq.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold); + createTabletReq.setVerticalCompactionNumColumnsPerGroup(verticalCompactionNumColumnsPerGroup); createTabletReq.setTdeAlgorithm(tdeAlgorithm); if (binlogConfig != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java index 7d4c6a3d022cda..4ba127f447286e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java @@ -54,6 +54,7 @@ public class UpdateTabletMetaInfoTask extends AgentTask { private int enableSingleReplicaCompaction = -1; private int skipWriteIndexOnLoad = -1; private int disableAutoCompaction = -1; + private int verticalCompactionNumColumnsPerGroup = -1; public UpdateTabletMetaInfoTask(long backendId, Set> tableIdWithSchemaHash) { super(null, backendId, TTaskType.UPDATE_TABLET_META_INFO, @@ -89,13 +90,15 @@ public UpdateTabletMetaInfoTask(long backendId, Map timeSeriesCompactionConfig, int enableSingleReplicaCompaction, int skipWriteIndexOnLoad, - int disableAutoCompaction) { + int disableAutoCompaction, + int verticalCompactionNumColumnsPerGroup) { this(backendId, tableIdWithSchemaHash, inMemory, storagePolicyId, binlogConfig, latch); this.compactionPolicy = compactionPolicy; this.timeSeriesCompactionConfig = timeSeriesCompactionConfig; this.enableSingleReplicaCompaction = enableSingleReplicaCompaction; this.skipWriteIndexOnLoad = skipWriteIndexOnLoad; this.disableAutoCompaction = disableAutoCompaction; + this.verticalCompactionNumColumnsPerGroup = verticalCompactionNumColumnsPerGroup; } public void countDownLatch(long backendId, Set> tablets) { @@ -179,6 +182,9 @@ public TUpdateTabletMetaInfoReq toThrift() { if (disableAutoCompaction >= 0) { metaInfo.setDisableAutoCompaction(disableAutoCompaction > 0); } + if (verticalCompactionNumColumnsPerGroup >= 0) { + metaInfo.setVerticalCompactionNumColumnsPerGroup(verticalCompactionNumColumnsPerGroup); + } updateTabletMetaInfoReq.addToTabletMetaInfos(metaInfo); } } else { diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index 39d822b5449b9a..4b71c38bbd9f95 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -116,7 +116,7 @@ public void setUp() throws AnalysisException { indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false, - storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize); + storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, 5); // drop dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1, schemaHash1, false); diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index eff201605fcaad..c8d8a40487632b 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -649,6 +649,7 @@ message TabletMetaInfoPB { // For update tablet meta optional int64 time_series_compaction_level_threshold = 12; optional bool disable_auto_compaction = 13; optional bool enable_mow_light_delete = 14; + optional int32 vertical_compaction_num_columns_per_group = 15; } message TabletCompactionJobPB { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index d812c6d7a9c83b..ee4ab3b51df9a9 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -618,6 +618,7 @@ message TabletMetaPB { optional int64 time_series_compaction_empty_rowsets_threshold = 32 [default = 5]; optional int64 time_series_compaction_level_threshold = 33 [default = 1]; optional EncryptionAlgorithmPB encryption_algorithm = 34; + optional int32 vertical_compaction_num_columns_per_group = 35 [default = 5]; // For cloud optional int64 index_id = 1000; @@ -675,6 +676,7 @@ message TabletMetaCloudPB { optional int64 time_series_compaction_empty_rowsets_threshold = 37 [default = 5]; optional int64 time_series_compaction_level_threshold = 38 [default = 1]; optional EncryptionAlgorithmPB encryption_algorithm = 39; + optional int32 vertical_compaction_num_columns_per_group = 40 [default = 5]; // Use for selectdb-cloud optional string table_name = 101; diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 9a3a7c3743397a..3ca6879a18feec 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -227,6 +227,7 @@ struct TCreateTabletReq { 28: optional TInvertedIndexStorageFormat inverted_index_storage_format = TInvertedIndexStorageFormat.DEFAULT // Deprecated 29: optional Types.TInvertedIndexFileStorageFormat inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V2 30: optional TEncryptionAlgorithm tde_algorithm + 31: optional i32 vertical_compaction_num_columns_per_group = 5 // For cloud 1000: optional bool is_in_memory = false @@ -539,6 +540,7 @@ struct TTabletMetaInfo { 16: optional bool disable_auto_compaction 17: optional i64 time_series_compaction_empty_rowsets_threshold 18: optional i64 time_series_compaction_level_threshold + 19: optional i32 vertical_compaction_num_columns_per_group } struct TUpdateTabletMetaInfoReq { diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out index a6ff1c341f8436..0f8b1a2acafd57 100644 --- a/regression-test/data/query_p0/system/test_table_properties.out +++ b/regression-test/data/query_p0/system/test_table_properties.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_check_1 -- -108 +111 -- !select_check_2 -- internal test_table_properties_db duplicate_table _auto_bucket false @@ -38,6 +38,7 @@ internal test_table_properties_db duplicate_table time_series_compaction_goal_si internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1 internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600 internal test_table_properties_db duplicate_table variant_enable_flatten_nested false +internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5 internal test_table_properties_db listtable _auto_bucket false internal test_table_properties_db listtable binlog.enable false internal test_table_properties_db listtable binlog.max_bytes 9223372036854775807 @@ -73,6 +74,7 @@ internal test_table_properties_db listtable time_series_compaction_goal_size_mby internal test_table_properties_db listtable time_series_compaction_level_threshold 1 internal test_table_properties_db listtable time_series_compaction_time_threshold_seconds 3600 internal test_table_properties_db listtable variant_enable_flatten_nested false +internal test_table_properties_db listtable vertical_compaction_num_columns_per_group 5 internal test_table_properties_db unique_table _auto_bucket false internal test_table_properties_db unique_table binlog.enable false internal test_table_properties_db unique_table binlog.max_bytes 9223372036854775807 @@ -108,6 +110,7 @@ internal test_table_properties_db unique_table time_series_compaction_goal_size_ internal test_table_properties_db unique_table time_series_compaction_level_threshold 1 internal test_table_properties_db unique_table time_series_compaction_time_threshold_seconds 3600 internal test_table_properties_db unique_table variant_enable_flatten_nested false +internal test_table_properties_db unique_table vertical_compaction_num_columns_per_group 5 -- !select_check_3 -- internal test_table_properties_db duplicate_table _auto_bucket false @@ -145,6 +148,7 @@ internal test_table_properties_db duplicate_table time_series_compaction_goal_si internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1 internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600 internal test_table_properties_db duplicate_table variant_enable_flatten_nested false +internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5 internal test_table_properties_db unique_table _auto_bucket false internal test_table_properties_db unique_table binlog.enable false internal test_table_properties_db unique_table binlog.max_bytes 9223372036854775807 @@ -180,6 +184,7 @@ internal test_table_properties_db unique_table time_series_compaction_goal_size_ internal test_table_properties_db unique_table time_series_compaction_level_threshold 1 internal test_table_properties_db unique_table time_series_compaction_time_threshold_seconds 3600 internal test_table_properties_db unique_table variant_enable_flatten_nested false +internal test_table_properties_db unique_table vertical_compaction_num_columns_per_group 5 -- !select_check_4 -- @@ -219,6 +224,7 @@ internal test_table_properties_db duplicate_table time_series_compaction_goal_si internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1 internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600 internal test_table_properties_db duplicate_table variant_enable_flatten_nested false +internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5 -- !select_check_6 -- diff --git a/regression-test/data/variant_p0/test_variant_modify_doc_materialization_min_rows.out b/regression-test/data/variant_p0/test_variant_modify_doc_materialization_min_rows.out new file mode 100644 index 00000000000000..56a06bcf8fafb1 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_modify_doc_materialization_min_rows.out @@ -0,0 +1,22 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_step1 -- +k int Yes true \N +v variant Yes false \N NONE + +-- !desc_step2 -- +k int Yes true \N +v variant Yes false \N NONE +v.path_d bigint Yes false \N NONE +v.path_e text Yes false \N NONE +v.path_f double Yes false \N NONE + +-- !desc_step3 -- +k int Yes true \N +v variant Yes false \N NONE +v.path_a bigint Yes false \N NONE +v.path_b text Yes false \N NONE +v.path_c double Yes false \N NONE +v.path_d bigint Yes false \N NONE +v.path_e text Yes false \N NONE +v.path_f double Yes false \N NONE + diff --git a/regression-test/suites/compaction/test_vertical_compaction_num_columns_per_group.groovy b/regression-test/suites/compaction/test_vertical_compaction_num_columns_per_group.groovy new file mode 100644 index 00000000000000..129cd6d0309402 --- /dev/null +++ b/regression-test/suites/compaction/test_vertical_compaction_num_columns_per_group.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_vertical_compaction_num_columns_per_group", "nonConcurrent") { + def tableName = "test_columns_per_group" + + // Test 1: Create table with property set to 2 + sql """ DROP TABLE IF EXISTS ${tableName}_2 """ + sql """ CREATE TABLE ${tableName}_2 ( + k1 INT, + v1 INT, v2 INT, v3 INT, v4 INT, v5 INT, + v6 INT, v7 INT, v8 INT, v9 INT, v10 INT + ) DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "vertical_compaction_num_columns_per_group" = "2", + "disable_auto_compaction" = "true" + ); """ + + // Get tablet info + def tablets = sql_return_maparray """ SHOW TABLETS FROM ${tableName}_2; """ + def tablet_id = tablets[0].TabletId + logger.info("Test 1 - tablet_id: ${tablet_id}") + + // Verify SHOW CREATE TABLE contains property + def createTableResult = sql """ SHOW CREATE TABLE ${tableName}_2; """ + def createTableStr = createTableResult[0][1] + logger.info("Test 1 - SHOW CREATE TABLE: ${createTableStr}") + assertTrue(createTableStr.contains('"vertical_compaction_num_columns_per_group" = "2"'), + "SHOW CREATE TABLE should contain vertical_compaction_num_columns_per_group=2 after CREATE") + + // Enable debug point with tablet_id matching - if value doesn't match, BE will LOG(FATAL) + try { + GetDebugPoint().enableDebugPointForAllBEs( + "Merger.vertical_merge_rowsets.check_num_columns_per_group", + [expected_value: "2", tablet_id: "${tablet_id}"]) + + // Insert data to trigger compaction + for (int i = 0; i < 5; i++) { + sql """ INSERT INTO ${tableName}_2 VALUES + (${i}, ${i}, ${i}, ${i}, ${i}, ${i}, + ${i}, ${i}, ${i}, ${i}, ${i}); """ + } + + // Trigger and wait for compaction + trigger_and_wait_compaction("${tableName}_2", "full") + logger.info("Test 1 - Compaction finished, value 2 verified for tablet ${tablet_id}") + + } finally { + GetDebugPoint().disableDebugPointForAllBEs( + "Merger.vertical_merge_rowsets.check_num_columns_per_group") + } + + // Test 2: Create table without setting property (should use default value 5) + sql """ DROP TABLE IF EXISTS ${tableName}_default """ + sql """ CREATE TABLE ${tableName}_default ( + k1 INT, + v1 INT, v2 INT, v3 INT, v4 INT, v5 INT + ) DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); """ + + def tablets_default = sql_return_maparray """ SHOW TABLETS FROM ${tableName}_default; """ + def tablet_id_default = tablets_default[0].TabletId + logger.info("Test 2 - tablet_id_default: ${tablet_id_default}") + + try { + // Verify default value is 5 + GetDebugPoint().enableDebugPointForAllBEs( + "Merger.vertical_merge_rowsets.check_num_columns_per_group", + [expected_value: "5", tablet_id: "${tablet_id_default}"]) + + // Insert data + for (int i = 0; i < 5; i++) { + sql """ INSERT INTO ${tableName}_default VALUES (${i}, ${i}, ${i}, ${i}, ${i}, ${i}); """ + } + + // Trigger and wait for compaction + trigger_and_wait_compaction("${tableName}_default", "full") + logger.info("Test 2 - Compaction finished, default value 5 verified for tablet ${tablet_id_default}") + + } finally { + GetDebugPoint().disableDebugPointForAllBEs( + "Merger.vertical_merge_rowsets.check_num_columns_per_group") + } + + // Test 3: ALTER TABLE to modify property from 2 to 10 + sql """ ALTER TABLE ${tableName}_2 SET ("vertical_compaction_num_columns_per_group" = "10"); """ + logger.info("Test 3 - ALTER TABLE executed, changed value from 2 to 10") + sql """sync""" + + Thread.sleep(1000) + // Verify SHOW CREATE TABLE reflects the change + def createTableResult3 = sql """ SHOW CREATE TABLE ${tableName}_2; """ + def createTableStr3 = createTableResult3[0][1] + logger.info("Test 3 - SHOW CREATE TABLE after ALTER: ${createTableStr3}") + assertTrue(createTableStr3.contains('"vertical_compaction_num_columns_per_group" = "10"'), + "SHOW CREATE TABLE should contain vertical_compaction_num_columns_per_group=10 after ALTER") + + + // Wait for ALTER TABLE to take effect + // In cloud mode, BE syncs from MS which may take longer + if (isCloudMode()) { + return + } + try { + // Verify modified value is 10 + GetDebugPoint().enableDebugPointForAllBEs( + "Merger.vertical_merge_rowsets.check_num_columns_per_group", + [expected_value: "10", tablet_id: "${tablet_id}"]) + + // Insert more data + for (int i = 5; i < 10; i++) { + sql """ INSERT INTO ${tableName}_2 VALUES + (${i}, ${i}, ${i}, ${i}, ${i}, ${i}, + ${i}, ${i}, ${i}, ${i}, ${i}); """ + } + + // Trigger and wait for compaction + trigger_and_wait_compaction("${tableName}_2", "full") + logger.info("Test 3 - Compaction finished, altered value 10 verified for tablet ${tablet_id}") + + } finally { + GetDebugPoint().disableDebugPointForAllBEs( + "Merger.vertical_merge_rowsets.check_num_columns_per_group") + } + + // Verify data correctness + def result = sql """ SELECT COUNT(*) FROM ${tableName}_2; """ + assertEquals(10, result[0][0]) + + // Check tablet meta shows correct value + def tabletMeta = sql_return_maparray """ SHOW TABLETS FROM ${tableName}_2; """ + logger.info("Test 3 - tablet meta after ALTER: ${tabletMeta[0]}") + + logger.info("=== All tests passed ===") + logger.info("If any debug point check failed, the BE would have crashed with LOG(FATAL)") +} diff --git a/regression-test/suites/variant_p0/test_variant_modify_doc_materialization_min_rows.groovy b/regression-test/suites/variant_p0/test_variant_modify_doc_materialization_min_rows.groovy new file mode 100644 index 00000000000000..4854ec8b8e63b7 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_modify_doc_materialization_min_rows.groovy @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_modify_doc_materialization_min_rows", "p0") { + def tableName = "test_variant_modify_doc_min_rows" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + + // Step 1: Create table with variant_doc_materialization_min_rows=10 + sql """ + CREATE TABLE ${tableName} ( + k INT, + v variant + ) ENGINE=OLAP + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + + // Verify SHOW CREATE TABLE shows min_rows=5 + def createResult = sql """ SHOW CREATE TABLE ${tableName} """ + def createStmt = createResult[0][1] + assertTrue(createStmt.contains('"variant_doc_materialization_min_rows" = "10"'), + "SHOW CREATE TABLE should show variant_doc_materialization_min_rows=10, got: ${createStmt}") + + // Verify tablet meta shows min_rows=5 + // Insert 3 rows with different paths (3 < 5, so no sub-columns should be materialized) + sql """ INSERT INTO ${tableName} VALUES + (1, '{"path_a": 100, "path_b": "hello", "path_c": 1.5}'), + (2, '{"path_a": 200, "path_b": "world", "path_c": 2.5}'), + (3, '{"path_a": 300, "path_b": "doris", "path_c": 3.5}') + """ + + // Sync rowsets + sql "SELECT * FROM ${tableName} LIMIT 1" + + // Enable describe_extend_variant_column and check DESC + sql """set describe_extend_variant_column = true""" + // Expect: only k and v columns, NO sub-columns extracted (3 rows < min_rows=5) + qt_desc_step1 """desc ${tableName}""" + + // Step 2: ALTER TABLE MODIFY COLUMN to change variant_doc_materialization_min_rows to 2 + sql """ + ALTER TABLE ${tableName} MODIFY COLUMN v variant; + """ + + // Verify SHOW CREATE TABLE shows min_rows=2 + createResult = sql """ SHOW CREATE TABLE ${tableName} """ + createStmt = createResult[0][1] + assertTrue(createStmt.contains('"variant_doc_materialization_min_rows" = "2"'), + "SHOW CREATE TABLE should show variant_doc_materialization_min_rows=2 after ALTER, got: ${createStmt}") + + // Insert 3 more rows with different paths (3 >= 2, so sub-columns should be materialized for new data) + sql """ INSERT INTO ${tableName} VALUES + (4, '{"path_d": 400, "path_e": "alpha", "path_f": 4.5}'), + (5, '{"path_d": 500, "path_e": "beta", "path_f": 5.5}'), + (6, '{"path_d": 600, "path_e": "gamma", "path_f": 6.5}') + """ + + // Sync rowsets + sql "SELECT * FROM ${tableName} LIMIT 1" + + // Expect: 3 sub-columns from new data (path_d, path_e, path_f) + qt_desc_step2 """desc ${tableName}""" + + // Step 3: Trigger compaction - old data (path_a, path_b, path_c) should also get materialized + trigger_and_wait_compaction(tableName, "full") + + // Sync rowsets + sql "SELECT * FROM ${tableName} LIMIT 1" + + // Expect: 6 sub-columns total (path_a through path_f all materialized after compaction with min_rows=2) + qt_desc_step3 """desc ${tableName}""" + + // Verify data integrity + def count = sql """ SELECT COUNT(*) FROM ${tableName} """ + assertEquals(6, count[0][0]) + + // Cleanup + sql """ DROP TABLE IF EXISTS ${tableName} """ +}