diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index e0799ee1256f0f..bcad8506d1ace8 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -102,6 +102,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in) if (in.has___split_schema()) { out->mutable___split_schema()->CopyFrom(in.__split_schema()); } + if (in.has_inverted_index_storage_format()) { + out->set_inverted_index_storage_format(in.inverted_index_storage_format()); + } if (in.has_visible_ts_ms()) { out->set_visible_ts_ms(in.visible_ts_ms()); } @@ -198,6 +201,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) { if (in.has___split_schema()) { out->mutable___split_schema()->Swap(in.mutable___split_schema()); } + if (in.has_inverted_index_storage_format()) { + out->set_inverted_index_storage_format(in.inverted_index_storage_format()); + } if (in.has_visible_ts_ms()) { out->set_visible_ts_ms(in.visible_ts_ms()); } @@ -310,6 +316,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in) if (in.has_reference_instance_id()) { out->set_reference_instance_id(in.reference_instance_id()); } + if (in.has_inverted_index_storage_format()) { + out->set_inverted_index_storage_format(in.inverted_index_storage_format()); + } auto* slice_locations = out->mutable_packed_slice_locations(); slice_locations->clear(); slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end()); @@ -405,6 +414,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) { if (in.has_reference_instance_id()) { out->set_reference_instance_id(in.reference_instance_id()); } + if (in.has_inverted_index_storage_format()) { + out->set_inverted_index_storage_format(in.inverted_index_storage_format()); + } auto* slice_locations = out->mutable_packed_slice_locations(); slice_locations->clear(); slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end()); @@ -790,6 +802,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in) if (in.has_encryption_algorithm()) { out->set_encryption_algorithm(in.encryption_algorithm()); } + if (in.has_inverted_index_storage_format()) { + out->set_inverted_index_storage_format(in.inverted_index_storage_format()); + } } void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) { @@ -875,6 +890,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) { if (in.has_encryption_algorithm()) { out->set_encryption_algorithm(in.encryption_algorithm()); } + if (in.has_inverted_index_storage_format()) { + out->set_inverted_index_storage_format(in.inverted_index_storage_format()); + } } TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) { @@ -967,6 +985,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in) if (in.has_encryption_algorithm()) { out->set_encryption_algorithm(in.encryption_algorithm()); } + if (in.has_inverted_index_storage_format()) { + out->set_inverted_index_storage_format(in.inverted_index_storage_format()); + } } void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) { @@ -1052,6 +1073,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) { if (in.has_encryption_algorithm()) { out->set_encryption_algorithm(in.encryption_algorithm()); } + if (in.has_inverted_index_storage_format()) { + out->set_inverted_index_storage_format(in.inverted_index_storage_format()); + } } } // namespace doris::cloud diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index f8dd3fbd2a2db8..f4470a70c28d39 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -772,10 +772,13 @@ int InstanceChecker::do_check() { } if (!index_ids.empty()) { const auto& index_map = rs_meta.packed_slice_locations(); + const auto index_format = + rs_meta.has_inverted_index_storage_format() + ? rs_meta.inverted_index_storage_format() + : rs_meta.tablet_schema().inverted_index_storage_format(); for (int i = 0; i < rs_meta.num_segments(); ++i) { std::vector index_path_v; - if (rs_meta.tablet_schema().inverted_index_storage_format() == - InvertedIndexStorageFormatPB::V1) { + if (index_format == InvertedIndexStorageFormatPB::V1) { for (const auto& index_id : index_ids) { LOG(INFO) << "check inverted index, tablet_id=" << rs_meta.tablet_id() << " rowset_id=" << rs_meta.rowset_id_v2() << " segment_id=" << i diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index 011062b115923f..0708cffb8ea481 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -3470,8 +3470,9 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) { // Process inverted indexes std::vector> index_ids; - // default format as v1. - InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V1; + InvertedIndexStorageFormatPB index_format = rs_meta_pb.has_inverted_index_storage_format() + ? rs_meta_pb.inverted_index_storage_format() + : InvertedIndexStorageFormatPB::V1; bool delete_rowset_data_by_prefix = false; if (rs_meta_pb.rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE) { // if rowset state is RowsetStatePB::BEGIN_PARTIAL_UPDATE, the number of segments data @@ -3483,7 +3484,8 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) { index_ids.emplace_back(index.index_id(), index.index_suffix_name()); } } - if (rs_meta_pb.tablet_schema().has_inverted_index_storage_format()) { + if (!rs_meta_pb.has_inverted_index_storage_format() && + rs_meta_pb.tablet_schema().has_inverted_index_storage_format()) { index_format = rs_meta_pb.tablet_schema().inverted_index_storage_format(); } } else if (!rs_meta_pb.has_index_id() || !rs_meta_pb.has_schema_version()) { @@ -3497,7 +3499,9 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) { TEST_SYNC_POINT_CALLBACK("InstanceRecycler::delete_rowset_data.tmp_rowset", &inverted_index_get_ret); if (inverted_index_get_ret == 0) { - index_format = index_info.first; + if (!rs_meta_pb.has_inverted_index_storage_format()) { + index_format = index_info.first; + } index_ids = index_info.second; } else if (inverted_index_get_ret == 1) { // 1. Schema kv not found means tablet has been recycled @@ -4235,7 +4239,9 @@ int InstanceRecycler::delete_rowset_data( // Process inverted indexes std::vector> index_ids; // default format as v1. - InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V1; + InvertedIndexStorageFormatPB index_format = rs.has_inverted_index_storage_format() + ? rs.inverted_index_storage_format() + : InvertedIndexStorageFormatPB::V1; int inverted_index_get_ret = 0; if (rs.has_tablet_schema()) { for (const auto& index : rs.tablet_schema().index()) { @@ -4243,7 +4249,8 @@ int InstanceRecycler::delete_rowset_data( index_ids.emplace_back(index.index_id(), index.index_suffix_name()); } } - if (rs.tablet_schema().has_inverted_index_storage_format()) { + if (!rs.has_inverted_index_storage_format() && + rs.tablet_schema().has_inverted_index_storage_format()) { index_format = rs.tablet_schema().inverted_index_storage_format(); } } else { @@ -4261,7 +4268,9 @@ int InstanceRecycler::delete_rowset_data( TEST_SYNC_POINT_CALLBACK("InstanceRecycler::delete_rowset_data.tmp_rowset", &inverted_index_get_ret); if (inverted_index_get_ret == 0) { - index_format = index_info.first; + if (!rs.has_inverted_index_storage_format()) { + index_format = index_info.first; + } index_ids = index_info.second; } else if (inverted_index_get_ret == 1) { // 1. Schema kv not found means tablet has been recycled diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index c1abd0a5e13a5f..8c635e8d6e95f1 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -8066,6 +8066,71 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) { } } +TEST(RecyclerTest, delete_tmp_rowset_data_with_rowset_idx_v3) { + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("delete_tmp_rowset_data_with_rowset_idx_v3"); + obj_info->set_ak(config::test_s3_ak); + obj_info->set_sk(config::test_s3_sk); + obj_info->set_endpoint(config::test_s3_endpoint); + obj_info->set_region(config::test_s3_region); + obj_info->set_bucket(config::test_s3_bucket); + obj_info->set_prefix("delete_tmp_rowset_data_with_rowset_idx_v3"); + + doris::TabletSchemaCloudPB schema; + schema.set_schema_version(1); + schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + auto index = schema.add_index(); + index->set_index_id(1); + index->set_index_type(IndexType::INVERTED); + + { + InstanceRecycler recycler(txn_kv, instance, thread_group, + std::make_shared(txn_kv)); + ASSERT_EQ(recycler.init(), 0); + auto accessor = recycler.accessor_map_.begin()->second; + std::map rowset_pbs; + doris::RowsetMetaCloudPB rowset; + rowset.set_rowset_id(0); // useless but required + rowset.set_rowset_id_v2("1"); + rowset.set_num_segments(1); + rowset.set_tablet_id(10000); + rowset.set_index_id(10001); + rowset.set_resource_id("delete_tmp_rowset_data_with_rowset_idx_v3"); + rowset.set_schema_version(schema.schema_version()); + rowset.mutable_tablet_schema()->CopyFrom(schema); + rowset.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); + create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, 1, true); + rowset_pbs.emplace(rowset.rowset_id_v2(), rowset); + + std::unordered_set list_files; + std::unique_ptr iter; + EXPECT_EQ(accessor->list_all(&iter), 0); + EXPECT_TRUE(iter->has_next()); + for (auto file = iter->next(); file.has_value(); file = iter->next()) { + list_files.insert(file->path); + } + EXPECT_EQ(list_files.size(), 2); + EXPECT_TRUE(list_files.contains("data/10000/1_0.dat")); + EXPECT_TRUE(list_files.contains("data/10000/1_0.idx")); + + ASSERT_EQ(0, + recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET, ctx)); + list_files.clear(); + iter.reset(); + EXPECT_EQ(accessor->list_all(&iter), 0); + EXPECT_FALSE(iter->has_next()); + for (auto file = iter->next(); file.has_value(); file = iter->next()) { + list_files.insert(file->path); + } + EXPECT_EQ(list_files.size(), 0); + } +} + TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) { auto* sp = SyncPoint::get_instance(); DORIS_CLOUD_DEFER { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 6e8c1f53d53688..b4e7d78d6b5f04 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -183,6 +183,9 @@ message RowsetMetaPB { // For row binlog LSN allocation (FE auto-inc RPC) optional int64 db_id = 1016; optional int64 table_id = 1017; + // Actual inverted-index format used by this rowset's files. It is independent + // from the shared (index_id, schema_version) schema key. + optional InvertedIndexStorageFormatPB inverted_index_storage_format = 1018; } message SchemaDictKeyList { @@ -301,6 +304,9 @@ message RowsetMetaCloudPB { optional int64 db_id = 115; optional int64 table_id = 116; + // Actual inverted-index format used by this rowset's files. It is independent + // from the shared (index_id, schema_version) schema key. + optional InvertedIndexStorageFormatPB inverted_index_storage_format = 117; } message SegmentStatisticsPB { @@ -732,6 +738,8 @@ message TabletMetaPB { optional string table_name = 1003; optional int64 ttl_seconds = 1004; optional int32 schema_version = 1005; // index_id, schema_version -> schema + // Immutable format of this tablet's inverted-index files. + optional InvertedIndexStorageFormatPB inverted_index_storage_format = 1006; } message TabletMetaCloudPB { @@ -791,6 +799,8 @@ message TabletMetaCloudPB { optional string table_name = 101; optional int64 ttl_seconds = 102; optional int32 schema_version = 103; // index_id, schema_version -> schema + // Immutable format of this tablet's inverted-index files. + optional InvertedIndexStorageFormatPB inverted_index_storage_format = 104; } message OLAPRawDeltaHeaderMessage {