Skip to content

[fix](cloud) preserve rowset inverted index format in recycler #66793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hoshinojyunn wants to merge 1 commit into
base: master
Choose a base branch
from
+120 −9
Open

[fix](cloud) preserve rowset inverted index format in recycler #66793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions be/src/cloud/pb_convert.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
}
if (in.has_inverted_index_storage_format()) {
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
}
if (in.has_visible_ts_ms()) {
out->set_visible_ts_ms(in.visible_ts_ms());
}
Expand Down Expand Up @@ -198,6 +201,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
if (in.has___split_schema()) {
out->mutable___split_schema()->Swap(in.mutable___split_schema());
}
if (in.has_inverted_index_storage_format()) {
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
}
if (in.has_visible_ts_ms()) {
out->set_visible_ts_ms(in.visible_ts_ms());
}
Expand Down Expand Up @@ -310,6 +316,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in)
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
if (in.has_inverted_index_storage_format()) {
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
}
auto* slice_locations = out->mutable_packed_slice_locations();
slice_locations->clear();
slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
Expand Down Expand Up @@ -405,6 +414,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
if (in.has_inverted_index_storage_format()) {
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
}
auto* slice_locations = out->mutable_packed_slice_locations();
slice_locations->clear();
slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
Expand Down Expand Up @@ -790,6 +802,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in)
if (in.has_encryption_algorithm()) {
out->set_encryption_algorithm(in.encryption_algorithm());
}
if (in.has_inverted_index_storage_format()) {
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
}
}

void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
Expand Down Expand Up @@ -875,6 +890,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
if (in.has_encryption_algorithm()) {
out->set_encryption_algorithm(in.encryption_algorithm());
}
if (in.has_inverted_index_storage_format()) {
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
}
}

TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) {
Expand Down Expand Up @@ -967,6 +985,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in)
if (in.has_encryption_algorithm()) {
out->set_encryption_algorithm(in.encryption_algorithm());
}
if (in.has_inverted_index_storage_format()) {
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
}
}

void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
Expand Down Expand Up @@ -1052,6 +1073,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
if (in.has_encryption_algorithm()) {
out->set_encryption_algorithm(in.encryption_algorithm());
}
if (in.has_inverted_index_storage_format()) {
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
}
}

} // namespace doris::cloud
7 changes: 5 additions & 2 deletions cloud/src/recycler/checker.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -772,10 +772,13 @@ int InstanceChecker::do_check() {
}
if (!index_ids.empty()) {
const auto& index_map = rs_meta.packed_slice_locations();
const auto index_format =
rs_meta.has_inverted_index_storage_format()
? rs_meta.inverted_index_storage_format()
: rs_meta.tablet_schema().inverted_index_storage_format();
for (int i = 0; i < rs_meta.num_segments(); ++i) {
std::vector<std::string> index_path_v;
if (rs_meta.tablet_schema().inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
if (index_format == InvertedIndexStorageFormatPB::V1) {
for (const auto& index_id : index_ids) {
LOG(INFO) << "check inverted index, tablet_id=" << rs_meta.tablet_id()
<< " rowset_id=" << rs_meta.rowset_id_v2() << " segment_id=" << i
Expand Down
23 changes: 16 additions & 7 deletions cloud/src/recycler/recycler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3470,8 +3470,9 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {

// Process inverted indexes
std::vector<std::pair<int64_t, std::string>> index_ids;
// default format as v1.
InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V1;
InvertedIndexStorageFormatPB index_format = rs_meta_pb.has_inverted_index_storage_format()
? rs_meta_pb.inverted_index_storage_format()
: InvertedIndexStorageFormatPB::V1;
bool delete_rowset_data_by_prefix = false;
if (rs_meta_pb.rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE) {
// if rowset state is RowsetStatePB::BEGIN_PARTIAL_UPDATE, the number of segments data
Expand All @@ -3483,7 +3484,8 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
index_ids.emplace_back(index.index_id(), index.index_suffix_name());
}
}
if (rs_meta_pb.tablet_schema().has_inverted_index_storage_format()) {
if (!rs_meta_pb.has_inverted_index_storage_format() &&
rs_meta_pb.tablet_schema().has_inverted_index_storage_format()) {
index_format = rs_meta_pb.tablet_schema().inverted_index_storage_format();
}
} else if (!rs_meta_pb.has_index_id() || !rs_meta_pb.has_schema_version()) {
Expand All @@ -3497,7 +3499,9 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
TEST_SYNC_POINT_CALLBACK("InstanceRecycler::delete_rowset_data.tmp_rowset",
&inverted_index_get_ret);
if (inverted_index_get_ret == 0) {
index_format = index_info.first;
if (!rs_meta_pb.has_inverted_index_storage_format()) {
index_format = index_info.first;
}
index_ids = index_info.second;
} else if (inverted_index_get_ret == 1) {
// 1. Schema kv not found means tablet has been recycled
Expand Down Expand Up @@ -4235,15 +4239,18 @@ int InstanceRecycler::delete_rowset_data(
// Process inverted indexes
std::vector<std::pair<int64_t, std::string>> index_ids;
// default format as v1.
InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V1;
InvertedIndexStorageFormatPB index_format = rs.has_inverted_index_storage_format()
? rs.inverted_index_storage_format()
: InvertedIndexStorageFormatPB::V1;
int inverted_index_get_ret = 0;
if (rs.has_tablet_schema()) {
for (const auto& index : rs.tablet_schema().index()) {
if (index.has_index_type() && index.index_type() == IndexType::INVERTED) {
index_ids.emplace_back(index.index_id(), index.index_suffix_name());
}
}
if (rs.tablet_schema().has_inverted_index_storage_format()) {
if (!rs.has_inverted_index_storage_format() &&
rs.tablet_schema().has_inverted_index_storage_format()) {
index_format = rs.tablet_schema().inverted_index_storage_format();
}
} else {
Expand All @@ -4261,7 +4268,9 @@ int InstanceRecycler::delete_rowset_data(
TEST_SYNC_POINT_CALLBACK("InstanceRecycler::delete_rowset_data.tmp_rowset",
&inverted_index_get_ret);
if (inverted_index_get_ret == 0) {
index_format = index_info.first;
if (!rs.has_inverted_index_storage_format()) {
index_format = index_info.first;
}
index_ids = index_info.second;
} else if (inverted_index_get_ret == 1) {
// 1. Schema kv not found means tablet has been recycled
Expand Down
65 changes: 65 additions & 0 deletions cloud/test/recycler_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8066,6 +8066,71 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) {
}
}

TEST(RecyclerTest, delete_tmp_rowset_data_with_rowset_idx_v3) {
auto txn_kv = std::make_shared<MemTxnKv>();
ASSERT_EQ(txn_kv->init(), 0);

InstanceInfoPB instance;
instance.set_instance_id(instance_id);
auto obj_info = instance.add_obj_info();
obj_info->set_id("delete_tmp_rowset_data_with_rowset_idx_v3");
obj_info->set_ak(config::test_s3_ak);
obj_info->set_sk(config::test_s3_sk);
obj_info->set_endpoint(config::test_s3_endpoint);
obj_info->set_region(config::test_s3_region);
obj_info->set_bucket(config::test_s3_bucket);
obj_info->set_prefix("delete_tmp_rowset_data_with_rowset_idx_v3");

doris::TabletSchemaCloudPB schema;
schema.set_schema_version(1);
schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
auto index = schema.add_index();
index->set_index_id(1);
index->set_index_type(IndexType::INVERTED);

{
InstanceRecycler recycler(txn_kv, instance, thread_group,
std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
auto accessor = recycler.accessor_map_.begin()->second;
std::map<std::string, doris::RowsetMetaCloudPB> rowset_pbs;
doris::RowsetMetaCloudPB rowset;
rowset.set_rowset_id(0); // useless but required
rowset.set_rowset_id_v2("1");
rowset.set_num_segments(1);
rowset.set_tablet_id(10000);
rowset.set_index_id(10001);
rowset.set_resource_id("delete_tmp_rowset_data_with_rowset_idx_v3");
rowset.set_schema_version(schema.schema_version());
rowset.mutable_tablet_schema()->CopyFrom(schema);
rowset.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3);
create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, 1, true);
rowset_pbs.emplace(rowset.rowset_id_v2(), rowset);

std::unordered_set<std::string> list_files;
std::unique_ptr<ListIterator> iter;
EXPECT_EQ(accessor->list_all(&iter), 0);
EXPECT_TRUE(iter->has_next());
for (auto file = iter->next(); file.has_value(); file = iter->next()) {
list_files.insert(file->path);
}
EXPECT_EQ(list_files.size(), 2);
EXPECT_TRUE(list_files.contains("data/10000/1_0.dat"));
EXPECT_TRUE(list_files.contains("data/10000/1_0.idx"));

ASSERT_EQ(0,
recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET, ctx));
list_files.clear();
iter.reset();
EXPECT_EQ(accessor->list_all(&iter), 0);
EXPECT_FALSE(iter->has_next());
for (auto file = iter->next(); file.has_value(); file = iter->next()) {
list_files.insert(file->path);
}
EXPECT_EQ(list_files.size(), 0);
}
}

TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) {
auto* sp = SyncPoint::get_instance();
DORIS_CLOUD_DEFER {
Expand Down
10 changes: 10 additions & 0 deletions gensrc/proto/olap_file.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,9 @@ message RowsetMetaPB {
// For row binlog LSN allocation (FE auto-inc RPC)
optional int64 db_id = 1016;
optional int64 table_id = 1017;
// Actual inverted-index format used by this rowset's files. It is independent
// from the shared (index_id, schema_version) schema key.
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 1018;
}

message SchemaDictKeyList {
Expand Down Expand Up @@ -301,6 +304,9 @@ message RowsetMetaCloudPB {
optional int64 db_id = 115;
optional int64 table_id = 116;

// Actual inverted-index format used by this rowset's files. It is independent
// from the shared (index_id, schema_version) schema key.
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 117;
}

message SegmentStatisticsPB {
Expand Down Expand Up @@ -732,6 +738,8 @@ message TabletMetaPB {
optional string table_name = 1003;
optional int64 ttl_seconds = 1004;
optional int32 schema_version = 1005; // index_id, schema_version -> schema
// Immutable format of this tablet's inverted-index files.
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 1006;
}

message TabletMetaCloudPB {
Expand Down Expand Up @@ -791,6 +799,8 @@ message TabletMetaCloudPB {
optional string table_name = 101;
optional int64 ttl_seconds = 102;
optional int32 schema_version = 103; // index_id, schema_version -> schema
// Immutable format of this tablet's inverted-index files.
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 104;
}

message OLAPRawDeltaHeaderMessage {
Expand Down
Loading