From d7e581e57e495934ad2ddff2324f9d2a38f97e4d Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Sat, 25 Jul 2026 13:48:10 +0200 Subject: [PATCH 1/2] *: re-vendor kvproto at b41e863 (2026-06-22) and record the vendored revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vendored protos were an unrecorded late-2023/early-2024 vintage (errorpb and pdpb untouched since #324, one hand-patched addition from #519). Nothing recorded which kvproto revision they came from, and the staleness now blocks real features. Re-vendor everything at pingcap/kvproto b41e86365ce0 — the revision client-go currently pins — and add proto/VERSION so the vintage is never again guesswork. Regenerated with the existing tonic-build pipeline (21 files). One mechanical source fix: pdpb.RequestHeader gained caller_component/caller_id, so the TSO stream's header construction now defaults the new fields. This commit is the re-vendor and nothing else. Newly available surface includes KvFlush / KvBufferBatchGet (pipelined DML), HealthFeedback, and pdpb.BatchScanRegions. Two new protocol semantics become VISIBLE with the refresh — errorpb.UndeterminedResult (an unknown raft apply outcome) and shared locks (SharedLock / SharedPessimisticLock, whose real holders live in shared_lock_infos). Handling them is deliberately left to follow-up PRs so this one stays a pure re-vendor. Neither is a regression introduced here: - UndeterminedResult falls through to the generic region-error arm, which backs off and retries — the same default the follow-up keeps for every plan that is not a commit point. - A shared-lock wrapper carries no key bytes on the wire regardless of which proto vintage parses it, so the keyspace truncation hazard predates this commit; the refresh only makes the wrapper identifiable. Tests: make check green; 67 lib tests; txn/raw/failpoint integration suites green against a local api-v2 cluster. Signed-off-by: Eduard R. --- proto/VERSION | 3 + proto/autoid.proto | 45 +- proto/brpb.proto | 1318 +++++++++----- proto/cdcpb.proto | 271 +-- proto/configpb.proto | 155 +- proto/coprocessor.proto | 259 ++- proto/deadlock.proto | 96 +- proto/debugpb.proto | 370 ++-- proto/diagnosticspb.proto | 106 +- proto/disaggregated.proto | 189 +- proto/disk_usage.proto | 16 +- proto/encryptionpb.proto | 175 +- proto/enginepb.proto | 94 +- proto/errorpb.proto | 221 +-- proto/gcpb.proto | 8 +- proto/import_kvpb.proto | 153 +- proto/import_sstpb.proto | 633 +++---- proto/include/eraftpb.proto | 202 +-- proto/include/gogoproto/gogo.proto | 8 + proto/include/rustproto.proto | 66 +- proto/keyspacepb.proto | 29 +- proto/kvrpcpb.proto | 2296 ++++++++++++++----------- proto/logbackuppb.proto | 55 +- proto/meta_storagepb.proto | 121 +- proto/metapb.proto | 197 ++- proto/mpp.proto | 109 +- proto/pdpb.proto | 1576 ++++++++++------- proto/raft_cmdpb.proto | 442 ++--- proto/raft_serverpb.proto | 408 ++--- proto/recoverdatapb.proto | 81 +- proto/replication_modepb.proto | 86 +- proto/resource_manager.proto | 156 +- proto/resource_usage_agent.proto | 47 +- proto/schedulingpb.proto | 274 +-- proto/tikvpb.proto | 498 +++--- proto/tracepb.proto | 64 +- proto/tsopb.proto | 150 +- src/generated/backup.rs | 566 +++++- src/generated/cdcpb.rs | 14 + src/generated/coprocessor.rs | 132 ++ src/generated/deadlock.rs | 25 +- src/generated/encryptionpb.rs | 85 + src/generated/errorpb.rs | 11 + src/generated/import_sstpb.rs | 169 +- src/generated/keyspacepb.rs | 33 + src/generated/kvrpcpb.rs | 381 +++- src/generated/logbackup.rs | 44 + src/generated/meta_storagepb.rs | 55 + src/generated/metapb.rs | 10 + src/generated/mpp.rs | 7 + src/generated/pdpb.rs | 665 ++++++- src/generated/raft_cmdpb.rs | 10 + src/generated/raft_serverpb.rs | 21 + src/generated/resource_manager.rs | 96 +- src/generated/resource_usage_agent.rs | 21 +- src/generated/schedulingpb.rs | 52 + src/generated/tikvpb.rs | 310 +++- src/generated/tsopb.rs | 10 + src/pd/timestamp.rs | 2 +- 59 files changed, 8931 insertions(+), 4765 deletions(-) create mode 100644 proto/VERSION diff --git a/proto/VERSION b/proto/VERSION new file mode 100644 index 00000000..6af27a69 --- /dev/null +++ b/proto/VERSION @@ -0,0 +1,3 @@ +pingcap/kvproto b41e86365ce0e0fba482005c0dfa5e2ca967d74c (2026-06-22) +Matches client-go v2.0.8-0.20260708122311-01bd8f99f4da (go.mod: kvproto v0.0.0-20260622063236-b41e86365ce0). +Re-vendor: copy proto/*.proto (matching names only) and include/ from that revision, then `make generate`. diff --git a/proto/autoid.proto b/proto/autoid.proto index 252ae58e..869b640d 100644 --- a/proto/autoid.proto +++ b/proto/autoid.proto @@ -4,44 +4,45 @@ package autoid; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - message AutoIDRequest { - int64 dbID = 1; - int64 tblID = 2; - bool isUnsigned = 3; - uint64 n = 4; - int64 increment = 5; - int64 offset = 6; - uint32 keyspaceID = 7; + int64 dbID = 1; + int64 tblID = 2; + bool isUnsigned = 3; + uint64 n = 4; + int64 increment = 5; + int64 offset = 6; + uint32 keyspaceID = 7; } message AutoIDResponse { - int64 min = 1; - int64 max = 2; + int64 min = 1; + int64 max = 2; - bytes errmsg = 3; + bytes errmsg = 3; } message RebaseRequest { - int64 dbID = 1; - int64 tblID = 2; - bool isUnsigned = 3; - int64 base = 4; - bool force = 5; + int64 dbID = 1; + int64 tblID = 2; + bool isUnsigned = 3; + int64 base = 4; + bool force = 5; } message RebaseResponse { - bytes errmsg = 1; + bytes errmsg = 1; } service AutoIDAlloc { - rpc AllocAutoID(AutoIDRequest) returns (AutoIDResponse) {} - rpc Rebase(RebaseRequest) returns (RebaseResponse) {} + rpc AllocAutoID(AutoIDRequest) returns (AutoIDResponse) {} + rpc Rebase(RebaseRequest) returns (RebaseResponse) {} } - diff --git a/proto/brpb.proto b/proto/brpb.proto index 13acb75c..f8c47d1d 100644 --- a/proto/brpb.proto +++ b/proto/brpb.proto @@ -1,325 +1,466 @@ syntax = "proto3"; package backup; -import "kvrpcpb.proto"; -import "errorpb.proto"; -import "metapb.proto"; import "encryptionpb.proto"; +import "errorpb.proto"; import "gogoproto/gogo.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; +enum PrepareSnapshotBackupRequestType { + // Update the lease of suspending some commands (Which may impact the + // disk snapshot backup) to be proposed. + // As long as the lease is kept, those commands may not be proposed. + // If the client has gone, the lease will be destroyed after its TTL. + UpdateLease = 0; + // Wait a region apply to the last index. + WaitApply = 1; + // Hint before closing the stream. Server will go back to normal after this. + // This should also return a final "UpdateLeaseResult" to the client, which + // indices whether the last lease is valid. + Finish = 2; +} + +message PrepareSnapshotBackupRequest { + // The type of the request. + PrepareSnapshotBackupRequestType ty = 1; + // The regions related to the request. + // If the request is "UpdateLease" or "Finish", it should be empty. + repeated metapb.Region regions = 2; + // The lease duration for "UpdateLease" request. + uint64 lease_in_seconds = 3; +} + +enum PrepareSnapshotBackupEventType { + // A region has finished wait apply. + WaitApplyDone = 0; + // A lease has been updated. + UpdateLeaseResult = 1; +} + +message PrepareSnapshotBackupResponse { + // The type of the event. + PrepareSnapshotBackupEventType ty = 1; + // The region bind to the type. + // For "WaitApplyDone". + metapb.Region region = 2; + // The error of the execution. + errorpb.Error error = 3; + // Whether the last lease is valid. + // For "UpdateLease" and "Finish". + bool last_lease_is_valid = 4; +} // The message save the metadata of a backup. message BackupMeta { - // ID and version of backuped cluster. - uint64 cluster_id = 1; - string cluster_version = 2; - // Save the version of BR running backup jobs. - string br_version = 11; - // The backupmeta scheme version. - int32 version = 12; - - // path field is no longer used. - reserved 3; reserved "path"; - // A set of files that compose a backup. - // Note: `files` is deprecated, as it bloats backupmeta. It is kept for - // compatibility, so new BR can restore older backups. - repeated File files = 4; - // An index to files contains data files. - MetaFile file_index = 13; - - // A pair of timestamp specifies a time range of a backup. - // For full backup, the start_version equals to the end_version, - // it means point in time. - // For incremental backup, the time range is specified as - // (start_version, end_version]. - uint64 start_version = 5; - uint64 end_version = 6; - - // Table metadata describes database and table info. - // Note: `schemas` is deprecated, as it bloats backupmeta. It is kept for - // compatibility, so new BR can restore older backups. - repeated Schema schemas = 7; - // An index to files contains Schemas. - MetaFile schema_index = 14; - - // If in raw kv mode, `start_versions`, `end_versions` and `schemas` will be - // ignored, and the backup data's range is represented by raw_ranges. - bool is_raw_kv = 8; - // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for - // compatibility, so new BR can restore older backups. - repeated RawRange raw_ranges = 9; - - // An index to files contains RawRanges. - MetaFile raw_range_index = 15; - - // In incremental backup, DDLs which are completed in - // (lastBackupTS, backupTS] will be stored here. - // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for - // compatibility, so new BR can restore older backups. - bytes ddls = 10; - // An index to files contains DDLs. - MetaFile ddl_indexes = 16; - // the backup result into `backupmeta` file - string backup_result = 17; - - // API version implies the encode of the key and value. - kvrpcpb.APIVersion api_version = 18; - - // the placement policy info in backup cluster. we assume the policy won't be too much for one cluster. - repeated PlacementPolicy policies = 19; - - // new_collations_enabled specifies the config `new_collations_enabled_on_first_bootstrap` in tidb. - string new_collations_enabled = 20; - - // If in txn kv mode, `schemas` will be ignored, the backup data's range is as same as normal backup. - bool is_txn_kv = 21; - - // maintain the id mapping from upstream cluster to downstream cluster. - repeated PitrDBMap db_maps = 22; - - BackupMode mode = 23; - - // record the backup range and the correspond SST files when using file-copy backup. - repeated BackupRange ranges = 24; - - // record the size of the backup data files and meta files - uint64 backup_size = 25; + option (gogoproto.goproto_unrecognized) = true; + + // ID and version of backuped cluster. + uint64 cluster_id = 1; + string cluster_version = 2; + // Save the version of BR running backup jobs. + string br_version = 11; + // The backupmeta scheme version. + int32 version = 12; + + // path field is no longer used. + reserved 3; + reserved "path"; + // A set of files that compose a backup. + // Note: `files` is deprecated, as it bloats backupmeta. It is kept for + // compatibility, so new BR can restore older backups. + repeated File files = 4; + // An index to files contains data files. + MetaFile file_index = 13; + + // A pair of timestamp specifies a time range of a backup. + // For full backup, the start_version equals to the end_version, + // it means point in time. + // For incremental backup, the time range is specified as + // (start_version, end_version]. + uint64 start_version = 5; + uint64 end_version = 6; + + // Table metadata describes database and table info. + // Note: `schemas` is deprecated, as it bloats backupmeta. It is kept for + // compatibility, so new BR can restore older backups. + repeated Schema schemas = 7; + // An index to files contains Schemas. + MetaFile schema_index = 14; + // Backup metadata compatibility version for restore. + // 0 means the backup was created before this field existed. + uint32 backup_schema_version = 26; + + // If in raw kv mode, `start_versions`, `end_versions` and `schemas` will be + // ignored, and the backup data's range is represented by raw_ranges. + bool is_raw_kv = 8; + // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for + // compatibility, so new BR can restore older backups. + repeated RawRange raw_ranges = 9; + + // An index to files contains RawRanges. + MetaFile raw_range_index = 15; + + // In incremental backup, DDLs which are completed in + // (lastBackupTS, backupTS] will be stored here. + // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for + // compatibility, so new BR can restore older backups. + bytes ddls = 10; + // An index to files contains DDLs. + MetaFile ddl_indexes = 16; + // the backup result into `backupmeta` file + string backup_result = 17; + + // API version implies the encode of the key and value. + kvrpcpb.APIVersion api_version = 18; + + // the placement policy info in backup cluster. we assume the policy won't be too much for one cluster. + repeated PlacementPolicy policies = 19; + + // new_collations_enabled specifies the config `new_collations_enabled_on_first_bootstrap` in tidb. + string new_collations_enabled = 20; + + // If in txn kv mode, `schemas` will be ignored, the backup data's range is as same as normal backup. + bool is_txn_kv = 21; + + // maintain the id mapping from upstream cluster to downstream cluster. + repeated PitrDBMap db_maps = 22; + + BackupMode mode = 23; + + // record the backup range and the correspond SST files when using file-copy backup. + repeated BackupRange ranges = 24; + + // record the size of the backup data files and meta files + uint64 backup_size = 25; } message BackupRange { - bytes start_key = 1; - bytes end_key = 2; + bytes start_key = 1; + bytes end_key = 2; + + repeated File files = 3; +} - repeated File files = 3; +// The message saves the physical table checksum +message TableMeta { + // physical id is either table id or partition id + int64 physical_id = 1; + uint64 crc64xor = 2; + uint64 total_kvs = 3; + uint64 total_bytes = 4; } message File { - string name = 1; - bytes sha256 = 2; + string name = 1; + bytes sha256 = 2; - bytes start_key = 3; - bytes end_key = 4; - uint64 start_version = 5; - uint64 end_version = 6; + bytes start_key = 3; + bytes end_key = 4; + uint64 start_version = 5; + uint64 end_version = 6; - uint64 crc64xor = 7; - uint64 total_kvs = 8; - uint64 total_bytes = 9; + // recorded the total statistics across multiple tables + uint64 crc64xor = 7; + uint64 total_kvs = 8; + uint64 total_bytes = 9; + // recorded when the SST kvs are from different tables + repeated TableMeta table_metas = 13; - string cf = 10; + string cf = 10; - uint64 size = 11; - // cipher_iv is used for AES cipher - bytes cipher_iv = 12; + uint64 size = 11; + // cipher_iv is used for AES cipher + bytes cipher_iv = 12; + + // Reserved for a downstream fork + reserved 1000 to 1199; } // MetaFile describes a multi-level index of data used in backup. message MetaFile { - // A set of files that contains a MetaFile. - // It is used as a multi-level index. - repeated File meta_files = 1; - // A set of files that contains user data. - repeated File data_files = 2; - // A set of files that contains Schemas. - repeated Schema schemas = 3; - // A set of files that contains RawRanges. - repeated RawRange raw_ranges = 4; - // A set of files that contains BackupRanges. - repeated BackupRange backup_ranges = 6; - // A set of files that contains DDLs. - repeated bytes ddls = 5; + // A set of files that contains a MetaFile. + // It is used as a multi-level index. + repeated File meta_files = 1; + // A set of files that contains user data. + repeated File data_files = 2; + // A set of files that contains Schemas. + repeated Schema schemas = 3; + // A set of files that contains RawRanges. + repeated RawRange raw_ranges = 4; + // A set of files that contains BackupRanges. + repeated BackupRange backup_ranges = 6; + // A set of files that contains DDLs. + repeated bytes ddls = 5; } message PlacementPolicy { - bytes info = 1; + bytes info = 1; +} + +message StatsBlock { + // The physical id which the stats json data belongs to. + int64 physical_id = 1; + + // The marshal data of the part of json_table. Currently it is the partition table. + bytes json_table = 2; +} + +message StatsFile { + // A set of files that contains json_tables. + repeated StatsBlock blocks = 1; +} + +message StatsFileIndex { + string name = 1; + bytes sha256 = 2; + + // the size of the file content encrypted + uint64 size_enc = 3; + // the size of the original file content + uint64 size_ori = 4; + // cipher_iv is used for AES cipher + bytes cipher_iv = 5; + + // If the size of the stats is very small, save the data into the metafile + bytes inline_data = 6; } message Schema { - bytes db = 1; - bytes table = 2; + bytes db = 1; + bytes table = 2; + + uint64 crc64xor = 3; + uint64 total_kvs = 4; + uint64 total_bytes = 5; + + uint32 tiflash_replicas = 6; - uint64 crc64xor = 3; - uint64 total_kvs = 4; - uint64 total_bytes = 5; + // stats represents the dump stats for a analyzed table, which generate by DumpStatsToJSON + // https://github.com/pingcap/tidb/blob/e136429d8dc5d70f43cd3f94179b0b9f47595097/statistics/handle/dump.go#L116 + bytes stats = 7; - uint32 tiflash_replicas = 6; + // An index of files that contains stats data + repeated StatsFileIndex stats_index = 8; - // stats represents the dump stats for a analyzed table, which generate by DumpStatsToJSON - // https://github.com/pingcap/tidb/blob/e136429d8dc5d70f43cd3f94179b0b9f47595097/statistics/handle/dump.go#L116 - bytes stats = 7; + // is_merge_option_allowed indicates whether merge option is allowed for this schema. + // For backward compatibility, this field defaults to false when not present. + bool is_merge_option_allowed = 9; + + // partition_merge_option_allowed indicates whether merge option is allowed for each partition. + // is_merge_option_allowed is table level, and the partition_merge_option_allowed is partition level. + map partition_merge_option_allowed = 10; } message IDMap { - int64 upstream_id = 1; - int64 downstream_id = 2; + int64 upstream_id = 1; + int64 downstream_id = 2; } -message PitrTableMap{ - string name = 1; - IDMap id_map = 2; - repeated IDMap partitions = 3; +message PitrTableMap { + string name = 1; + IDMap id_map = 2; + repeated IDMap partitions = 3; + // whether this table has been filtered out + bool filtered_out = 4; } -message PitrDBMap{ - string name = 1; - IDMap id_map = 2; - repeated PitrTableMap tables = 3; +message PitrDBMap { + string name = 1; + IDMap id_map = 2; + repeated PitrTableMap tables = 3; + // whether this db has been filtered out + bool filtered_out = 4; + // whether this db has been reused + // skip restoring the metakv of database in log restore if it is true + bool reused = 5; } message RawRange { - bytes start_key = 1; - bytes end_key = 2; - string cf = 3; + bytes start_key = 1; + bytes end_key = 2; + string cf = 3; } message ClusterIDError { - uint64 current = 1; - uint64 request = 2; + uint64 current = 1; + uint64 request = 2; } message Error { - string msg = 1; - oneof detail { - ClusterIDError cluster_id_error = 3; - kvrpcpb.KeyError kv_error = 4; - errorpb.Error region_error = 5; - } + string msg = 1; + oneof detail { + ClusterIDError cluster_id_error = 3; + kvrpcpb.KeyError kv_error = 4; + errorpb.Error region_error = 5; + } } // sst files or log files compression algorithm // for log files, unknown means not use compression algorithm enum CompressionType { - UNKNOWN = 0; - LZ4 = 1; - SNAPPY = 2; - ZSTD = 3; + UNKNOWN = 0; + LZ4 = 1; + SNAPPY = 2; + ZSTD = 3; +} + +message StreamBackupTaskSecurityConfig { + oneof encryption { + // not recommended in production. configure KMS based or locally managed master key instead in TiKV StreamBackupConfig + CipherInfo plaintext_data_key = 1; + MasterKeyConfig master_key_config = 2; + } } // BackupMpde represents the mode of this whole backup request to the cluster. // and we need to store it in `backupmeta`. enum BackupMode { - // scan and generate precise SST file of a backup range. - // then we don't need to do anything in future restore. - // Note: this mode will cost the CPU resource of TiKV. - SCAN = 0; - // check and upload the coarse overlap SST files of a backup range. - // then we need to use a merge iterator to filter unexpected kv in future restore. - // Note: this mode will save the CPU resource of TiKV. - FILE = 1; + // scan and generate precise SST file of a backup range. + // then we don't need to do anything in future restore. + // Note: this mode will cost the CPU resource of TiKV. + SCAN = 0; + // check and upload the coarse overlap SST files of a backup range. + // then we need to use a merge iterator to filter unexpected kv in future restore. + // Note: this mode will save the CPU resource of TiKV. + FILE = 1; } message CipherInfo { - encryptionpb.EncryptionMethod cipher_type = 1; - bytes cipher_key = 2; + encryptionpb.EncryptionMethod cipher_type = 1; + bytes cipher_key = 2; +} + +message MasterKeyConfig { + encryptionpb.EncryptionMethod encryption_type = 1; + repeated encryptionpb.MasterKey master_keys = 2; +} + +message SortedSubRanges { + repeated kvrpcpb.KeyRange sub_ranges = 1; } message BackupRequest { - uint64 cluster_id = 1; - - bytes start_key = 2; - bytes end_key = 3; - uint64 start_version = 4; - uint64 end_version = 5; - - // path field is deprecated, use storage_backend instead - reserved 6; reserved "path"; - - // The I/O rate limit for backup request. - uint64 rate_limit = 7; - // The concurrency for executing the backup request in every tikv node. - uint32 concurrency = 8; - - StorageBackend storage_backend = 9; - - // If raw kv mode is enabled, `start_version` and `end_version` will be ignored, and `cf` - // specifies which cf to backup. - bool is_raw_kv = 10; - string cf = 11; - // algorithm used for compress sst files - CompressionType compression_type = 12; - // sst compression level, some algorithms support negative compression levels - int32 compression_level = 13; - // The cipher_info is Used to encrypt sst - CipherInfo cipher_info = 14; - - // dst_api_version indicates the key-value encoding version used by the - // generated SST file. Accepted values: - // 1. "v1": the generated SST files are encoded with api-v1, can be restored - // to TiKV clusters whose api version is set to v1. - // 2. "v2": the generated SST files are encoded with api-v2, can be restored - // to TiKV clusters whose api version is set to v2. - kvrpcpb.APIVersion dst_api_version = 15; - - // with checkpoint, some subintervals of the range have been backed up and recorded. - // only the remaining sub ranges of the range need to be backed up this time. - repeated kvrpcpb.KeyRange sub_ranges = 16; - - // replica_read indicates whether to use replica read for backup. - // If it is false, the backup will only read data from leader. - // If it is true, the backup will read data from both leader and follower. - bool replica_read = 17; - BackupMode mode = 18; - - // unique_id represents the handle of this backup. after we implement file-copy backup. - // we need generate some internal states during the whole backup precedure. - // this unique id is help to find the state effictively. - string unique_id = 19; - kvrpcpb.Context context = 20; + uint64 cluster_id = 1; + + bytes start_key = 2; + bytes end_key = 3; + uint64 start_version = 4; + uint64 end_version = 5; + + // path field is deprecated, use storage_backend instead + reserved 6; + reserved "path"; + + // The I/O rate limit for backup request. + uint64 rate_limit = 7; + // The concurrency for executing the backup request in every tikv node. + uint32 concurrency = 8; + + StorageBackend storage_backend = 9; + + // If raw kv mode is enabled, `start_version` and `end_version` will be ignored, and `cf` + // specifies which cf to backup. + bool is_raw_kv = 10; + string cf = 11; + // algorithm used for compress sst files + CompressionType compression_type = 12; + // sst compression level, some algorithms support negative compression levels + int32 compression_level = 13; + // The cipher_info is Used to encrypt sst + CipherInfo cipher_info = 14; + + // dst_api_version indicates the key-value encoding version used by the + // generated SST file. Accepted values: + // 1. "v1": the generated SST files are encoded with api-v1, can be restored + // to TiKV clusters whose api version is set to v1. + // 2. "v2": the generated SST files are encoded with api-v2, can be restored + // to TiKV clusters whose api version is set to v2. + kvrpcpb.APIVersion dst_api_version = 15; + + // with checkpoint, some subintervals of the range have been backed up and recorded. + // only the remaining sub ranges of the range need to be backed up this time. + repeated kvrpcpb.KeyRange sub_ranges = 16; + + // only ranges in the same groups can be merged into one SST file. + // the ranges in each groups are sorted. + repeated SortedSubRanges sorted_sub_ranges_groups = 21; + + // replica_read indicates whether to use replica read for backup. + // If it is false, the backup will only read data from leader. + // If it is true, the backup will read data from both leader and follower. + bool replica_read = 17; + BackupMode mode = 18; + + // unique_id represents the handle of this backup. after we implement file-copy backup. + // we need generate some internal states during the whole backup precedure. + // this unique id is help to find the state effictively. + string unique_id = 19; + kvrpcpb.Context context = 20; + + // Reserved for a downstream fork + reserved 1000 to 1199; } message StreamBackupTaskInfo { - // The storage for backup, parsed by BR. - StorageBackend storage = 1; - // The time range for backing up. - uint64 start_ts = 2; - uint64 end_ts = 3; - - // Misc meta datas. - // The name of the task, also the ID of the task. - string name = 4; - // The table filter of the task. - // Only for displaying the task info. - repeated string table_filter = 5; - - // compression type - CompressionType compression_type = 6; - - // The last timestamp of the task has been updated. - // This is a simple solution for unfrequent config changing: - // When we watched a config change(via polling or etcd watching), - // We perform a incremental scan between [last_update_ts, now), - // for filling the diff data during conf changing. - // The current implementation scan [0, now) for every task ranges newly added, - // So this field is reserved for future usage. - reserved "last_update_ts"; - - // We use '/tidb/br-stream/ranges// -> ' - // for storing all the target ranges of the task, the "ranges" field will not be saved in the taskinfo. - reserved "ranges"; - - // The status field allow us mark the task as 'paused'. - // We use '/tidb/br-stream/pause/ -> ()' for pausing a task for now. - // If we need more complexed status transforming, maybe add the status field back would be fine. - reserved "status"; + // The storage for backup, parsed by BR. + StorageBackend storage = 1; + // The time range for backing up. + uint64 start_ts = 2; + uint64 end_ts = 3; + + // Misc meta data. + // The name of the task, also the ID of the task. + string name = 4; + // The table filter of the task. + // Only for displaying the task info. + repeated string table_filter = 5; + + // compression type + CompressionType compression_type = 6; + + // security config for backup files + StreamBackupTaskSecurityConfig security_config = 7; + + // The last timestamp of the task has been updated. + // This is a simple solution for unfrequent config changing: + // When we watched a config change(via polling or etcd watching), + // We perform a incremental scan between [last_update_ts, now), + // for filling the diff data during conf changing. + // The current implementation scan [0, now) for every task ranges newly added, + // So this field is reserved for future usage. + reserved "last_update_ts"; + + // We use '/tidb/br-stream/ranges// -> ' + // for storing all the target ranges of the task, the "ranges" field will not be saved in the taskinfo. + reserved "ranges"; + + // The status field allow us mark the task as 'paused'. + // We use '/tidb/br-stream/pause/ -> ()' for pausing a task for now. + // If we need more complexed status transforming, maybe add the status field back would be fine. + reserved "status"; } message StorageBackend { - oneof backend { - Noop noop = 1; - Local local = 2; - S3 s3 = 3; - GCS gcs = 4; - CloudDynamic cloud_dynamic = 5; - HDFS hdfs = 6; - AzureBlobStorage azure_blob_storage = 7; - } + oneof backend { + Noop noop = 1; + Local local = 2; + S3 s3 = 3; + GCS gcs = 4; + CloudDynamic cloud_dynamic = 5; + HDFS hdfs = 6; + AzureBlobStorage azure_blob_storage = 7; + } } // Noop storage backend saves files into void. @@ -327,319 +468,546 @@ message Noop {} // Local storage backend saves files into local disk message Local { - string path = 1; + string path = 1; } // S3 storage backend saves files into S3 compatible storages // For non-aws providers, endpoint must be provided message S3 { - string endpoint = 1; - string region = 2; - string bucket = 3; - string prefix = 4; - string storage_class = 5; - // server side encryption - string sse = 6; - string acl = 7; - string access_key = 8; - string secret_access_key = 9; - bool force_path_style = 10; - string sse_kms_key_id = 11; - string role_arn = 12; - string external_id = 13; - bool object_lock_enabled = 14; - string session_token = 15; - string provider = 16; - string profile = 17; + string endpoint = 1; + string region = 2; + string bucket = 3; + string prefix = 4; + string storage_class = 5; + // server side encryption + string sse = 6; + string acl = 7; + string access_key = 8; + string secret_access_key = 9; + bool force_path_style = 10; + string sse_kms_key_id = 11; + string role_arn = 12; + string external_id = 13; + bool object_lock_enabled = 14; + string session_token = 15; + string provider = 16; + string profile = 17; } // GCS storage backend saves files into google cloud storage. message GCS { - string endpoint = 1; - string bucket = 2; - string prefix = 3; - string storage_class = 4; - // If not empty, applies a predefined set of access controls. - // See https://cloud.google.com/storage/docs/access-control/lists#predefined-acl - // for valid values. - string predefined_acl = 5; - // Service Account Credentials JSON blob - // You can get one from https://console.cloud.google.com/apis/credentials, and - // copy the content, set it as string here. - string credentials_blob = 6; + string endpoint = 1; + string bucket = 2; + string prefix = 3; + string storage_class = 4; + // If not empty, applies a predefined set of access controls. + // See https://cloud.google.com/storage/docs/access-control/lists#predefined-acl + // for valid values. + string predefined_acl = 5; + // Service Account Credentials JSON blob + // You can get one from https://console.cloud.google.com/apis/credentials, and + // copy the content, set it as string here. + string credentials_blob = 6; } // The encryption algorithm must be AES256. message AzureCustomerKey { - // A Base64-encoded AES-256 encryption key value. - string encryption_key = 1; - // The Base64-encoded SHA256 of the encryption key. - string encryption_key_sha256 = 2; + // A Base64-encoded AES-256 encryption key value. + string encryption_key = 1; + // The Base64-encoded SHA256 of the encryption key. + string encryption_key_sha256 = 2; } // AzureBlobStorage storage backend saves files into azure blob storage. message AzureBlobStorage { - string endpoint = 1; - // Alias: container - string bucket = 2; - // Notice: prefix starts without `/`, otherwise the first directory's name is empty. - string prefix = 3; - // Alias: access_tier. - // See https://docs.microsoft.com/en-us/azure/storage/blobs/access-tiers-overview - string storage_class = 4; - - // if empty, try to read account_name from the node's environment variable $AZURE_STORAGE_ACCOUNT. - string account_name = 5; - - // There are 3 kinds of credentials, and the priority order is - // `SAS > Shared key > Azure AD (env) > Shared key (env)`. - // - // 1. Use shared key to access the azure blob - // If the node's environment variables($AZURE_CLIENT_ID, $AZURE_TENANT_ID, $AZURE_CLIENT_SECRET) exist, - // prefer to use token to access the azure blob. - // - // See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key - // - // Otherwise, if empty, try to read shared key from the node's environment variable $AZURE_STORAGE_KEY. - string shared_key = 6; - - // 2. Use Azure AD (Azure Active Directory) to access the azure blob - // - // See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-azure-active-directory - // - // The Azure AD would generate the token, which tasks some time. - // So it is not recommanded to generate the token in each request. - // // AzureActiveDirectory azure_ad = #; - // - // 3. Use SAS (shared access signature) - // - // See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature - // - string access_sig = 8; - - // Server Side Encryption, 2 types in total: - // - // 1. Specify an encryption scope for uploaded blobs. - // - // See https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-scope-manage?tabs=powershell#upload-a-blob-with-an-encryption-scope - // - string encryption_scope = 9; - - // 2. Provide an encryption key on a request to blob storage. - // - // See https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys - // - AzureCustomerKey encryption_key = 10; + string endpoint = 1; + // Alias: container + string bucket = 2; + // Notice: prefix starts without `/`, otherwise the first directory's name is empty. + string prefix = 3; + // Alias: access_tier. + // See https://docs.microsoft.com/en-us/azure/storage/blobs/access-tiers-overview + string storage_class = 4; + + // if empty, try to read account_name from the node's environment variable $AZURE_STORAGE_ACCOUNT. + string account_name = 5; + + // There are 3 kinds of credentials, and the priority order is + // `SAS > Shared key > Azure AD (env) > Shared key (env)`. + // + // 1. Use shared key to access the azure blob + // If the node's environment variables($AZURE_CLIENT_ID, $AZURE_TENANT_ID, $AZURE_CLIENT_SECRET) exist, + // prefer to use token to access the azure blob. + // + // See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key + // + // Otherwise, if empty, try to read shared key from the node's environment variable $AZURE_STORAGE_KEY. + string shared_key = 6; + + // 2. Use Azure AD (Azure Active Directory) to access the azure blob + // + // See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-azure-active-directory + // + // The Azure AD would generate the token, which tasks some time. + // So it is not recommanded to generate the token in each request. + // // AzureActiveDirectory azure_ad = #; + // + // 3. Use SAS (shared access signature) + // + // See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature + // + string access_sig = 8; + + // Server Side Encryption, 2 types in total: + // + // 1. Specify an encryption scope for uploaded blobs. + // + // See https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-scope-manage?tabs=powershell#upload-a-blob-with-an-encryption-scope + // + string encryption_scope = 9; + + // 2. Provide an encryption key on a request to blob storage. + // + // See https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys + // + AzureCustomerKey encryption_key = 10; } message Bucket { - string endpoint = 1; - string region = 3; - string bucket = 4; - string prefix = 5; - string storage_class = 6; + string endpoint = 1; + string region = 3; + string bucket = 4; + string prefix = 5; + string storage_class = 6; } // CloudDynamic allows testing new cloud providers and new fields without changing protobuf definitions message CloudDynamic { - Bucket bucket = 1; - string provider_name = 2; // s3, gcs and azureBlobStorage are supported - map attrs = 3; + Bucket bucket = 1; + string provider_name = 2; // s3, gcs and azureBlobStorage are supported + map attrs = 3; } // HDFS storage backend saves file into HDFS compatible storages message HDFS { - // a URL: hdfs:///some/path or hdfs://host:port/some/path - string remote = 1; + // a URL: hdfs:///some/path or hdfs://host:port/some/path + string remote = 1; } - message BackupResponse { - Error error = 1; - bytes start_key = 2; - bytes end_key = 3; - repeated File files = 4; + Error error = 1; + bytes start_key = 2; + bytes end_key = 3; + repeated File files = 4; - // API version implies the encode of the key and value. - kvrpcpb.APIVersion api_version = 5; + // API version implies the encode of the key and value. + kvrpcpb.APIVersion api_version = 5; } -message CleanupRequest { - // unique_id represents the unique handle of the whole backup predecure. - // it generated in prepare request and corrosponed to one specific backup. - string unique_id = 1; +message CleanupRequest { + // unique_id represents the unique handle of the whole backup predecure. + // it generated in prepare request and corrosponed to one specific backup. + string unique_id = 1; } message CleanupResponse { - Error error = 1; - bool success = 2; + Error error = 1; + bool success = 2; } -message PrepareRequest { - // whether save state to the storage. - bool save_to_storage = 1; +message PrepareRequest { + // whether save state to the storage. + bool save_to_storage = 1; } message PrepareResponse { - Error error = 1; - // unique_id represents the unique handle of the whole backup predecure. - // if unique_id = 0 means prepare failed. - // if unique_id > 0 means prepare success and all states saved with this unique info. - string unique_id = 2; - uint64 collect_file_count = 3; - uint64 collect_file_size = 4; + Error error = 1; + // unique_id represents the unique handle of the whole backup predecure. + // if unique_id = 0 means prepare failed. + // if unique_id > 0 means prepare success and all states saved with this unique info. + string unique_id = 2; + uint64 collect_file_count = 3; + uint64 collect_file_size = 4; } -message CheckAdminRequest { -} +message CheckAdminRequest {} message CheckAdminResponse { - Error error = 1; - metapb.Region region = 2; - bool has_pending_admin = 3; + Error error = 1; + metapb.Region region = 2; + bool has_pending_admin = 3; } service Backup { - rpc backup(BackupRequest) returns (stream BackupResponse) {} - // CheckPendingAdminOp used for snapshot backup. before we start snapshot for a TiKV. - // we need stop all schedule first and make sure all in-flight schedule has finished. - // this rpc check all pending conf change for leader. - rpc CheckPendingAdminOp(CheckAdminRequest) returns (stream CheckAdminResponse) {} + rpc backup(BackupRequest) returns (stream BackupResponse) {} + + // CheckPendingAdminOp used for snapshot backup. before we start snapshot for a TiKV. + // we need stop all schedule first and make sure all in-flight schedule has finished. + // this rpc check all pending conf change for leader. + rpc CheckPendingAdminOp(CheckAdminRequest) returns (stream CheckAdminResponse) {} - // prepare is used for file-copy backup. before we start the backup for a TiKV. - // we need invoke this function to generate the SST files map. or we get nothing to backup. - rpc prepare(PrepareRequest) returns (PrepareResponse) {} + // PrepareSnapshotBackup is an advanced version of preparing snapshot backup. + // Check the defination of `PrepareSnapshotBackupRequest` for more. + rpc PrepareSnapshotBackup(stream PrepareSnapshotBackupRequest) returns (stream PrepareSnapshotBackupResponse); - // cleanup used for file-copy backup. after we finish the backup for a TiKV. - // we need clean some internel state. e.g. checkpoint, SST File maps - rpc cleanup(CleanupRequest) returns (CleanupResponse) {} + // prepare is used for file-copy backup. before we start the backup for a TiKV. + // we need invoke this function to generate the SST files map. or we get nothing to backup. + rpc prepare(PrepareRequest) returns (PrepareResponse) {} + + // cleanup used for file-copy backup. after we finish the backup for a TiKV. + // we need clean some internel state. e.g. checkpoint, SST File maps + rpc cleanup(CleanupRequest) returns (CleanupResponse) {} } // ExternalStorage is a service for using a cloud backend from StorageBackend to store files. // This can be used to backup and restore SST files. service ExternalStorage { - // Restore to a file - rpc restore(ExternalStorageRestoreRequest) returns (ExternalStorageRestoreResponse) {} - // Save a file to storage - rpc save(ExternalStorageSaveRequest) returns (ExternalStorageSaveResponse) {} + // Restore to a file + rpc restore(ExternalStorageRestoreRequest) returns (ExternalStorageRestoreResponse) {} + // Save a file to storage + rpc save(ExternalStorageSaveRequest) returns (ExternalStorageSaveResponse) {} } message ExternalStorageRestoreRequest { - StorageBackend storage_backend = 1; - string object_name = 2; - string restore_name = 3; - uint64 content_length = 4; + StorageBackend storage_backend = 1; + string object_name = 2; + string restore_name = 3; + uint64 content_length = 4; } -message ExternalStorageRestoreResponse { -} +message ExternalStorageRestoreResponse {} message ExternalStorageSaveRequest { - StorageBackend storage_backend = 1; - string object_name = 2; - uint64 content_length = 3; + StorageBackend storage_backend = 1; + string object_name = 2; + uint64 content_length = 3; } -message ExternalStorageSaveResponse { -} +message ExternalStorageSaveResponse {} enum MetaVersion { - V1 = 0; - V2 = 1; + V1 = 0; + V2 = 1; } message Metadata { - // deprecated after(in) v6.3.0 TiKV cluster - repeated DataFileInfo files = 1; - repeated DataFileGroup file_groups = 6; - int64 store_id = 2; - uint64 resolved_ts = 3; - uint64 max_ts = 4; - uint64 min_ts = 5; - MetaVersion meta_version = 7; + // deprecated after(in) v6.3.0 TiKV cluster + repeated DataFileInfo files = 1; + repeated DataFileGroup file_groups = 6; + int64 store_id = 2; + uint64 resolved_ts = 3; + uint64 max_ts = 4; + uint64 min_ts = 5; + MetaVersion meta_version = 7; } enum FileType { - Delete = 0; - Put = 1; + Delete = 0; + Put = 1; } // DataFileGroup is the merged file info in log-backup message DataFileGroup { - // Path of the file. - string path = 1; + option (rustproto.carllerche_bytes_for_bytes) = true; + option (rustproto.carllerche_bytes_for_string) = true; + + // Path of the file. + string path = 1; - // Partitions of the file. - repeated DataFileInfo data_files_info = 2; + // Partitions of the file. + repeated DataFileInfo data_files_info = 2; - /// Below are extra information of the file, for better filtering files. - // The min ts of the keys in the file. - uint64 min_ts = 3; - // The max ts of the keys in the file. - uint64 max_ts = 4; - // The resolved ts of the region when saving the file. - uint64 min_resolved_ts = 5; + /// Below are extra information of the file, for better filtering files. + // The min ts of the keys in the file. + uint64 min_ts = 3; + // The max ts of the keys in the file. + uint64 max_ts = 4; + // The resolved ts of the region when saving the file. + uint64 min_resolved_ts = 5; - // The file length after compressed. - uint64 length = 6; + // The file length after compressed. + uint64 length = 6; } message DataFileInfo { - // SHA256 of the file. - bytes sha256 = 1; - // Path of the file. - string path = 2; - int64 number_of_entries = 3; - - /// Below are extra information of the file, for better filtering files. - // The min ts of the keys in the file. - uint64 min_ts = 4; - // The max ts of the keys in the file. - uint64 max_ts = 5; - // The resolved ts of the region when saving the file. - uint64 resolved_ts = 6; - // The region of the file. - int64 region_id = 7; - // The key range of the file. - // Encoded and starts with 'z'(internal key). - bytes start_key = 8; - bytes end_key = 9; - // The column family of the file. - string cf = 10; - // The operation type of the file. - FileType type = 11; - - // Whether the data file contains meta keys(m prefixed keys) only. - bool is_meta = 12; - // The table ID of the file contains, when `is_meta` is true, would be ignored. - int64 table_id = 13; - - // The file length. - uint64 length = 14; - - // The minimal begin ts in default cf if this file is write cf. - uint64 min_begin_ts_in_default_cf = 15; - - // Offset of the partition. compatible with V1 and V2. - uint64 range_offset = 16; - - // The range length of the merged file, if it exists. - uint64 range_length = 17; - - // The compression type for the file. - CompressionType compression_type = 18; - - // It may support encrypting at future. - reserved "iv"; + option (rustproto.carllerche_bytes_for_bytes) = true; + option (rustproto.carllerche_bytes_for_string) = true; + + // Checksum of the plaintext file, i.e., pre-compression, pre-encryption. + bytes sha256 = 1; + // Path of the file. + string path = 2; + int64 number_of_entries = 3; + + /// Below are extra information of the file, for better filtering files. + // The min ts of the keys in the file. + uint64 min_ts = 4; + // The max ts of the keys in the file. + uint64 max_ts = 5; + // The resolved ts of the region when saving the file. + uint64 resolved_ts = 6; + // The region of the file. + int64 region_id = 7; + // The key range of the file. + // Encoded and starts with 'z'(internal key). + bytes start_key = 8; + bytes end_key = 9; + // The column family of the file. + string cf = 10; + // The operation type of the file. + FileType type = 11; + + // Whether the data file contains meta keys(m prefixed keys) only. + bool is_meta = 12; + // The table ID of the file contains, when `is_meta` is true, would be ignored. + int64 table_id = 13; + + // The file length. + uint64 length = 14; + + // The minimal begin ts in default cf if this file is write cf. + uint64 min_begin_ts_in_default_cf = 15; + + // Offset of the partition. compatible with V1 and V2. + uint64 range_offset = 16; + + // The range length of the merged file, if it exists. + uint64 range_length = 17; + + // The compression type for the file. + CompressionType compression_type = 18; + + // The CRC64XOR of all contents of the file. + // In older versions, this might be empty. + uint64 crc64xor = 19; + + // The original region range the log file belongs to. + // In older versions, this might be empty. + // This may not be accurative: file may contain keys less than this. + bytes region_start_key = 20; + // The original region range the log file belongs to. + // In older versions, this might be empty. + // This may not be accurative: file may contain keys greater than this. + bytes region_end_key = 21; + // The region epoch that the log file belongs to. + // In older versions, this might be empty. + // If a region get split or merged during observing, the file may contain multi epoches. + // This may not be complete: file may contain records from other versions. + // + // If there is exactly one epoch, + // `region_start_key` and `region_end_key` must match this epoch. + repeated metapb.RegionEpoch region_epoch = 22; + + // Encryption information of this data file, not set if plaintext. + encryptionpb.FileEncryptionInfo file_encryption_info = 23; } message StreamBackupError { - // the unix epoch time (in millisecs) of the time the error reported. - uint64 happen_at = 1; - // the unified error code of the error. - string error_code = 2; - // the user-friendly error message. - string error_message = 3; - // the store id of who issues the error. - uint64 store_id = 4; + // the unix epoch time (in millisecs) of the time the error reported. + uint64 happen_at = 1; + // the unified error code of the error. + string error_code = 2; + // the user-friendly error message. + string error_message = 3; + // the store id of who issues the error. + uint64 store_id = 4; +} + +message Span { + uint64 offset = 1; + uint64 length = 2; +} + +message SpansOfFile { + string path = 1; + repeated Span spans = 2; +} + +message LogFileSubcompactionMeta { + // The whole size of this subcompaction. + // It should be the physical size of all input logs. + uint64 size = 1; + // The region that we perform the subcompaction. + uint64 region_id = 2; + string cf = 3; + + uint64 input_max_ts = 4; + uint64 input_min_ts = 5; + // The whole compaction's configure. + uint64 compact_from_ts = 6; + uint64 compact_until_ts = 7; + // The key range of the input logs. + // They are encoded user key. (i.e. mem-comparable, no 'z' prefix) + bytes min_key = 8; + // The largest key from the input. + bytes max_key = 9; + // The type of operations in the file. (Write or Delete) + FileType ty = 10; + int64 table_id = 11; + + // The input log files of this. + repeated SpansOfFile sources = 12; +} + +// A subcompaction from a compaction. +// Subcompactions are actions that collect then sort key values from one region. +message LogFileSubcompaction { + // The metadata of the subcompaction. + LogFileSubcompactionMeta meta = 1; + // The generated SSTs. + repeated File sst_outputs = 2; + // The hint for the history of the region that contains the keys being compacted. + // This may be helpful for splitting, but it does not guarantee completeness and accuracy. + repeated RegionMetaHint region_meta_hints = 3; +} + +message RegionMetaHint { + // You may get region id from `LogFileSubcompactionMeta`. + + // The left boundary of the region of this epoch. + // It is an byte-comparable encoded key without TS. + bytes start_key = 1; + // The right boundary of the region of the epoch. + bytes end_key = 2; + // The region epoch. + metapb.RegionEpoch region_epoch = 3; +} + +// Batched version of `LogFileSubcompaction`. +// So we can store many subcompactions to one file, to reduce the number of file. +message LogFileSubcompactions { + repeated LogFileSubcompaction subcompactions = 1; +} + +message LogFileCompaction { + // The selector of compaction source. + // Files contains ANY record greater than this (and less than `until_ts`) will be selected. + // The source and artificats may contain records out of the `from_ts` and `until_ts` range. + // When restoring, please use `input_min_ts` to filter out not needed compactions instead. + uint64 compaction_from_ts = 1; + // The selector of compaction source. + // Files contains ANY record less than this (and not less than `from_ts`) will be selected. + // The source and artificats may contain records out of the `from_ts` and `until_ts` range. + // When restoring, please use `input_max_ts` to filter out not needed compactions instead. + uint64 compaction_until_ts = 2; + // The user provided name for the compaction. + string name = 3; + // The statistics or other useful information for human. + string comments = 4; + + // relative path to the generated subcompaction metas. ("artificates") + string artifacts = 5; + // the XOR of all artificate's hash. + uint64 artifacts_hash = 6; + + // relative path to the subcompaction generated SST files. + // this is a hint for garbage-collecting, so we don't need to read the metas one by one. + // Those SSTs cannot be directly used, what you need is probably in `artificates`. + string generated_files = 7; + + // The smallest TS of record contained among all selected files of this compaction. + uint64 input_min_ts = 8; + // The largest TS of record contained among all selected files of this compaction. + uint64 input_max_ts = 9; + + // Total size of generated compact-log-backup cmeta files. + uint64 generated_cmeta_files_total_size = 10; + // Total size of generated compact-log-backup SST files. + uint64 generated_sst_files_total_size = 11; +} + +message MetaEdit { + // Path to the meta file. + string path = 1; + // Delete the physical files (MetaFileGroup) in the meta file. + repeated string delete_physical_files = 2; + // Delete the logical files (MetaFileInfo) in the meta file. + // Note: Even the operation have been performed in the meta, + // this modification should be kept as long as the corresponding physical + // file not deleted. Or we may cannot know when to delete the physical file. + // Then the file will be leak until truncated. + repeated DeleteSpansOfFile delete_logical_files = 3; + // Whether to delete the metadata itself. + bool destruct_self = 4; + + // Whether all data kv files have been compacted but meta kv file still remains. + // this field used to determine whether we can ignore the physical data kv log files in a compaction. + bool all_data_files_compacted = 5; +} + +// RewrittenTableID records a sort of modification over the SSTs during restoring. +// +// When "backing up" this "restored" SST, we want to backup the original SST before any rewrite. +// But in some cases, SSTs are not restored "as is", TiKV may rewrite or truncate its +// content by the client's request. +message RewrittenTableID { + // The table ID in the backup data. + int64 ancestor_upstream = 1; + // The rewritten table ID during restoring. + int64 upstream = 2; +} + +// IngestedSSTs is created by a client that wants to put data to the cluster by `Ingest` APIs, +// these write cannot be directly recorded by log backup. The client should put a migration +// that contains the SSTs it uploaded. +message IngestedSSTs { + // The table IDs rewritten during restoring. + repeated RewrittenTableID rewritten_tables = 1; + // The SST files restored. + repeated File files = 2; + // Treating the whole batch of SSTs as an huge atomic + // write as this timestamp. + // That is, when user wants to PiTR to sometime after this, + // everything restored will present after the PiTR. + // Otherwise nothing will present. + uint64 as_if_ts = 3; + // The hint of the common prefix of the files. + // Used for speed up garbage collecting. + string files_prefix_hint = 4; + // Whether this batch have finished. + // If not, `as_if_ts` should be ignored and this should only + // be restored when there is a finished backup with the same + // `backup_uuid`. + bool finished = 5; + // When checkpoint enabled, one restoration may be separated to many + // of `IngestedSST`s. + // `IngestedSST`s sharing the same UUID should be treated as one. + // That is: + // - The `as_if_ts` should be the largest one among all finished ingested SST batches. + // - When restoring, all batches sharing the same uuid should be restored or not. + bytes backup_uuid = 6; +} + +// An extended version of `SpansOfFile`, added more metadata for the +// execution of delayed deletion. +message DeleteSpansOfFile { + string path = 1; + repeated Span spans = 2; + + // This is a hint for garbage collecting. + // So we can easily check whether a set of spans consists the whole file. + uint64 whole_file_length = 3; +} + +enum MigrationVersion { + // The initial version. Somehow "UNKNOWN". + M0 = 0; + // Added `creator` and `version`. + M1 = 1; + // Added `ingested_sst_paths`. + M2 = 2; +} + +message Migration { + string creator = 5; + MigrationVersion version = 6; + + repeated MetaEdit edit_meta = 1; + uint64 truncated_to = 2; + repeated LogFileCompaction compactions = 3; + // Delete all files matches this prefix. + repeated string destruct_prefix = 4; + // The ingested SSTs. + // It is mutable during restoring, hence a reference. + repeated string ingested_sst_paths = 7; } diff --git a/proto/cdcpb.proto b/proto/cdcpb.proto index 6d27130f..3a89a204 100644 --- a/proto/cdcpb.proto +++ b/proto/cdcpb.proto @@ -1,191 +1,204 @@ syntax = "proto3"; package cdcpb; -import "raft_cmdpb.proto"; -import "metapb.proto"; import "errorpb.proto"; -import "kvrpcpb.proto"; - import "gogoproto/gogo.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; +import "raft_cmdpb.proto"; import "rustproto.proto"; -option(gogoproto.sizer_all) = true; -option(gogoproto.marshaler_all) = true; -option(gogoproto.unmarshaler_all) = true; -option(rustproto.lite_runtime_all) = true; - option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; message Header { - uint64 cluster_id = 1; - string ticdc_version = 2; + uint64 cluster_id = 1; + string ticdc_version = 2; } message DuplicateRequest { - uint64 region_id = 1; + uint64 region_id = 1; } message Compatibility { - string required_version = 1; + string required_version = 1; } // ClusterIDMismatch is an error variable that // tells people that the cluster ID of the request does not match the TiKV cluster ID. message ClusterIDMismatch { - // The current tikv cluster ID. - uint64 current = 1; - // The cluster ID of the TiCDC request. - uint64 request = 2; + // The current tikv cluster ID. + uint64 current = 1; + // The cluster ID of the TiCDC request. + uint64 request = 2; +} + +// Congested is an error variable that +// tells people that the TiKV-CDC is congested. +message Congested { + // The region ID that triggers the congestion. + uint64 region_id = 1; } message Error { - errorpb.NotLeader not_leader = 1; - errorpb.RegionNotFound region_not_found = 2; - errorpb.EpochNotMatch epoch_not_match = 3; - DuplicateRequest duplicate_request = 4; - Compatibility compatibility = 5; - ClusterIDMismatch cluster_id_mismatch = 6; - errorpb.ServerIsBusy server_is_busy = 7; + errorpb.NotLeader not_leader = 1; + errorpb.RegionNotFound region_not_found = 2; + errorpb.EpochNotMatch epoch_not_match = 3; + DuplicateRequest duplicate_request = 4; + Compatibility compatibility = 5; + ClusterIDMismatch cluster_id_mismatch = 6; + errorpb.ServerIsBusy server_is_busy = 7; + Congested congested = 8; } message TxnInfo { - uint64 start_ts = 1; - bytes primary = 2; + uint64 start_ts = 1; + bytes primary = 2; } message TxnStatus { - uint64 start_ts = 1; - uint64 min_commit_ts = 2; - uint64 commit_ts = 3; - bool is_rolled_back = 4; + uint64 start_ts = 1; + uint64 min_commit_ts = 2; + uint64 commit_ts = 3; + bool is_rolled_back = 4; } message Event { - enum LogType { - UNKNOWN = 0; - PREWRITE = 1; - COMMIT = 2; - ROLLBACK = 3; - COMMITTED = 4; - INITIALIZED = 5; - } - - message Row { - uint64 start_ts = 1; - uint64 commit_ts = 2; - LogType type = 3; - - enum OpType { - UNKNOWN = 0; - PUT = 1; - DELETE = 2; - } - OpType op_type = 4; - bytes key = 5; - bytes value = 6; - bytes old_value = 7; - // expire_ts_unix_secs is used for RawKV (see `ChangeDataRequest.KvApi`), - // and represents the expiration time of this row. - // Absolute time, seconds since Unix epoch. - uint64 expire_ts_unix_secs = 8; - // The source of this row. - uint64 txn_source = 9; - } - - message Entries { - repeated Row entries = 1; - } - - message Admin { - raft_cmdpb.AdminRequest admin_request = 1; - raft_cmdpb.AdminResponse admin_response = 2; - } + enum LogType { + UNKNOWN = 0; + PREWRITE = 1; + COMMIT = 2; + ROLLBACK = 3; + COMMITTED = 4; + INITIALIZED = 5; + } + + message Row { + uint64 start_ts = 1; + uint64 commit_ts = 2; + LogType type = 3; - message LongTxn { - repeated TxnInfo txn_info = 1; - } + // generation is for pipelined DML protocol. See kvrpcpb.FlushRequest.generation. + uint64 generation = 10; - uint64 region_id = 1; - uint64 index = 2; - uint64 request_id = 7; - oneof event { - Entries entries = 3; - Admin admin = 4; - Error error = 5; - uint64 resolved_ts = 6 [deprecated=true]; - // Note that field 7 is taken by request_id. - LongTxn long_txn = 8; - // More region level events ... + enum OpType { + UNKNOWN = 0; + PUT = 1; + DELETE = 2; } + OpType op_type = 4; + + bytes key = 5; + bytes value = 6; + bytes old_value = 7; + // expire_ts_unix_secs is used for RawKV (see `ChangeDataRequest.KvApi`), + // and represents the expiration time of this row. + // Absolute time, seconds since Unix epoch. + uint64 expire_ts_unix_secs = 8; + // The source of this row. + uint64 txn_source = 9; + } + + message Entries { + repeated Row entries = 1; + } + + message Admin { + raft_cmdpb.AdminRequest admin_request = 1; + raft_cmdpb.AdminResponse admin_response = 2; + } + + message LongTxn { + repeated TxnInfo txn_info = 1; + } + + uint64 region_id = 1; + uint64 index = 2; + uint64 request_id = 7; + oneof event { + Entries entries = 3; + Admin admin = 4; + Error error = 5; + uint64 resolved_ts = 6 [deprecated = true]; + // Note that field 7 is taken by request_id. + LongTxn long_txn = 8; + // More region level events ... + } } // NOTE: events and resolved_ts won't appear simultaneously in one ChangeDataEvent. message ChangeDataEvent { - repeated Event events = 1; - ResolvedTs resolved_ts = 2; - // More store level events ... + repeated Event events = 1; + ResolvedTs resolved_ts = 2; + // More store level events ... } message ResolvedTs { - repeated uint64 regions = 1; - uint64 ts = 2; - uint64 request_id = 3; + repeated uint64 regions = 1; + uint64 ts = 2; + uint64 request_id = 3; } message ChangeDataRequest { - message Register {} + message Register {} - message Deregister {} + message Deregister {} - message NotifyTxnStatus { - repeated TxnStatus txn_status = 1; - } + message NotifyTxnStatus { + repeated TxnStatus txn_status = 1; + } - Header header = 1; - uint64 region_id = 2; - metapb.RegionEpoch region_epoch = 3; + Header header = 1; + uint64 region_id = 2; + metapb.RegionEpoch region_epoch = 3; - uint64 checkpoint_ts = 4; - bytes start_key = 5; - bytes end_key = 6; - // Used for CDC to identify events corresponding to different requests. - // Generally in one call stream, a region can be subscribed multiple - // times with different `request_id`s. - uint64 request_id = 7; + uint64 checkpoint_ts = 4; + bytes start_key = 5; + bytes end_key = 6; + // Used for CDC to identify events corresponding to different requests. + // Generally in one call stream, a region can be subscribed multiple + // times with different `request_id`s. + uint64 request_id = 7; - kvrpcpb.ExtraOp extra_op = 8; + kvrpcpb.ExtraOp extra_op = 8; - oneof request { - // A normal request that trying to register change data feed on a region. - Register register = 9; + oneof request { + // A normal request that trying to register change data feed on a region. + Register register = 9; - // Notify the region that some of the running transactions on the region has a pushed - // min_commit_ts so that the resolved_ts can be advanced. - NotifyTxnStatus notify_txn_status = 10; + // Notify the region that some of the running transactions on the region has a pushed + // min_commit_ts so that the resolved_ts can be advanced. + NotifyTxnStatus notify_txn_status = 10; - Deregister deregister = 13; - } + Deregister deregister = 13; + } - // KvAPI specifies to capture data written by different KV API. - // See more details in https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md. - enum KvAPI { - TiDB = 0; - RawKV = 1; - TxnKV = 2; - } + // KvAPI specifies to capture data written by different KV API. + // See more details in https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md. + enum KvAPI { + TiDB = 0; + RawKV = 1; + TxnKV = 2; + } - KvAPI kv_api = 11; + KvAPI kv_api = 11; - // Whether to filter out the value write by cdc itself. - bool filter_loop = 12; + // Whether to filter out the value write by cdc itself. + bool filter_loop = 12; } service ChangeData { - rpc EventFeed(stream ChangeDataRequest) returns(stream ChangeDataEvent); + rpc EventFeed(stream ChangeDataRequest) returns (stream ChangeDataEvent); - // EventFeedV2 is like EventFeed, with some new changes: - // * clients send requested features in HTTP/2 headers; - // * if servers meets unsupported feature request, - // it can fail the stream with an UNIMPLEMENTED error. - rpc EventFeedV2(stream ChangeDataRequest) returns (stream ChangeDataEvent); + // EventFeedV2 is like EventFeed, with some new changes: + // * clients send requested features in HTTP/2 headers; + // * if servers meets unsupported feature request, + // it can fail the stream with an UNIMPLEMENTED error. + rpc EventFeedV2(stream ChangeDataRequest) returns (stream ChangeDataEvent); } diff --git a/proto/configpb.proto b/proto/configpb.proto index 0276328c..951d7e98 100644 --- a/proto/configpb.proto +++ b/proto/configpb.proto @@ -2,47 +2,46 @@ syntax = "proto3"; package configpb; import "gogoproto/gogo.proto"; -import "rustproto.proto"; import "google/api/annotations.proto"; +import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; service Config { - rpc Create(CreateRequest) returns (CreateResponse) {} - rpc GetAll(GetAllRequest) returns (GetAllResponse) {} - rpc Get(GetRequest) returns (GetResponse) { - option (google.api.http) = { - get: "/component" - }; - } - rpc Update(UpdateRequest) returns (UpdateResponse) { - option (google.api.http) = { - post: "/component" - body: "*" - }; - } - rpc Delete(DeleteRequest) returns (DeleteResponse) { - option (google.api.http) = { - delete: "/component" - }; - } + rpc Create(CreateRequest) returns (CreateResponse) {} + rpc GetAll(GetAllRequest) returns (GetAllResponse) {} + rpc Get(GetRequest) returns (GetResponse) { + option (google.api.http) = {get: "/component"}; + } + rpc Update(UpdateRequest) returns (UpdateResponse) { + option (google.api.http) = { + post: "/component" + body: "*" + }; + } + rpc Delete(DeleteRequest) returns (DeleteResponse) { + option (google.api.http) = {delete: "/component"}; + } } enum StatusCode { - UNKNOWN = 0; - OK = 1; - WRONG_VERSION = 2; - NOT_CHANGE = 3; - COMPONENT_NOT_FOUND = 4; - COMPONENT_ID_NOT_FOUND = 5; + UNKNOWN = 0; + OK = 1; + WRONG_VERSION = 2; + NOT_CHANGE = 3; + COMPONENT_NOT_FOUND = 4; + COMPONENT_ID_NOT_FOUND = 5; } message Status { - StatusCode code = 1; - string message = 2; + StatusCode code = 1; + string message = 2; } // The version is used to tell the configuration which can be shared @@ -52,102 +51,102 @@ message Status { // For local version, every component will have one to represent // the version of these configuration which cannot be shared. message Version { - uint64 local = 1; - uint64 global = 2; + uint64 local = 1; + uint64 global = 2; } message Local { - string component_id = 1; + string component_id = 1; } message Global { - string component = 1; + string component = 1; } message ConfigKind { - oneof kind { - Local local = 1; - Global global = 2; - } + oneof kind { + Local local = 1; + Global global = 2; + } } message ConfigEntry { - string name = 1; - string value = 2; + string name = 1; + string value = 2; } message LocalConfig { - Version version = 1; - string component = 2; - string component_id = 3; - string config = 4; + Version version = 1; + string component = 2; + string component_id = 3; + string config = 4; } message Header { - uint64 cluster_id = 1; + uint64 cluster_id = 1; } message CreateRequest { - Header header = 1; - Version version = 2; - string component = 3; - string component_id = 4; - string config = 5; + Header header = 1; + Version version = 2; + string component = 3; + string component_id = 4; + string config = 5; } message CreateResponse { - Header header = 1; - Status status = 2; - Version version = 3; - string config = 4; + Header header = 1; + Status status = 2; + Version version = 3; + string config = 4; } message GetAllRequest { - Header header = 1; + Header header = 1; } message GetAllResponse { - Header header = 1; - Status status = 2; - repeated LocalConfig local_configs = 3; + Header header = 1; + Status status = 2; + repeated LocalConfig local_configs = 3; } message GetRequest { - Header header = 1; - Version version = 2; - string component = 3; - string component_id = 4; + Header header = 1; + Version version = 2; + string component = 3; + string component_id = 4; } message GetResponse { - Header header = 1; - Status status = 2; - Version version = 3; - string config = 4; + Header header = 1; + Status status = 2; + Version version = 3; + string config = 4; } message UpdateRequest { - Header header = 1; - Version version = 2; - ConfigKind kind = 3; - repeated ConfigEntry entries = 4; + Header header = 1; + Version version = 2; + ConfigKind kind = 3; + repeated ConfigEntry entries = 4; } message UpdateResponse { - Header header = 1; - Status status = 2; - Version version = 3; - string config = 4; + Header header = 1; + Status status = 2; + Version version = 3; + string config = 4; } message DeleteRequest { - Header header = 1; - Version version = 2; - ConfigKind kind = 3; + Header header = 1; + Version version = 2; + ConfigKind kind = 3; } message DeleteResponse { - Header header = 1; - Status status = 2; - Version version = 3; + Header header = 1; + Status status = 2; + Version version = 3; } diff --git a/proto/coprocessor.proto b/proto/coprocessor.proto index d0f83a0d..222f0e53 100644 --- a/proto/coprocessor.proto +++ b/proto/coprocessor.proto @@ -2,121 +2,220 @@ syntax = "proto3"; package coprocessor; import "errorpb.proto"; -import "kvrpcpb.proto"; import "gogoproto/gogo.proto"; -import "rustproto.proto"; +import "kvrpcpb.proto"; import "metapb.proto"; +import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - - // [start, end) message KeyRange { - bytes start = 1; - bytes end = 2; + bytes start = 1; + bytes end = 2; +} + +// KeyRange with an attached read_ts (version). +// It is used by TiCI versioned lookup. Callers must ensure `range` is a point range. +message VersionedKeyRange { + KeyRange range = 1; + uint64 read_ts = 2; } message Request { - kvrpcpb.Context context = 1; - int64 tp = 2; - bytes data = 3; - uint64 start_ts = 7; - repeated KeyRange ranges = 4; - - // If cache is enabled, TiKV returns cache hit instead of data if - // its last version matches this `cache_if_match_version`. - bool is_cache_enabled = 5; - uint64 cache_if_match_version = 6; - // Any schema-ful storage to validate schema correctness if necessary. - int64 schema_ver = 8; - bool is_trace_enabled = 9; - // paging_size is 0 when it's disabled, otherwise, it should be a positive number. - uint64 paging_size = 10; - // tasks stores the batched coprocessor tasks sent to the same tikv store. - repeated StoreBatchTask tasks = 11; - uint64 connection_id = 12; // This is the session id between a client and tidb - string connection_alias = 13; // This is the session alias between a client and tidb + kvrpcpb.Context context = 1; + int64 tp = 2; + bytes data = 3; + uint64 start_ts = 7; + repeated KeyRange ranges = 4; + + // If cache is enabled, TiKV returns cache hit instead of data if + // its last version matches this `cache_if_match_version`. + bool is_cache_enabled = 5; + uint64 cache_if_match_version = 6; + // Any schema-ful storage to validate schema correctness if necessary. + int64 schema_ver = 8; + bool is_trace_enabled = 9; + // paging_size is 0 when it's disabled, otherwise, it should be a positive number. + uint64 paging_size = 10; + // tasks stores the batched coprocessor tasks sent to the same tikv store. + repeated StoreBatchTask tasks = 11; + uint64 connection_id = 12; // This is the session id between a client and tidb + string connection_alias = 13; // This is the session alias between a client and tidb + + repeated TableShardInfos table_shard_infos = 14; // Shard infos for FTS index, used by TiFlash reading TiCI. + // Versioned point ranges for TiCI lookup. + // When `versioned_ranges` is non-empty, all `versioned_ranges[i].range` must be point range. + repeated VersionedKeyRange versioned_ranges = 15; + // max_keys_read is 0 when disabled, otherwise limits the number of storage + // engine keys scanned per coprocessor task. It is a per-task hard ceiling + // applied uniformly to any coprocessor request, used to bound worst-case + // scan amplification regardless of how the request is paginated. + uint64 max_keys_read = 16; + // paging_size_bytes is 0 when disabled, otherwise it should be a positive number. + // When set, within a paged coprocessor request, TiKV stops scanning the current + // page once accumulated scanned bytes reach this limit and returns the page + // boundary so the next page can resume from there. + // + // Unlike max_keys_read, which is a per-task hard ceiling on keys scanned, + // paging_size_bytes is an independent rate-control dimension intended for + // Resource Control: it bounds the byte volume of a single page so that RU + // pre-charging in PD's resource controller can be performed at byte + // granularity, complementing the existing row-count-based paging + // (paging_size). The two fields target different scenarios and may be set + // independently. + uint64 paging_size_bytes = 17; } message Response { - bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; - errorpb.Error region_error = 2; - kvrpcpb.LockInfo locked = 3; - string other_error = 4; - KeyRange range = 5; - - // This field is always filled for compatibility consideration. However - // newer TiDB should respect `exec_details_v2` field instead. - kvrpcpb.ExecDetails exec_details = 6; - // This field is provided in later versions, containing more detailed - // information. - kvrpcpb.ExecDetailsV2 exec_details_v2 = 11; + bytes data = 1 [ + (gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", + (gogoproto.nullable) = false + ]; + errorpb.Error region_error = 2; + kvrpcpb.LockInfo locked = 3; + string other_error = 4; + KeyRange range = 5; + + // This field is always filled for compatibility consideration. However + // newer TiDB should respect `exec_details_v2` field instead. + kvrpcpb.ExecDetails exec_details = 6; + // This field is provided in later versions, containing more detailed + // information. + kvrpcpb.ExecDetailsV2 exec_details_v2 = 11; + + bool is_cache_hit = 7; + uint64 cache_last_version = 8; + bool can_be_cached = 9; + + reserved 10; + + // Contains the latest buckets version of the region. + // Clients should query PD to update buckets in cache if its is stale. + uint64 latest_buckets_version = 12; + + // StoreBatchTaskResponse is the collection of batch task responses. + repeated StoreBatchTaskResponse batch_responses = 13; +} - bool is_cache_hit = 7; - uint64 cache_last_version = 8; - bool can_be_cached = 9; +message RegionInfo { + uint64 region_id = 1; + metapb.RegionEpoch region_epoch = 2; + repeated KeyRange ranges = 3; +} - reserved 10; +message ShardInfo { + uint64 shard_id = 1; + uint64 shard_epoch = 2; + repeated KeyRange ranges = 3; +} - // Contains the latest buckets version of the region. - // Clients should query PD to update buckets in cache if its is stale. - uint64 latest_buckets_version = 12; +message TableShardInfos { + // The executor ID is used to identify the tici executor. + string executor_id = 1; + // The shard_infos contains the shard information for each tici executor. + repeated ShardInfo shard_infos = 2; +} - // StoreBatchTaskResponse is the collection of batch task responses. - repeated StoreBatchTaskResponse batch_responses = 13; +message TiCIEstimateCountRequest { + kvrpcpb.Context context = 1; + uint64 start_ts = 2; + int64 table_id = 3; + int64 index_id = 4; + bytes fts_query_info = 5; + string time_zone_name = 6; + int64 time_zone_offset = 7; + repeated ShardInfo shard_infos = 8; } -message RegionInfo { - uint64 region_id = 1; - metapb.RegionEpoch region_epoch = 2; - repeated KeyRange ranges = 3; +message TiCIEstimateCountResponse { + uint64 est_count = 1; + string other_error = 2; } message TableRegions { - int64 physical_table_id = 1; - repeated RegionInfo regions = 2; + int64 physical_table_id = 1; + repeated RegionInfo regions = 2; } message BatchRequest { - kvrpcpb.Context context = 1; - int64 tp = 2; - bytes data = 3; - repeated RegionInfo regions = 4; - uint64 start_ts = 5; - // Any schema-ful storage to validate schema correctness if necessary. - int64 schema_ver = 6; - // Used for partition table scan - repeated TableRegions table_regions = 7; - string log_id = 8; - uint64 connection_id = 9; // This is the session id between a client and tidb - string connection_alias = 10; // This is the session alias between a client and tidb + kvrpcpb.Context context = 1; + int64 tp = 2; + bytes data = 3; + repeated RegionInfo regions = 4; + uint64 start_ts = 5; + // Any schema-ful storage to validate schema correctness if necessary. + int64 schema_ver = 6; + // Used for partition table scan + repeated TableRegions table_regions = 7; + string log_id = 8; + uint64 connection_id = 9; // This is the session id between a client and tidb + string connection_alias = 10; // This is the session alias between a client and tidb + repeated TableShardInfos table_shard_infos = 11; // Shard infos for FTS index, used by TiFlash reading TiCI. } message BatchResponse { - bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; - string other_error = 2; - kvrpcpb.ExecDetails exec_details = 3; - repeated metapb.Region retry_regions = 4; + bytes data = 1 [ + (gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", + (gogoproto.nullable) = false + ]; + string other_error = 2; + kvrpcpb.ExecDetails exec_details = 3; + repeated metapb.Region retry_regions = 4; + repeated ShardInfo retry_shards = 5; } message StoreBatchTask { - uint64 region_id = 1; - metapb.RegionEpoch region_epoch = 2; - metapb.Peer peer = 3; - repeated KeyRange ranges = 4; - uint64 task_id = 5; + uint64 region_id = 1; + metapb.RegionEpoch region_epoch = 2; + metapb.Peer peer = 3; + repeated KeyRange ranges = 4; + uint64 task_id = 5; + // Versioned point ranges for TiCI lookup. + // When `versioned_ranges` is non-empty, all `versioned_ranges[i].range` must be point range. + repeated VersionedKeyRange versioned_ranges = 6; + // Bucket metadata version used to validate this child task. + uint64 buckets_version = 7; } message StoreBatchTaskResponse { - bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; - errorpb.Error region_error = 2; - kvrpcpb.LockInfo locked = 3; - string other_error = 4; - uint64 task_id = 5; - kvrpcpb.ExecDetailsV2 exec_details_v2 = 6; + bytes data = 1 [ + (gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", + (gogoproto.nullable) = false + ]; + errorpb.Error region_error = 2; + kvrpcpb.LockInfo locked = 3; + string other_error = 4; + uint64 task_id = 5; + kvrpcpb.ExecDetailsV2 exec_details_v2 = 6; +} + +message DelegateRequest { + kvrpcpb.Context context = 1; + uint64 start_ts = 2; + repeated KeyRange ranges = 3; + // Used for avoid redundant mem-table copying. + // If the sequence is the same, tikv-server will not return the mem-table. + uint64 mem_table_sequence = 4; + + // Used for avoid redundant snapshot copying. + // If the sequence is the same, tikv-server will not return the snapshot. + uint64 snapshot_sequence = 5; +} + +message DelegateResponse { + bytes mem_table_data = 1; + bytes snapshot = 2; + errorpb.Error region_error = 3; + kvrpcpb.LockInfo locked = 4; + string other_error = 5; + // Used for avoid redundant mem-table copying. + uint64 mem_table_sequence = 6; } diff --git a/proto/deadlock.proto b/proto/deadlock.proto index da66a8c2..056c23cd 100644 --- a/proto/deadlock.proto +++ b/proto/deadlock.proto @@ -2,59 +2,81 @@ syntax = "proto3"; package deadlock; import "gogoproto/gogo.proto"; +import "rustproto.proto"; -message WaitForEntriesRequest { -} +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; + +message WaitForEntriesRequest {} message WaitForEntriesResponse { - repeated WaitForEntry entries = 1 [(gogoproto.nullable) = false]; + repeated WaitForEntry entries = 1 [(gogoproto.nullable) = false]; } message WaitForEntry { - // The transaction id that is waiting. - uint64 txn = 1; - // The transaction id that is being waited for. - uint64 wait_for_txn = 2; - // The hash value of the key is being waited for. - uint64 key_hash = 3; - // The key the current txn is trying to lock. - bytes key = 4; - // The tag came from the lock request's context. - bytes resource_group_tag = 5; - // Milliseconds it has been waits. - uint64 wait_time = 6; + // The transaction id that is waiting. + uint64 txn = 1; + // The transaction id that is being waited for. + uint64 wait_for_txn = 2; + // The hash value of the key is being waited for. + uint64 key_hash = 3; + // The key the current txn is trying to lock. + bytes key = 4; + // The tag came from the lock request's context. + bytes resource_group_tag = 5; + // Milliseconds it has been waits. + uint64 wait_time = 6; } enum DeadlockRequestType { - Detect = 0; - // CleanUpWaitFor cleans a single entry the transaction is waiting. - CleanUpWaitFor = 1; - // CleanUp cleans all entries the transaction is waiting. - CleanUp = 2; + Detect = 0; + // CleanUpWaitFor cleans a single entry the transaction is waiting. + CleanUpWaitFor = 1; + // CleanUp cleans all entries the transaction is waiting. + CleanUp = 2; +} + +message ReplaceLockByKeyItem { + uint64 key_hash = 1; + bytes key = 2; + uint64 old_lock_ts = 3; + uint64 new_lock_ts = 4; +} + +message ReplaceLocksByKeysRequest { + repeated ReplaceLockByKeyItem items = 1; } message DeadlockRequest { - DeadlockRequestType tp = 1; - WaitForEntry entry = 2 [(gogoproto.nullable) = false]; + DeadlockRequestType tp = 1; + WaitForEntry entry = 2 [(gogoproto.nullable) = false]; + ReplaceLocksByKeysRequest replace_locks_by_keys = 3; } message DeadlockResponse { - // The same entry sent by DeadlockRequest, identifies the sender. - WaitForEntry entry = 1 [(gogoproto.nullable) = false]; - // The key hash of the lock that is hold by the waiting transaction. - uint64 deadlock_key_hash = 2; - // The other entries of the dead lock circle. The current entry is in `entry` field and not - // included in this field. - repeated WaitForEntry wait_chain = 3; + // The same entry sent by DeadlockRequest, identifies the sender. + WaitForEntry entry = 1 [(gogoproto.nullable) = false]; + // The key hash of the lock that is hold by the waiting transaction. The hash of the `deadlock_key` field. + uint64 deadlock_key_hash = 2; + // The other entries of the dead lock circle. The current entry is in `entry` field and not + // included in this field. + repeated WaitForEntry wait_chain = 3; + // The key of the lock that is hold by the waiting transaction. + bytes deadlock_key = 4; } service Deadlock { - // Get local wait for entries, should be handle by every node. - // The owner should sent this request to all members to build the complete wait for graph. - rpc GetWaitForEntries(WaitForEntriesRequest) returns (WaitForEntriesResponse) {} - - // Detect should only sent to the owner. only be handled by the owner. - // The DeadlockResponse is sent back only if there is deadlock detected. - // CleanUpWaitFor and CleanUp doesn't return responses. - rpc Detect(stream DeadlockRequest) returns (stream DeadlockResponse) {} + // Get local wait for entries, should be handle by every node. + // The owner should sent this request to all members to build the complete wait for graph. + rpc GetWaitForEntries(WaitForEntriesRequest) returns (WaitForEntriesResponse) {} + + // Detect should only sent to the owner. only be handled by the owner. + // The DeadlockResponse is sent back only if there is deadlock detected. + // CleanUpWaitFor and CleanUp doesn't return responses. + rpc Detect(stream DeadlockRequest) returns (stream DeadlockResponse) {} } diff --git a/proto/debugpb.proto b/proto/debugpb.proto index dc612d14..65b3d492 100644 --- a/proto/debugpb.proto +++ b/proto/debugpb.proto @@ -2,18 +2,20 @@ syntax = "proto3"; package debugpb; import "eraftpb.proto"; +import "gogoproto/gogo.proto"; import "kvrpcpb.proto"; import "raft_serverpb.proto"; -import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // Debug service for TiKV. // // Errors are defined as follow: @@ -24,318 +26,308 @@ option java_package = "org.tikv.kvproto"; // reason can be found in grpc message. // Note: It bypasses raft layer. service Debug { - // Read a value arbitrarily for a key. - // Note: Server uses key directly w/o any encoding. - rpc Get(GetRequest) returns (GetResponse) {} + // Read a value arbitrarily for a key. + // Note: Server uses key directly w/o any encoding. + rpc Get(GetRequest) returns (GetResponse) {} - // Read raft info. - rpc RaftLog(RaftLogRequest) returns (RaftLogResponse) {} - rpc RegionInfo(RegionInfoRequest) returns (RegionInfoResponse) {} + // Read raft info. + rpc RaftLog(RaftLogRequest) returns (RaftLogResponse) {} + rpc RegionInfo(RegionInfoRequest) returns (RegionInfoResponse) {} - // Calculate size of a region. - // Note: DO NOT CALL IT IN PRODUCTION, it's really expensive. - rpc RegionSize(RegionSizeRequest) returns (RegionSizeResponse) {} + // Calculate size of a region. + // Note: DO NOT CALL IT IN PRODUCTION, it's really expensive. + rpc RegionSize(RegionSizeRequest) returns (RegionSizeResponse) {} - // Scan a specific range. - // Note: DO NOT CALL IT IN PRODUCTION, it's really expensive. - // Server uses keys directly w/o any encoding. - rpc ScanMvcc(ScanMvccRequest) returns (stream ScanMvccResponse) {} + // Scan a specific range. + // Note: DO NOT CALL IT IN PRODUCTION, it's really expensive. + // Server uses keys directly w/o any encoding. + rpc ScanMvcc(ScanMvccRequest) returns (stream ScanMvccResponse) {} - // Compact a column family in a specified range. - // Note: Server uses keys directly w/o any encoding. - rpc Compact(CompactRequest) returns (CompactResponse) {} + // Compact a column family in a specified range. + // Note: Server uses keys directly w/o any encoding. + rpc Compact(CompactRequest) returns (CompactResponse) {} - // Inject a fail point. Currently, it's only used in tests. - // Note: DO NOT CALL IT IN PRODUCTION. - rpc InjectFailPoint(InjectFailPointRequest) returns (InjectFailPointResponse) {} - // Recover from a fail point. - rpc RecoverFailPoint(RecoverFailPointRequest) returns (RecoverFailPointResponse) {} - // List all fail points. - rpc ListFailPoints(ListFailPointsRequest) returns (ListFailPointsResponse) {} + // Inject a fail point. Currently, it's only used in tests. + // Note: DO NOT CALL IT IN PRODUCTION. + rpc InjectFailPoint(InjectFailPointRequest) returns (InjectFailPointResponse) {} + // Recover from a fail point. + rpc RecoverFailPoint(RecoverFailPointRequest) returns (RecoverFailPointResponse) {} + // List all fail points. + rpc ListFailPoints(ListFailPointsRequest) returns (ListFailPointsResponse) {} - // Get Metrics - rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse){} + // Get Metrics + rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} - // Do a consistent check for a region. - rpc CheckRegionConsistency(RegionConsistencyCheckRequest) returns (RegionConsistencyCheckResponse) {} + // Do a consistent check for a region. + rpc CheckRegionConsistency(RegionConsistencyCheckRequest) returns (RegionConsistencyCheckResponse) {} - // dynamically modify tikv's config - rpc ModifyTikvConfig(ModifyTikvConfigRequest) returns (ModifyTikvConfigResponse) {} + // dynamically modify tikv's config + rpc ModifyTikvConfig(ModifyTikvConfigRequest) returns (ModifyTikvConfigResponse) {} - // Get region properties - rpc GetRegionProperties(GetRegionPropertiesRequest) returns (GetRegionPropertiesResponse) {} + // Get region properties + rpc GetRegionProperties(GetRegionPropertiesRequest) returns (GetRegionPropertiesResponse) {} - // Get store ID - rpc GetStoreInfo(GetStoreInfoRequest) returns (GetStoreInfoResponse) {} + // Get store ID + rpc GetStoreInfo(GetStoreInfoRequest) returns (GetStoreInfoResponse) {} - // Get cluster ID - rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {} + // Get cluster ID + rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {} - // Get all region IDs in the store - rpc GetAllRegionsInStore(GetAllRegionsInStoreRequest) returns (GetAllRegionsInStoreResponse) {} + // Get all region IDs in the store + rpc GetAllRegionsInStore(GetAllRegionsInStoreRequest) returns (GetAllRegionsInStoreResponse) {} - // Make this TiKV node return to the status on this node to certain ts. - rpc ResetToVersion(ResetToVersionRequest) returns (ResetToVersionResponse) {} + // Make this TiKV node return to the status on this node to certain ts. + rpc ResetToVersion(ResetToVersionRequest) returns (ResetToVersionResponse) {} - // Get range properties - rpc GetRangeProperties(GetRangePropertiesRequest) returns (GetRangePropertiesResponse) {} + // Get range properties + rpc GetRangeProperties(GetRangePropertiesRequest) returns (GetRangePropertiesResponse) {} - // Flashback given key range to a specified version. - rpc FlashbackToVersion(FlashbackToVersionRequest) returns (FlashbackToVersionResponse) {} - - // GetRegionReadProgress returns the some useful info in RegionReadProgress - rpc GetRegionReadProgress(GetRegionReadProgressRequest) returns (GetRegionReadProgressResponse) {} + // Flashback given key range to a specified version. + rpc FlashbackToVersion(FlashbackToVersionRequest) returns (FlashbackToVersionResponse) {} + + // GetRegionReadProgress returns the some useful info in RegionReadProgress + rpc GetRegionReadProgress(GetRegionReadProgressRequest) returns (GetRegionReadProgressResponse) {} } enum DB { - INVALID = 0; - KV = 1; - RAFT = 2; + INVALID = 0; + KV = 1; + RAFT = 2; } enum MODULE { - UNUSED = 0; - KVDB = 1; - RAFTDB = 2; - READPOOL = 3; - SERVER = 4; - STORAGE = 5; - PD = 6; - METRIC = 7; - COPROCESSOR = 8; - SECURITY = 9; - IMPORT = 10; + UNUSED = 0; + KVDB = 1; + RAFTDB = 2; + READPOOL = 3; + SERVER = 4; + STORAGE = 5; + PD = 6; + METRIC = 7; + COPROCESSOR = 8; + SECURITY = 9; + IMPORT = 10; } message GetRequest { - DB db = 1; - string cf = 2; - bytes key = 3; + DB db = 1; + string cf = 2; + bytes key = 3; } message GetResponse { - bytes value = 1; + bytes value = 1; } message RaftLogRequest { - uint64 region_id = 1; - uint64 log_index = 2; + uint64 region_id = 1; + uint64 log_index = 2; } message RaftLogResponse { - eraftpb.Entry entry = 1; + eraftpb.Entry entry = 1; } message RegionInfoRequest { - uint64 region_id = 1; + uint64 region_id = 1; } message RegionInfoResponse { - raft_serverpb.RaftLocalState raft_local_state = 1; - raft_serverpb.RaftApplyState raft_apply_state = 2; - raft_serverpb.RegionLocalState region_local_state = 3; + raft_serverpb.RaftLocalState raft_local_state = 1; + raft_serverpb.RaftApplyState raft_apply_state = 2; + raft_serverpb.RegionLocalState region_local_state = 3; } message RegionSizeRequest { - uint64 region_id = 1; - repeated string cfs = 2; + uint64 region_id = 1; + repeated string cfs = 2; } message RegionSizeResponse { - message Entry { - string cf = 1; - uint64 size = 2; - } + message Entry { + string cf = 1; + uint64 size = 2; + } - repeated Entry entries = 1; + repeated Entry entries = 1; } message ScanMvccRequest { - bytes from_key = 1; - bytes to_key = 2; - uint64 limit = 3; + bytes from_key = 1; + bytes to_key = 2; + uint64 limit = 3; } message ScanMvccResponse { - bytes key = 1; - kvrpcpb.MvccInfo info = 2; + bytes key = 1; + kvrpcpb.MvccInfo info = 2; } enum BottommostLevelCompaction { - // Skip bottommost level compaction - Skip = 0; - // Force bottommost level compaction - Force = 1; - // Compact bottommost level if there is a compaction filter. - IfHaveCompactionFilter = 2; + // Skip bottommost level compaction + Skip = 0; + // Force bottommost level compaction + Force = 1; + // Compact bottommost level if there is a compaction filter. + IfHaveCompactionFilter = 2; } message CompactRequest { - DB db = 1; - string cf = 2; - bytes from_key = 3; - bytes to_key = 4; - uint32 threads = 5; - BottommostLevelCompaction bottommost_level_compaction = 6; + DB db = 1; + string cf = 2; + bytes from_key = 3; + bytes to_key = 4; + uint32 threads = 5; + BottommostLevelCompaction bottommost_level_compaction = 6; } -message CompactResponse { -} +message CompactResponse {} message InjectFailPointRequest { - string name = 1; - string actions = 2; + string name = 1; + string actions = 2; } -message InjectFailPointResponse { -} +message InjectFailPointResponse {} message RecoverFailPointRequest { - string name = 1; + string name = 1; } -message RecoverFailPointResponse { -} +message RecoverFailPointResponse {} -message ListFailPointsRequest { -} +message ListFailPointsRequest {} message ListFailPointsResponse { - message Entry { - string name = 1; - string actions = 2; - } + message Entry { + string name = 1; + string actions = 2; + } - repeated Entry entries = 1; + repeated Entry entries = 1; } message GetMetricsRequest { - bool all = 1; + bool all = 1; } message GetMetricsResponse { - string prometheus = 1; - string rocksdb_kv = 2; - string rocksdb_raft = 3; - string jemalloc = 4; - uint64 store_id = 5; + string prometheus = 1; + string rocksdb_kv = 2; + string rocksdb_raft = 3; + string jemalloc = 4; + uint64 store_id = 5; } message RegionConsistencyCheckRequest { - uint64 region_id = 1; + uint64 region_id = 1; } -message RegionConsistencyCheckResponse { -} +message RegionConsistencyCheckResponse {} message ModifyTikvConfigRequest { - MODULE module = 1; - string config_name = 2; - string config_value = 3; + MODULE module = 1; + string config_name = 2; + string config_value = 3; } -message ModifyTikvConfigResponse { -} +message ModifyTikvConfigResponse {} message Property { - string name = 1; - string value = 2; + string name = 1; + string value = 2; } message GetRegionPropertiesRequest { - uint64 region_id = 1; + uint64 region_id = 1; } message GetRegionPropertiesResponse { - repeated Property props = 1; + repeated Property props = 1; } -message GetStoreInfoRequest { -} +message GetStoreInfoRequest {} message GetStoreInfoResponse { - uint64 store_id = 1; - kvrpcpb.APIVersion api_version = 2; + uint64 store_id = 1; + kvrpcpb.APIVersion api_version = 2; } -message GetClusterInfoRequest { -} +message GetClusterInfoRequest {} message GetClusterInfoResponse { - uint64 cluster_id = 1; + uint64 cluster_id = 1; } -message GetAllRegionsInStoreRequest { -} +message GetAllRegionsInStoreRequest {} message GetAllRegionsInStoreResponse { - repeated uint64 regions = 1; + repeated uint64 regions = 1; } message ResetToVersionRequest { - uint64 ts = 1; + uint64 ts = 1; } -message ResetToVersionResponse { -} +message ResetToVersionResponse {} message GetRangePropertiesRequest { - bytes start_key = 1; - bytes end_key = 2; + bytes start_key = 1; + bytes end_key = 2; } message GetRangePropertiesResponse { - message RangeProperty { - string key = 1; - string value = 2; - } + message RangeProperty { + string key = 1; + string value = 2; + } - repeated RangeProperty properties = 1; + repeated RangeProperty properties = 1; } message FlashbackToVersionRequest { - kvrpcpb.Context context = 1; - uint64 version = 2; - uint64 region_id = 3; - bytes start_key = 4; - bytes end_key = 5; - uint64 start_ts = 6; - uint64 commit_ts = 7; + kvrpcpb.Context context = 1; + uint64 version = 2; + uint64 region_id = 3; + bytes start_key = 4; + bytes end_key = 5; + uint64 start_ts = 6; + uint64 commit_ts = 7; } message FlashbackToVersionResponse { - string error = 1; + string error = 1; } message GetRegionReadProgressRequest { - uint64 region_id = 1; - bool log_locks = 2; // when set to true, print a log of the locks with min start_ts in the resolver. - uint64 min_start_ts = 3; // only print locks whose start_ts >= min_start_ts. Can be used to find certain transaction. + uint64 region_id = 1; + bool log_locks = 2; // when set to true, print a log of the locks with min start_ts in the resolver. + uint64 min_start_ts = 3; // only print locks whose start_ts >= min_start_ts. Can be used to find certain transaction. } message GetRegionReadProgressResponse { - // below are from region_read_progress module - uint64 safe_ts = 1; - uint64 applied_index = 2; - uint64 pending_front_applied_index = 3; - uint64 pending_front_ts = 4; - uint64 pending_back_applied_index = 5; - uint64 pending_back_ts = 6; - bool region_read_progress_paused = 7; - uint64 duration_to_last_update_safe_ts_ms = 8; - uint64 duration_to_last_consume_leader_ms = 9; - bool region_read_progress_exist = 10; - uint64 read_state_ts = 18; - uint64 read_state_apply_index = 19; - bool discard = 20; - - // below are from resolved-ts module - uint64 resolved_ts = 11; - uint64 resolver_tracked_index = 12; - bool resolver_exist = 13; - bool resolver_stopped = 14; - uint64 num_locks = 16; - uint64 num_transactions = 17; - - string error = 15; -} \ No newline at end of file + // below are from region_read_progress module + uint64 safe_ts = 1; + uint64 applied_index = 2; + uint64 pending_front_applied_index = 3; + uint64 pending_front_ts = 4; + uint64 pending_back_applied_index = 5; + uint64 pending_back_ts = 6; + bool region_read_progress_paused = 7; + uint64 duration_to_last_update_safe_ts_ms = 8; + uint64 duration_to_last_consume_leader_ms = 9; + bool region_read_progress_exist = 10; + uint64 read_state_ts = 18; + uint64 read_state_apply_index = 19; + bool discard = 20; + + // below are from resolved-ts module + uint64 resolved_ts = 11; + uint64 resolver_tracked_index = 12; + bool resolver_exist = 13; + bool resolver_stopped = 14; + uint64 num_locks = 16; + uint64 num_transactions = 17; + + string error = 15; +} diff --git a/proto/diagnosticspb.proto b/proto/diagnosticspb.proto index 548b63ad..dc63d936 100644 --- a/proto/diagnosticspb.proto +++ b/proto/diagnosticspb.proto @@ -4,89 +4,91 @@ package diagnosticspb; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // Diagnostics service for TiDB cluster components. service Diagnostics { - // Searchs log in the target node - rpc search_log(SearchLogRequest) returns (stream SearchLogResponse) {}; - // Retrieves server info in the target node - rpc server_info(ServerInfoRequest) returns (ServerInfoResponse) {}; + // Searchs log in the target node + rpc search_log(SearchLogRequest) returns (stream SearchLogResponse) {} + // Retrieves server info in the target node + rpc server_info(ServerInfoRequest) returns (ServerInfoResponse) {} } enum LogLevel { - UNKNOWN = 0; - Debug = 1; - Info = 2; - Warn = 3; - Trace = 4; - Critical = 5; - Error = 6; + UNKNOWN = 0; + Debug = 1; + Info = 2; + Warn = 3; + Trace = 4; + Critical = 5; + Error = 6; } message SearchLogRequest { - enum Target { - Normal = 0; - Slow = 1; - } - int64 start_time = 1; - int64 end_time = 2; - repeated LogLevel levels = 3; - // We use a string array to represent multiple CNF pattern sceniaor like: - // SELECT * FROM t WHERE c LIKE '%s%' and c REGEXP '.*a.*' because - // Golang and Rust don't support perl-like (?=re1)(?=re2) - repeated string patterns = 4; - Target target = 5; + enum Target { + Normal = 0; + Slow = 1; + } + int64 start_time = 1; + int64 end_time = 2; + repeated LogLevel levels = 3; + // We use a string array to represent multiple CNF pattern sceniaor like: + // SELECT * FROM t WHERE c LIKE '%s%' and c REGEXP '.*a.*' because + // Golang and Rust don't support perl-like (?=re1)(?=re2) + repeated string patterns = 4; + Target target = 5; } message SearchLogResponse { - repeated LogMessage messages = 1; + repeated LogMessage messages = 1; } message LogMessage { - int64 time = 1; - LogLevel level = 2; - string message = 3; + int64 time = 1; + LogLevel level = 2; + string message = 3; } enum ServerInfoType { - All = 0; - HardwareInfo = 1; - SystemInfo = 2; - LoadInfo = 3; + All = 0; + HardwareInfo = 1; + SystemInfo = 2; + LoadInfo = 3; } message ServerInfoRequest { - ServerInfoType tp = 1; + ServerInfoType tp = 1; } message ServerInfoPair { - string key = 1; - string value = 2; + string key = 1; + string value = 2; } message ServerInfoItem { - // cpu, memory, disk, network ... - string tp = 1; - // eg. network: lo1/eth0, cpu: core1/core2, disk: sda1/sda2 - string name = 2; - // all key-value pairs for specified item, e.g: - // ServerInfoItem { - // tp = "network" - // name = "eth0" - // paris = [ - // ServerInfoPair { key = "readbytes", value = "4k"}, - // ServerInfoPair { key = "writebytes", value = "1k"}, - // ] - // } - repeated ServerInfoPair pairs = 3; + // cpu, memory, disk, network ... + string tp = 1; + // eg. network: lo1/eth0, cpu: core1/core2, disk: sda1/sda2 + string name = 2; + // all key-value pairs for specified item, e.g: + // ServerInfoItem { + // tp = "network" + // name = "eth0" + // paris = [ + // ServerInfoPair { key = "readbytes", value = "4k"}, + // ServerInfoPair { key = "writebytes", value = "1k"}, + // ] + // } + repeated ServerInfoPair pairs = 3; } message ServerInfoResponse { - repeated ServerInfoItem items = 1; + repeated ServerInfoItem items = 1; } diff --git a/proto/disaggregated.proto b/proto/disaggregated.proto index adaa8f61..445e8ffd 100644 --- a/proto/disaggregated.proto +++ b/proto/disaggregated.proto @@ -2,168 +2,177 @@ syntax = "proto3"; package disaggregated; import "coprocessor.proto"; +import "gogoproto/gogo.proto"; import "kvrpcpb.proto"; +import "rustproto.proto"; option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; /// For S3 lock service /// message S3LockResult { - oneof error { - Success success = 1; - NotOwner not_owner = 2; - Conflict conflict = 3; - } + oneof error { + Success success = 1; + NotOwner not_owner = 2; + Conflict conflict = 3; + } } message Success {} + // Error caused by S3GC owner changed // client should retry -message NotOwner{ -} +message NotOwner {} + // Error caused by concurrency conflict, // request cancel message Conflict { - string reason = 1; + string reason = 1; } message TryAddLockRequest { - // The data file key to add lock - bytes data_file_key = 1; - // The lock store id - uint64 lock_store_id = 3; - // The upload sequence number of lock store - uint64 lock_seq = 4; + // The data file key to add lock + bytes data_file_key = 1; + // The lock store id + uint64 lock_store_id = 3; + // The upload sequence number of lock store + uint64 lock_seq = 4; } message TryAddLockResponse { - S3LockResult result = 1; + S3LockResult result = 1; } message TryMarkDeleteRequest { - // The data file key to be marked as deleted - bytes data_file_key = 1; + // The data file key to be marked as deleted + bytes data_file_key = 1; } message TryMarkDeleteResponse { - S3LockResult result = 1; + S3LockResult result = 1; } /// For disagg compute node init its disagg configuration /// -message GetDisaggConfigRequest { -} +message GetDisaggConfigRequest {} message DisaggS3Config { - string bucket = 1; - string root = 2; - string endpoint = 3; + string bucket = 1; + string root = 2; + string endpoint = 3; } message GetDisaggConfigResponse { - DisaggS3Config s3_config = 1; + DisaggS3Config s3_config = 1; } /// For compute task dispatch and data exchange /// message DisaggTaskMeta { - uint64 start_ts = 1; // start ts of a query - // gather_id + query_ts + server_id + local_query_id to represent a global unique query. - int64 gather_id = 9; // used to distinguish different gathers in the mpp query - uint64 query_ts = 2; // timestamp when start to execute query, used for TiFlash miniTSO schedule. - uint64 server_id = 3; // TiDB server id - uint64 local_query_id = 4; // unique local query_id if tidb don't restart. - int64 task_id = 5; // if task id is -1 , it indicates a tidb task. - string executor_id = 6; // the exectuor id - uint32 keyspace_id = 7; // keyspace id of the request - kvrpcpb.APIVersion api_version = 8; // API version of the request - uint64 connection_id = 10; // This is the session id between a client and tidb - string connection_alias = 11; // This is the session alias between a client and tidb + uint64 start_ts = 1; // start ts of a query + // gather_id + query_ts + server_id + local_query_id to represent a global unique query. + int64 gather_id = 9; // used to distinguish different gathers in the mpp query + uint64 query_ts = 2; // timestamp when start to execute query, used for TiFlash miniTSO schedule. + uint64 server_id = 3; // TiDB server id + uint64 local_query_id = 4; // unique local query_id if tidb don't restart. + int64 task_id = 5; // if task id is -1 , it indicates a tidb task. + string executor_id = 6; // the exectuor id + uint32 keyspace_id = 7; // keyspace id of the request + kvrpcpb.APIVersion api_version = 8; // API version of the request + uint64 connection_id = 10; // This is the session id between a client and tidb + string connection_alias = 11; // This is the session alias between a client and tidb } message DisaggReadError { - int32 code = 1; - string msg = 2; + int32 code = 1; + string msg = 2; } message EstablishDisaggTaskError { - oneof errors { - ErrorRegion error_region = 1; - ErrorLocked error_locked = 2; + oneof errors { + ErrorRegion error_region = 1; + ErrorLocked error_locked = 2; - ErrorOther error_other = 99; - } + ErrorOther error_other = 99; + } } message ErrorRegion { - string msg = 1; - // The read node needs to update its region cache about these regions. - repeated uint64 region_ids = 2; + string msg = 1; + // The read node needs to update its region cache about these regions. + repeated uint64 region_ids = 2; } message ErrorLocked { - string msg = 1; - // The read node needs to resolve these locks. - repeated kvrpcpb.LockInfo locked = 2; + string msg = 1; + // The read node needs to resolve these locks. + repeated kvrpcpb.LockInfo locked = 2; } message ErrorOther { - int32 code = 1; - string msg = 2; + int32 code = 1; + string msg = 2; } message EstablishDisaggTaskRequest { - DisaggTaskMeta meta = 1; - string address = 2; // target address of this task. - // The write node needs to ensure that subsequent - // FetchDisaggPagesRequest can be processed within timeout_s. - // unit: seconds - int64 timeout_s = 3; - // The key ranges, Region meta that read node need to execute TableScan - repeated coprocessor.RegionInfo regions = 4; - int64 schema_ver = 5; - // Used for PartitionTableScan - repeated coprocessor.TableRegions table_regions = 6; - // The encoded TableScan/PartitionTableScan + Selection. - bytes encoded_plan = 7; + DisaggTaskMeta meta = 1; + string address = 2; // target address of this task. + // The write node needs to ensure that subsequent + // FetchDisaggPagesRequest can be processed within timeout_s. + // unit: seconds + int64 timeout_s = 3; + // The key ranges, Region meta that read node need to execute TableScan + repeated coprocessor.RegionInfo regions = 4; + int64 schema_ver = 5; + // Used for PartitionTableScan + repeated coprocessor.TableRegions table_regions = 6; + // The encoded TableScan/PartitionTableScan + Selection. + bytes encoded_plan = 7; } message EstablishDisaggTaskResponse { - EstablishDisaggTaskError error = 1; + EstablishDisaggTaskError error = 1; - // Write node maintains a snapshot with a lease time. - // Read node should read the delta pages - // (ColumnFileInMemory and ColumnFileTiny) - // along with this store_id and snapshot_id. - uint64 store_id = 3; // metapb.Store.id - DisaggTaskMeta snapshot_id = 4; - // Serialized disaggregated tasks (per physical table) - repeated bytes tables = 5; + // Write node maintains a snapshot with a lease time. + // Read node should read the delta pages + // (ColumnFileInMemory and ColumnFileTiny) + // along with this store_id and snapshot_id. + uint64 store_id = 3; // metapb.Store.id + DisaggTaskMeta snapshot_id = 4; + // Serialized disaggregated tasks (per physical table) + repeated bytes tables = 5; } message CancelDisaggTaskRequest { - DisaggTaskMeta meta = 1; + DisaggTaskMeta meta = 1; } message CancelDisaggTaskResponse {} message FetchDisaggPagesRequest { - // The snapshot id to fetch pages - DisaggTaskMeta snapshot_id = 1; - int64 table_id = 2; - uint64 segment_id = 3; - // It must be a subset of the delta pages ids returned - // in EstablishDisaggTaskResponse.segments - repeated uint64 page_ids = 4; + // The snapshot id to fetch pages + DisaggTaskMeta snapshot_id = 1; + int64 table_id = 2; + uint64 segment_id = 3; + // It must be a subset of the delta pages ids returned + // in EstablishDisaggTaskResponse.segments + repeated uint64 page_ids = 4; } message PagesPacket { - DisaggReadError error = 1; + DisaggReadError error = 1; - // Serialized column file data - // * ColumnFilePersisted alone with its schema, page data, field offsets - repeated bytes pages = 2; - // * ColumnFileInMemory alone with its serialized block - repeated bytes chunks = 3; + // Serialized column file data + // * ColumnFilePersisted alone with its schema, page data, field offsets + repeated bytes pages = 2; + // * ColumnFileInMemory alone with its serialized block + repeated bytes chunks = 3; - // Return tipb.SelectResponse.execution_summaries in the - // last packet - repeated bytes summaries = 4; + // Return tipb.SelectResponse.execution_summaries in the + // last packet + repeated bytes summaries = 4; } diff --git a/proto/disk_usage.proto b/proto/disk_usage.proto index 8670f172..2c49ff3e 100644 --- a/proto/disk_usage.proto +++ b/proto/disk_usage.proto @@ -1,14 +1,20 @@ syntax = "proto3"; package disk_usage; +import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (rustproto.lite_runtime_all) = true; - option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; enum DiskUsage { - Normal = 0; - AlmostFull = 1; - AlreadyFull = 2; + Normal = 0; + AlmostFull = 1; + AlreadyFull = 2; } diff --git a/proto/encryptionpb.proto b/proto/encryptionpb.proto index 2328088c..7eb2978c 100644 --- a/proto/encryptionpb.proto +++ b/proto/encryptionpb.proto @@ -8,71 +8,73 @@ package encryptionpb; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // General encryption metadata for any data type. message EncryptionMeta { - // ID of the key used to encrypt the data. - uint64 key_id = 1; - // Initialization vector (IV) of the data. - bytes iv = 2; + // ID of the key used to encrypt the data. + uint64 key_id = 1; + // Initialization vector (IV) of the data. + bytes iv = 2; } // Information about an encrypted file. message FileInfo { - // ID of the key used to encrypt the file. - uint64 key_id = 1; - // Initialization vector (IV) of the file. - bytes iv = 2; - // Method of encryption algorithm used to encrypted the file. - EncryptionMethod method = 3; + // ID of the key used to encrypt the file. + uint64 key_id = 1; + // Initialization vector (IV) of the file. + bytes iv = 2; + // Method of encryption algorithm used to encrypted the file. + EncryptionMethod method = 3; } message FileDictionary { - // A map of file name to file info. - map files = 1; + // A map of file name to file info. + map files = 1; } enum EncryptionMethod { - UNKNOWN = 0; - PLAINTEXT = 1; - AES128_CTR = 2; - AES192_CTR = 3; - AES256_CTR = 4; - SM4_CTR = 5; + UNKNOWN = 0; + PLAINTEXT = 1; + AES128_CTR = 2; + AES192_CTR = 3; + AES256_CTR = 4; + SM4_CTR = 5; } // The key used to encrypt the user data. message DataKey { - // A sequence of secret bytes used to encrypt data. - bytes key = 1; - // Method of encryption algorithm used to encrypted data. - EncryptionMethod method = 2; - // Creation time of the key. - uint64 creation_time = 3; - // A flag for the key have ever been exposed. - bool was_exposed = 4; + // A sequence of secret bytes used to encrypt data. + bytes key = 1; + // Method of encryption algorithm used to encrypted data. + EncryptionMethod method = 2; + // Creation time of the key. + uint64 creation_time = 3; + // A flag for the key have ever been exposed. + bool was_exposed = 4; } message KeyDictionary { - // A map of key ID to dat key. - map keys = 1; - // ID of a key currently in use. - uint64 current_key_id = 2; + // A map of key ID to dat key. + map keys = 1; + // ID of a key currently in use. + uint64 current_key_id = 2; } // Master key config. message MasterKey { - oneof backend { - MasterKeyPlaintext plaintext = 1; - MasterKeyFile file = 2; - MasterKeyKms kms = 3; - } + oneof backend { + MasterKeyPlaintext plaintext = 1; + MasterKeyFile file = 2; + MasterKeyKms kms = 3; + } } // MasterKeyPlaintext indicates content is stored as plaintext. @@ -81,35 +83,86 @@ message MasterKeyPlaintext {} // MasterKeyFile is a master key backed by a file containing encryption key in human-readable // hex format. message MasterKeyFile { - // Local file path. - string path = 1; + // Local file path. + string path = 1; } // MasterKeyKms is a master key backed by KMS service that manages the encryption key, // and provide API to encrypt and decrypt a data key, which is used to encrypt the content. message MasterKeyKms { - // KMS vendor. - string vendor = 1; - // KMS key id. - string key_id = 2; - // KMS region. - string region = 3; - // KMS endpoint. Normally not needed. - string endpoint = 4; + // KMS vendor. + string vendor = 1; + // KMS key id. + string key_id = 2; + // KMS region. + string region = 3; + // KMS endpoint. Normally not needed. + string endpoint = 4; + // optional, used to set up azure master key backend + AzureKms azure_kms = 5; + // optional, used to set up gcp master key backend + GcpKms gcp_kms = 6; + // optional, used to set up aws master key backend + AwsKms aws_kms = 7; +} + +message AzureKms { + string tenant_id = 1; + string client_id = 2; + string client_secret = 3; + // Key vault to encrypt/decrypt data key. + string key_vault_url = 4; + // optional hsm used to generate data key + string hsm_name = 5; + string hsm_url = 6; + string client_certificate = 7; + string client_certificate_path = 8; + string client_certificate_password = 9; +} + +message GcpKms { + string credential = 1; +} + +message AwsKms { + string access_key = 1; + string secret_access_key = 2; } message EncryptedContent { - // Metadata of the encrypted content. - // Eg. IV, method and KMS key ID - // It is preferred to define new fields for extra metadata than using this metadata map. - map metadata = 1; - // Encrypted content. - bytes content = 2; - // Master key used to encrypt the content. - MasterKey master_key = 3; - // Initilization vector (IV) used. - bytes iv = 4; - // Encrypted data key generated by KMS and used to actually encrypt data. - // Valid only when KMS is used. - bytes ciphertext_key = 5; + // Metadata of the encrypted content. + // Eg. IV, method and KMS key ID + // It is preferred to define new fields for extra metadata than using this metadata map. + map metadata = 1; + // Encrypted content. + bytes content = 2; + // Master key used to encrypt the content. + MasterKey master_key = 3; + // Initilization vector (IV) used. + bytes iv = 4; + // Encrypted data key generated by KMS and used to actually encrypt data. + // Valid only when KMS is used. + bytes ciphertext_key = 5; +} + +message FileEncryptionInfo { + oneof mode { + PlainTextDataKey plain_text_data_key = 1; + MasterKeyBased master_key_based = 2; + } + // file encryption method + encryptionpb.EncryptionMethod encryption_method = 3; + // iv to encrypt the file by data key + bytes file_iv = 4; + // file checksum after encryption, optional if using GCM + bytes checksum = 5; +} + +// not recommended in production. +// user needs to pass back the same data key for restore. +message PlainTextDataKey {} + +message MasterKeyBased { + // encrypted data key with metadata + repeated encryptionpb.EncryptedContent data_key_encrypted_content = 1; } diff --git a/proto/enginepb.proto b/proto/enginepb.proto index c0eb17c7..68358d4e 100644 --- a/proto/enginepb.proto +++ b/proto/enginepb.proto @@ -1,86 +1,96 @@ syntax = "proto3"; package enginepb; +import "gogoproto/gogo.proto"; import "metapb.proto"; import "raft_cmdpb.proto"; import "raft_serverpb.proto"; +import "rustproto.proto"; + +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; message CommandRequestHeader { - uint64 region_id = 1; - uint64 index = 2; - uint64 term = 3; + uint64 region_id = 1; + uint64 index = 2; + uint64 term = 3; - // Flush in-memory data to disk. - bool sync_log = 4; - // Destroy the region. - bool destroy = 5; + // Flush in-memory data to disk. + bool sync_log = 4; + // Destroy the region. + bool destroy = 5; - // Additional information for the request. - bytes context = 6; + // Additional information for the request. + bytes context = 6; } message CommandRequest { - CommandRequestHeader header = 1; + CommandRequestHeader header = 1; - // We don't enclose normal requests and administrator request - // at same time. + // We don't enclose normal requests and administrator request + // at same time. - // kv put / delete - repeated raft_cmdpb.Request requests = 2; + // kv put / delete + repeated raft_cmdpb.Request requests = 2; - // region metadata manipulation command. - raft_cmdpb.AdminRequest admin_request = 3; - // region metadata manipulation result. - raft_cmdpb.AdminResponse admin_response = 4; + // region metadata manipulation command. + raft_cmdpb.AdminRequest admin_request = 3; + // region metadata manipulation result. + raft_cmdpb.AdminResponse admin_response = 4; } message CommandRequestBatch { - repeated CommandRequest requests = 1; + repeated CommandRequest requests = 1; } message CommandResponseHeader { - uint64 region_id = 1; - // Region is destroyed. - bool destroyed = 2; + uint64 region_id = 1; + // Region is destroyed. + bool destroyed = 2; } message CommandResponse { - CommandResponseHeader header = 1; + CommandResponseHeader header = 1; - raft_serverpb.RaftApplyState apply_state = 2; - uint64 applied_term = 3; + raft_serverpb.RaftApplyState apply_state = 2; + uint64 applied_term = 3; } message CommandResponseBatch { - repeated CommandResponse responses = 1; + repeated CommandResponse responses = 1; } message SnapshotState { - metapb.Region region = 1; - metapb.Peer peer = 2; - raft_serverpb.RaftApplyState apply_state = 3; + metapb.Region region = 1; + metapb.Peer peer = 2; + raft_serverpb.RaftApplyState apply_state = 3; } message SnapshotData { - string cf = 1; - uint32 checksum = 2; - repeated raft_serverpb.KeyValue data = 3; + string cf = 1; + uint32 checksum = 2; + repeated raft_serverpb.KeyValue data = 3; } message SnapshotRequest { - oneof chunk { - // The first message for snapshots. - // It contains the latest region information after applied snapshot. - SnapshotState state = 1; - - // Following messages are always data. - SnapshotData data = 2; - } + oneof chunk { + // The first message for snapshots. + // It contains the latest region information after applied snapshot. + SnapshotState state = 1; + + // Following messages are always data. + SnapshotData data = 2; + } } message SnapshotDone {} service Engine { - rpc ApplyCommandBatch(stream CommandRequestBatch) returns (stream CommandResponseBatch) {} - rpc ApplySnapshot(stream SnapshotRequest) returns (SnapshotDone) {} + rpc ApplyCommandBatch(stream CommandRequestBatch) returns (stream CommandResponseBatch) {} + rpc ApplySnapshot(stream SnapshotRequest) returns (SnapshotDone) {} } diff --git a/proto/errorpb.proto b/proto/errorpb.proto index 11628cdf..f05368e0 100644 --- a/proto/errorpb.proto +++ b/proto/errorpb.proto @@ -1,208 +1,217 @@ syntax = "proto3"; package errorpb; -import "metapb.proto"; import "gogoproto/gogo.proto"; +import "metapb.proto"; import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - -// NotLeader is the error variant that tells a request be handle by raft leader +// NotLeader is the error variant that tells a request be handle by raft leader // is sent to raft follower or learner. message NotLeader { - // The requested region ID - uint64 region_id = 1; - // Region leader of the requested region - metapb.Peer leader = 2; + // The requested region ID + uint64 region_id = 1; + // Region leader of the requested region + metapb.Peer leader = 2; } // IsWitness is the error variant that tells a request be handle by witness // which should be forbidden and retry. message IsWitness { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } // BucketVersionNotMatch is the error variant that tells the request buckets version is not match. // client should update the buckets version and retry. -message BucketVersionNotMatch{ - uint64 version = 1; - repeated bytes keys = 2; +message BucketVersionNotMatch { + uint64 version = 1; + repeated bytes keys = 2; } message DiskFull { - // The requested store ID - repeated uint64 store_id = 1; - // The detailed info - string reason = 2; + // The requested store ID + repeated uint64 store_id = 1; + // The detailed info + string reason = 2; } -// StoreNotMatch is the error variant that tells the request is sent to wrong store. +// StoreNotMatch is the error variant that tells the request is sent to wrong store. // (i.e. inconsistency of the store ID that request shows and the real store ID of this server.) message StoreNotMatch { - // Store id in request - uint64 request_store_id = 1; - // Actual store id - uint64 actual_store_id = 2; + // Store id in request + uint64 request_store_id = 1; + // Actual store id + uint64 actual_store_id = 2; } // RegionNotFound is the error variant that tells there isn't any region in this TiKV // matches the requested region ID. message RegionNotFound { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } // RegionNotInitialized is the error variant that tells there isn't any initialized peer // matchesthe request region ID. -message RegionNotInitialized { - // The request region ID - uint64 region_id = 1; +message RegionNotInitialized { + // The request region ID + uint64 region_id = 1; } // KeyNotInRegion is the error variant that tells the key the request requires isn't present in -// this region. +// this region. message KeyNotInRegion { - // The requested key - bytes key = 1; - // The requested region ID - uint64 region_id = 2; - // Start key of the requested region - bytes start_key = 3; - // Snd key of the requested region - bytes end_key = 4; + // The requested key + bytes key = 1; + // The requested region ID + uint64 region_id = 2; + // Start key of the requested region + bytes start_key = 3; + // Snd key of the requested region + bytes end_key = 4; } // EpochNotMatch is the error variant that tells a region has been updated. // (e.g. by splitting / merging, or raft Confchange.) // Hence, a command is based on a stale version of a region. message EpochNotMatch { - // Available regions that may be siblings of the requested one. - repeated metapb.Region current_regions = 1; + // Available regions that may be siblings of the requested one. + repeated metapb.Region current_regions = 1; } // ServerIsBusy is the error variant that tells the server is too busy to response. message ServerIsBusy { - string reason = 1; - // The suggested backoff time - uint64 backoff_ms = 2; - uint32 estimated_wait_ms = 3; - // Current applied_index at the leader, may be used in replica read. - uint64 applied_index = 4; + string reason = 1; + // The suggested backoff time + uint64 backoff_ms = 2; + uint32 estimated_wait_ms = 3; + // Current applied_index at the leader, may be used in replica read. + uint64 applied_index = 4; } // StaleCommand is the error variant that tells the command is stale, that is, // the current request term is lower than current raft term. // This can be retried at most time. -message StaleCommand { -} +message StaleCommand {} // RaftEntryTooLarge is the error variant that tells the request is too large to be serialized to a // reasonable small raft entry. // (i.e. greater than the configured value `raft_entry_max_size` in `raftstore`) message RaftEntryTooLarge { - // The requested region ID - uint64 region_id = 1; - // Size of the raft entry - uint64 entry_size = 2; + // The requested region ID + uint64 region_id = 1; + // Size of the raft entry + uint64 entry_size = 2; } // MaxTimestampNotSynced is the error variant that tells the peer has just become a leader and // updating the max timestamp in the concurrency manager from PD TSO is ongoing. In this case, // the prewrite of an async commit transaction cannot succeed. The client can backoff and // resend the request. -message MaxTimestampNotSynced { -} +message MaxTimestampNotSynced {} // ReadIndexNotReady is the error variant that tells the read index request is not ready, that is, // the current region is in a status that not ready to serve the read index request. For example, // region is in splitting or merging status. // This can be retried at most time. message ReadIndexNotReady { - // The reason why the region is not ready to serve read index request - string reason = 1; - // The requested region ID - uint64 region_id = 2; + // The reason why the region is not ready to serve read index request + string reason = 1; + // The requested region ID + uint64 region_id = 2; } // ProposalInMergingMode is the error variant that tells the proposal is rejected because raft is // in the merging mode. This may happen when BR/Lightning try to ingest SST. // This can be retried at most time. message ProposalInMergingMode { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } message DataIsNotReady { - // The requested region ID - uint64 region_id = 1; - uint64 peer_id = 2; - uint64 safe_ts = 3; + // The requested region ID + uint64 region_id = 1; + uint64 peer_id = 2; + uint64 safe_ts = 3; } message RecoveryInProgress { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } message FlashbackInProgress { - // The requested region ID - uint64 region_id = 1; - uint64 flashback_start_ts = 2; + // The requested region ID + uint64 region_id = 1; + uint64 flashback_start_ts = 2; } message FlashbackNotPrepared { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } // MismatchPeerId is the error variant that tells the request is sent to wrong peer. // Client receives this error should reload the region info and retry. message MismatchPeerId { - uint64 request_peer_id = 1; - uint64 store_peer_id = 2; + uint64 request_peer_id = 1; + uint64 store_peer_id = 2; +} + +// UndeterminedResult is the error variant that tells the result is not determined yet. +// For example, the raft protocol timed out and the apply result is unknown. +message UndeterminedResult { + string message = 1; } // Error wraps all region errors, indicates an error encountered by a request. message Error { - reserved "stale_epoch"; - - // The error message - string message = 1; - NotLeader not_leader = 2; - RegionNotFound region_not_found = 3; - KeyNotInRegion key_not_in_region = 4; - EpochNotMatch epoch_not_match = 5; - ServerIsBusy server_is_busy = 6; - StaleCommand stale_command = 7; - StoreNotMatch store_not_match = 8; - RaftEntryTooLarge raft_entry_too_large = 9; - MaxTimestampNotSynced max_timestamp_not_synced = 10; - ReadIndexNotReady read_index_not_ready = 11; - ProposalInMergingMode proposal_in_merging_mode = 12; - DataIsNotReady data_is_not_ready = 13; - RegionNotInitialized region_not_initialized = 14; - DiskFull disk_full = 15; - // Online recovery is still in performing, reject writes to avoid potential issues - RecoveryInProgress RecoveryInProgress = 16; - // Flashback is still in performing, reject any read or write to avoid potential issues. - // NOTICE: this error is non-retryable, the request should fail ASAP when it meets this error. - FlashbackInProgress FlashbackInProgress = 17; - // If the second phase flashback request is sent to a region that is not prepared for the flashback, - // this error will be returned. - // NOTICE: this error is non-retryable, the client should retry the first phase flashback request when it meets this error. - FlashbackNotPrepared FlashbackNotPrepared = 18; - // IsWitness is the error variant that tells a request be handle by witness - // which should be forbidden and retry. - IsWitness is_witness = 19; - - MismatchPeerId mismatch_peer_id = 20; - - // BucketVersionNotMatch is the error variant that tells the request buckets version is not match. - BucketVersionNotMatch bucket_version_not_match = 21; + reserved "stale_epoch"; + + // The error message + string message = 1; + NotLeader not_leader = 2; + RegionNotFound region_not_found = 3; + KeyNotInRegion key_not_in_region = 4; + EpochNotMatch epoch_not_match = 5; + ServerIsBusy server_is_busy = 6; + StaleCommand stale_command = 7; + StoreNotMatch store_not_match = 8; + RaftEntryTooLarge raft_entry_too_large = 9; + MaxTimestampNotSynced max_timestamp_not_synced = 10; + ReadIndexNotReady read_index_not_ready = 11; + ProposalInMergingMode proposal_in_merging_mode = 12; + DataIsNotReady data_is_not_ready = 13; + RegionNotInitialized region_not_initialized = 14; + DiskFull disk_full = 15; + // Online recovery is still in performing, reject writes to avoid potential issues + RecoveryInProgress RecoveryInProgress = 16; + // Flashback is still in performing, reject any read or write to avoid potential issues. + // NOTICE: this error is non-retryable, the request should fail ASAP when it meets this error. + FlashbackInProgress FlashbackInProgress = 17; + // If the second phase flashback request is sent to a region that is not prepared for the flashback, + // this error will be returned. + // NOTICE: this error is non-retryable, the client should retry the first phase flashback request when it meets this error. + FlashbackNotPrepared FlashbackNotPrepared = 18; + // IsWitness is the error variant that tells a request be handle by witness + // which should be forbidden and retry. + IsWitness is_witness = 19; + + MismatchPeerId mismatch_peer_id = 20; + + // BucketVersionNotMatch is the error variant that tells the request buckets version is not match. + BucketVersionNotMatch bucket_version_not_match = 21; + + // UndeterminedResult is the error variant that tells the result is not determined yet. + UndeterminedResult undetermined_result = 22; } diff --git a/proto/gcpb.proto b/proto/gcpb.proto index 053872b8..8af33908 100644 --- a/proto/gcpb.proto +++ b/proto/gcpb.proto @@ -4,13 +4,15 @@ package gcpb; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - service GC { rpc ListKeySpaces(ListKeySpacesRequest) returns (ListKeySpacesResponse) {} diff --git a/proto/import_kvpb.proto b/proto/import_kvpb.proto index fc72adf1..6495867b 100644 --- a/proto/import_kvpb.proto +++ b/proto/import_kvpb.proto @@ -2,17 +2,19 @@ syntax = "proto3"; package import_kvpb; -import "import_sstpb.proto"; import "gogoproto/gogo.proto"; +import "import_sstpb.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // ImportKV provides a service to import key-value pairs to TiKV. // // In order to import key-value pairs to TiKV, the user should: @@ -32,137 +34,130 @@ option java_package = "org.tikv.kvproto"; // in the engine. An engine can not be cleaned up when it is // writing or importing. service ImportKV { - // Switch the target cluster to normal/import mode. - rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} - // Open an engine. - rpc OpenEngine(OpenEngineRequest) returns (OpenEngineResponse) {} - // Open a write stream to the engine. - rpc WriteEngine(stream WriteEngineRequest) returns (WriteEngineResponse) {} - // Write to engine, single message version - rpc WriteEngineV3(WriteEngineV3Request) returns (WriteEngineResponse) {} - // Close the engine. - rpc CloseEngine(CloseEngineRequest) returns (CloseEngineResponse) {} - // Import the engine to the target cluster. - rpc ImportEngine(ImportEngineRequest) returns (ImportEngineResponse) {} - // Clean up the engine. - rpc CleanupEngine(CleanupEngineRequest) returns (CleanupEngineResponse) {} - // Compact the target cluster for better performance. - rpc CompactCluster(CompactClusterRequest) returns (CompactClusterResponse) {} - // Get current version and commit hash - rpc GetVersion(GetVersionRequest) returns (GetVersionResponse) {} - // Get importer metrics - rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} + // Switch the target cluster to normal/import mode. + rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} + // Open an engine. + rpc OpenEngine(OpenEngineRequest) returns (OpenEngineResponse) {} + // Open a write stream to the engine. + rpc WriteEngine(stream WriteEngineRequest) returns (WriteEngineResponse) {} + // Write to engine, single message version + rpc WriteEngineV3(WriteEngineV3Request) returns (WriteEngineResponse) {} + // Close the engine. + rpc CloseEngine(CloseEngineRequest) returns (CloseEngineResponse) {} + // Import the engine to the target cluster. + rpc ImportEngine(ImportEngineRequest) returns (ImportEngineResponse) {} + // Clean up the engine. + rpc CleanupEngine(CleanupEngineRequest) returns (CleanupEngineResponse) {} + // Compact the target cluster for better performance. + rpc CompactCluster(CompactClusterRequest) returns (CompactClusterResponse) {} + // Get current version and commit hash + rpc GetVersion(GetVersionRequest) returns (GetVersionResponse) {} + // Get importer metrics + rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} } message SwitchModeRequest { - string pd_addr = 1; - import_sstpb.SwitchModeRequest request = 2; + string pd_addr = 1; + import_sstpb.SwitchModeRequest request = 2; } -message SwitchModeResponse { -} +message SwitchModeResponse {} message OpenEngineRequest { - bytes uuid = 1; - bytes key_prefix = 2; + bytes uuid = 1; + bytes key_prefix = 2; } -message OpenEngineResponse { -} +message OpenEngineResponse {} message WriteHead { - bytes uuid = 1; + bytes uuid = 1; } message Mutation { - enum OP { - Put = 0; - } - OP op = 1; - bytes key = 2; - bytes value = 3; + enum OP { + Put = 0; + } + OP op = 1; + bytes key = 2; + bytes value = 3; } message WriteBatch { - uint64 commit_ts = 1; - repeated Mutation mutations = 2; + uint64 commit_ts = 1; + repeated Mutation mutations = 2; } message WriteEngineRequest { - oneof chunk { - WriteHead head = 1; - WriteBatch batch = 2; - } + oneof chunk { + WriteHead head = 1; + WriteBatch batch = 2; + } } message KVPair { - bytes key = 1; - bytes value = 2; + bytes key = 1; + bytes value = 2; } message WriteEngineV3Request { - bytes uuid = 1; - uint64 commit_ts = 2; - repeated KVPair pairs = 3; + bytes uuid = 1; + uint64 commit_ts = 2; + repeated KVPair pairs = 3; } message WriteEngineResponse { - Error error = 1; + Error error = 1; } message CloseEngineRequest { - bytes uuid = 1; + bytes uuid = 1; } message CloseEngineResponse { - Error error = 1; + Error error = 1; } message ImportEngineRequest { - bytes uuid = 1; - string pd_addr = 2; + bytes uuid = 1; + string pd_addr = 2; } -message ImportEngineResponse { -} +message ImportEngineResponse {} message CleanupEngineRequest { - bytes uuid = 1; + bytes uuid = 1; } -message CleanupEngineResponse { -} +message CleanupEngineResponse {} message CompactClusterRequest { - string pd_addr = 1; - import_sstpb.CompactRequest request = 2; + string pd_addr = 1; + import_sstpb.CompactRequest request = 2; } -message CompactClusterResponse { -} +message CompactClusterResponse {} -message GetVersionRequest { -} +message GetVersionRequest {} message GetVersionResponse { - string version = 1; - string commit = 2; + string version = 1; + string commit = 2; } -message GetMetricsRequest { -} +message GetMetricsRequest {} message GetMetricsResponse { - string prometheus = 1; + string prometheus = 1; } message Error { - message EngineNotFound { - bytes uuid = 1; - } - // This can happen if the client hasn't opened the engine, or the server - // restarts while the client is writing or closing. An unclosed engine will - // be removed on server restart, so the client should not continue but - // restart the previous job in that case. - EngineNotFound engine_not_found = 1; + message EngineNotFound { + bytes uuid = 1; + } + // This can happen if the client hasn't opened the engine, or the server + // restarts while the client is writing or closing. An unclosed engine will + // be removed on server restart, so the client should not continue but + // restart the previous job in that case. + EngineNotFound engine_not_found = 1; } diff --git a/proto/import_sstpb.proto b/proto/import_sstpb.proto index 089dd35f..b230d31b 100644 --- a/proto/import_sstpb.proto +++ b/proto/import_sstpb.proto @@ -2,20 +2,23 @@ syntax = "proto3"; package import_sstpb; -import "metapb.proto"; +import "brpb.proto"; +import "encryptionpb.proto"; import "errorpb.proto"; -import "kvrpcpb.proto"; import "gogoproto/gogo.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; import "rustproto.proto"; -import "brpb.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // ImportSST provides a service to import a generated SST file to a region in TiKV. // // In order to import an SST file to a region, the user should: @@ -32,417 +35,465 @@ option java_package = "org.tikv.kvproto"; // file's metadata, to guarantee that the region's epoch must be the same // between the SST file is uploaded and ingested later. service ImportSST { - // Switch to normal/import mode. - rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} - // Get import mode(normal/import). - rpc GetMode(GetModeRequest) returns (GetModeResponse) {} - // Upload an SST file to a server. - rpc Upload(stream UploadRequest) returns (UploadResponse) {} - // Ingest an uploaded SST file to a region. - rpc Ingest(IngestRequest) returns (IngestResponse) {} - // Compact the specific range for better performance. - rpc Compact(CompactRequest) returns (CompactResponse) {} - - rpc SetDownloadSpeedLimit(SetDownloadSpeedLimitRequest) returns (SetDownloadSpeedLimitResponse) {} - // Download an SST file from an external storage, and performs key-rewrite - // after downloading. - rpc Download(DownloadRequest) returns (DownloadResponse) {} - - // Open a write stream to generate sst files - rpc Write(stream WriteRequest) returns (WriteResponse) {} - rpc RawWrite(stream RawWriteRequest) returns (RawWriteResponse) {} - - // Ingest Multiple files in one request - rpc MultiIngest(MultiIngestRequest) returns (IngestResponse) {} - - // Collect duplicate data from TiKV. - rpc DuplicateDetect(DuplicateDetectRequest) returns (stream DuplicateDetectResponse) {} - - // Apply download & apply increment kv files to TiKV. - rpc Apply(ApplyRequest) returns (ApplyResponse) {} - - // ClearFiles clear applied file after restore succeed. - rpc ClearFiles(ClearRequest) returns (ClearResponse) {} - - // Suspend ingest for data listeners don't support catching import data. - rpc SuspendImportRPC(SuspendImportRPCRequest) returns (SuspendImportRPCResponse) {} - + // Switch to normal/import mode. + rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} + // Get import mode(normal/import). + rpc GetMode(GetModeRequest) returns (GetModeResponse) {} + // Upload an SST file to a server. + rpc Upload(stream UploadRequest) returns (UploadResponse) {} + // Ingest an uploaded SST file to a region. + rpc Ingest(IngestRequest) returns (IngestResponse) {} + // Compact the specific range for better performance. + rpc Compact(CompactRequest) returns (CompactResponse) {} + + rpc SetDownloadSpeedLimit(SetDownloadSpeedLimitRequest) returns (SetDownloadSpeedLimitResponse) {} + // Download an SST file from an external storage, and performs key-rewrite + // after downloading. + rpc Download(DownloadRequest) returns (DownloadResponse) {} + // Download SST files in batch from external storage, perform key rewrite, + // and merge them into one SST after downloading. + rpc BatchDownload(DownloadRequest) returns (DownloadResponse) {} + // Download SST files in batch from external storage, perform key rewrite, + // merge them into one SST, and keep only the latest MVCC version after downloading. + rpc BatchDownloadLatestMVCC(DownloadRequest) returns (DownloadResponse) {} + + // Open a write stream to generate sst files + rpc Write(stream WriteRequest) returns (WriteResponse) {} + rpc RawWrite(stream RawWriteRequest) returns (RawWriteResponse) {} + + // Ingest Multiple files in one request + rpc MultiIngest(MultiIngestRequest) returns (IngestResponse) {} + + // Collect duplicate data from TiKV. + rpc DuplicateDetect(DuplicateDetectRequest) returns (stream DuplicateDetectResponse) {} + + // Apply download & apply increment kv files to TiKV. + rpc Apply(ApplyRequest) returns (ApplyResponse) {} + + // ClearFiles clear applied file after restore succeed. + rpc ClearFiles(ClearRequest) returns (ClearResponse) {} + + // Suspend ingest for data listeners don't support catching import data. + rpc SuspendImportRPC(SuspendImportRPCRequest) returns (SuspendImportRPCResponse) {} + + // AddForcePartitionRange marks a range in tikv that any compact overlaps with this range + // should generates SST files partitioned at region boundaries as well as this range boundary. + // TiKV will also try to do manual compact(if needed) after setting this range to eusure + // any incoming SST under this range can be ingested into the bottom level if there is no real kv overlap. + rpc AddForcePartitionRange(AddPartitionRangeRequest) returns (AddPartitionRangeResponse) {} + + // Remove the force partition range after the task is finished. If this function is not called, + // tikv will cleanup the range after TTL to ensure it can be cleaned eventually. + rpc RemoveForcePartitionRange(RemovePartitionRangeRequest) returns (RemovePartitionRangeResponse) {} } message SuspendImportRPCRequest { - // whether to suspend new imports. - bool should_suspend_imports = 1; + // whether to suspend new imports. + bool should_suspend_imports = 1; - // the duration of import service suspension - // when should_deny_imports is false, - // this won't take effect. - uint64 duration_in_secs = 2; - // The identifier for the caller. - string caller = 3; + // the duration of import service suspension + // when should_deny_imports is false, + // this won't take effect. + uint64 duration_in_secs = 2; + // The identifier for the caller. + string caller = 3; } message SuspendImportRPCResponse { - // The last state before this RPC. - bool already_suspended = 1; + // The last state before this RPC. + bool already_suspended = 1; } enum SwitchMode { - Normal = 0; - Import = 1; + Normal = 0; + Import = 1; } message SwitchModeRequest { - SwitchMode mode = 1; - repeated Range ranges = 2; + SwitchMode mode = 1; + repeated Range ranges = 2; } -message SwitchModeResponse { -} +message SwitchModeResponse {} -message GetModeRequest { -} +message GetModeRequest {} message GetModeResponse { - SwitchMode mode = 1; + SwitchMode mode = 1; } message Range { - bytes start = 1; - bytes end = 2; + bytes start = 1; + bytes end = 2; } message SSTMeta { - bytes uuid = 1; - Range range = 2; - uint32 crc32 = 3; - uint64 length = 4; - string cf_name = 5; - uint64 region_id = 6; - metapb.RegionEpoch region_epoch = 7; - bool end_key_exclusive = 8; - // total_kvs and total_bytes is equivalent to PD's approximate_keys and approximate_size - // set these values can save time from tikv upload keys and size to PD through Heartbeat. - uint64 total_kvs = 9; - uint64 total_bytes = 10; - // API version implies the encode of the key and value. - kvrpcpb.APIVersion api_version = 11; - // cipher_iv is used to encrypt/decrypt sst - bytes cipher_iv = 12; + bytes uuid = 1; + Range range = 2; + uint32 crc32 = 3; + uint64 length = 4; + string cf_name = 5; + uint64 region_id = 6; + metapb.RegionEpoch region_epoch = 7; + bool end_key_exclusive = 8; + // total_kvs and total_bytes is equivalent to PD's approximate_keys and approximate_size + // set these values can save time from tikv upload keys and size to PD through Heartbeat. + uint64 total_kvs = 9; + uint64 total_bytes = 10; + // API version implies the encode of the key and value. + kvrpcpb.APIVersion api_version = 11; + // cipher_iv is used to encrypt/decrypt sst + bytes cipher_iv = 12; + + // Reserved for a downstream fork + reserved 1000 to 1199; } // A rewrite rule is applied on the *encoded* keys (the internal storage // representation). message RewriteRule { - bytes old_key_prefix = 1; - bytes new_key_prefix = 2; - uint64 new_timestamp = 3; + bytes old_key_prefix = 1; + bytes new_key_prefix = 2; + // (Optional) Rewrite all keys in the range to use this timestamp. + uint64 new_timestamp = 3; + // (Optional) Skip keys with timestamps greater than this during download, useful for compacted SST backups. + uint64 ignore_after_timestamp = 4; + // (Optional) Skip write CF keys with timestamps less than this during download. Default CF keys are preserved. + uint64 ignore_before_timestamp = 5; } message UploadRequest { - oneof chunk { - SSTMeta meta = 1; - bytes data = 2; - } + oneof chunk { + SSTMeta meta = 1; + bytes data = 2; + } } -message UploadResponse { -} +message UploadResponse {} message IngestRequest { - kvrpcpb.Context context = 1; - SSTMeta sst = 2; + kvrpcpb.Context context = 1; + SSTMeta sst = 2; } message MultiIngestRequest { - kvrpcpb.Context context = 1; - repeated SSTMeta ssts = 2; + kvrpcpb.Context context = 1; + repeated SSTMeta ssts = 2; } message IngestResponse { - errorpb.Error error = 1; + errorpb.Error error = 1; } message CompactRequest { - // Compact files in the range and above the output level. - // Compact all files if the range is not specified. - // Compact all files to the bottommost level if the output level is -1. - Range range = 1; - int32 output_level = 2; - kvrpcpb.Context context = 3; + // Compact files in the range and above the output level. + // Compact all files if the range is not specified. + // Compact all files to the bottommost level if the output level is -1. + Range range = 1; + int32 output_level = 2; + kvrpcpb.Context context = 3; } -message CompactResponse { -} +message CompactResponse {} message DownloadRequest { - // Map represents the map of . - // We'll generate all SSTMeta into one SST File. - map ssts = 1; - // resolved_ts is used to merge related SST Files. - uint64 resolved_ts = 3; - - // The SST meta used to identify the downloaded file. - // Must be the same among all nodes in the same Raft group. - // Note: the "crc32" and "cf_name" fields are ignored in this request, - // and the "range" field represents the closed key range after rewrite - // (as origin keys in encoded representation). - SSTMeta sst = 2 [(gogoproto.nullable) = false]; - - // The url field is deprecated, use storage_backend instead - reserved 8; reserved "url"; - - // The file name of the SST file. - string name = 9; - - // Performs a key prefix rewrite after downloading the SST file. - // All keys in the SST will be rewritten as: - // - // new_key = new_key_prefix + old_key[len(old_key_prefix)..] - // - // When used for TiDB, rewriting the prefix changes the table ID. Please - // note that key-rewrite is applied on the origin keys in encoded - // representation (the SST itself should still use data keys in encoded - // representation). - // - // You need to ensure that the keys before and after rewriting are in the - // same order, otherwise the RPC request will fail. - RewriteRule rewrite_rule = 13 [(gogoproto.nullable) = false]; - - backup.StorageBackend storage_backend = 14; - // The identity for the stroage backend. - // When this field presents, the storage would be cached. - // If there is a cached storage, TiKV would use it driectly. - string storage_cache_id = 17; - - bool is_raw_kv = 15; - // cipher_info is used to decrypt sst when download sst - backup.CipherInfo cipher_info = 16; - - // The type of the download request. - DownloadRequestType request_type = 18; - kvrpcpb.Context context = 19; + // Map represents the map of . + // We'll generate all SSTMeta into one SST File. + map ssts = 1; + // resolved_ts is used to merge related SST Files. + uint64 resolved_ts = 3; + + // The SST meta used to identify the downloaded file. + // Must be the same among all nodes in the same Raft group. + // Note: the "crc32" and "cf_name" fields are ignored in this request, + // and the "range" field represents the closed key range after rewrite + // (as origin keys in encoded representation). + SSTMeta sst = 2 [(gogoproto.nullable) = false]; + + // The url field is deprecated, use storage_backend instead + reserved 8; + reserved "url"; + + // The file name of the SST file. + string name = 9; + + // Performs a key prefix rewrite after downloading the SST file. + // All keys in the SST will be rewritten as: + // + // new_key = new_key_prefix + old_key[len(old_key_prefix)..] + // + // When used for TiDB, rewriting the prefix changes the table ID. Please + // note that key-rewrite is applied on the origin keys in encoded + // representation (the SST itself should still use data keys in encoded + // representation). + // + // You need to ensure that the keys before and after rewriting are in the + // same order, otherwise the RPC request will fail. + RewriteRule rewrite_rule = 13 [(gogoproto.nullable) = false]; + repeated RewriteRule sorted_rewrite_rules = 20; + + backup.StorageBackend storage_backend = 14; + // The identity for the stroage backend. + // When this field presents, the storage would be cached. + // If there is a cached storage, TiKV would use it driectly. + string storage_cache_id = 17; + + bool is_raw_kv = 15; + // cipher_info is used to decrypt sst when download sst + backup.CipherInfo cipher_info = 16; + + // The type of the download request. + DownloadRequestType request_type = 18; + kvrpcpb.Context context = 19; } enum DownloadRequestType { - // For the compatibility with old version of TiDBs - Legacy = 0; - // For the TiDBs with newer versions that support keyspace feature. - Keyspace = 1; + // For the compatibility with old version of TiDBs + Legacy = 0; + // For the TiDBs with newer versions that support keyspace feature. + Keyspace = 1; } // For now it is just used for distinguishing the error of the request with the error // of gRPC, add more concrete types if it is necessary later. message Error { - string message = 1; - - // We meet some internal errors of the store. - errorpb.Error store_error = 2; + string message = 1; + + // We meet some internal errors of the store. + errorpb.Error store_error = 2; } message DownloadResponse { - // The actual key range (after rewrite) of the downloaded SST. The range is - // inclusive in both ends. - Range range = 1 [(gogoproto.nullable) = false]; + // The actual key range (after rewrite) of the downloaded SST. The range is + // inclusive in both ends. + Range range = 1 [(gogoproto.nullable) = false]; - // Whether the SST is empty. An empty SST is prohibited in TiKV, do not - // ingest if this field is true. - // (Deprecated, should be replaced by checking `length == 0` in the future) - bool is_empty = 2; + // Whether the SST is empty. An empty SST is prohibited in TiKV, do not + // ingest if this field is true. + // (Deprecated, should be replaced by checking `length == 0` in the future) + bool is_empty = 2; - Error error = 3; + Error error = 3; - // The CRC32 checksum of the rewritten SST file (implementation can return - // zero, indicating the CRC32 was not calculated). - uint32 crc32 = 4; - // The actual length of the rewritten SST file. - uint64 length = 5; + // The CRC32 checksum of the rewritten SST file (implementation can return + // zero, indicating the CRC32 was not calculated). + uint32 crc32 = 4; + // The actual length of the rewritten SST file. + uint64 length = 5; - // This field only return when file-copy backup enabled. - // Because it will merge many SST files in a download request. - repeated SSTMeta ssts = 6; + // This field only return when file-copy backup enabled. + // Because it will merge many SST files in a download request. + repeated SSTMeta ssts = 6; } message SetDownloadSpeedLimitRequest { - // The download speed limit (bytes/second). Set to 0 for unlimited speed. - uint64 speed_limit = 1; -} + // The download speed limit (bytes/second). Set to 0 for unlimited speed. + uint64 speed_limit = 1; + + // The download speed limit task id. + string task_id = 2; -message SetDownloadSpeedLimitResponse { + // The download speed limit ttl. Set to 0 means it will never expire. + uint64 ttl_seconds = 3; } +message SetDownloadSpeedLimitResponse {} + message Pair { - bytes key = 1; - bytes value = 2; - enum OP { - Put = 0; - Delete = 1; - } - OP op = 3; + bytes key = 1; + bytes value = 2; + enum OP { + Put = 0; + Delete = 1; + } + OP op = 3; } message WriteBatch { - uint64 commit_ts = 1; - repeated Pair pairs = 2; + uint64 commit_ts = 1; + repeated Pair pairs = 2; } message WriteRequest { - oneof chunk { - SSTMeta meta = 1; - WriteBatch batch = 2; - } - kvrpcpb.Context context = 3; + oneof chunk { + SSTMeta meta = 1; + WriteBatch batch = 2; + } + kvrpcpb.Context context = 3; } message WriteResponse { - Error error = 1; - repeated SSTMeta metas = 2; + Error error = 1; + repeated SSTMeta metas = 2; } message RawWriteBatch { - uint64 ttl = 1; - repeated Pair pairs = 2; + uint64 ttl = 1; + repeated Pair pairs = 2; - // To be compatible with the key encoding of API V2. - // This field should be generated from the client instead of the server, - // since the message will be send to all the replicas of a region. - // Otherwise, the underlying data generated by the server would be inconsistent which is hard to scale - // for other features like MVCC over RawKV. - uint64 ts = 3; + // To be compatible with the key encoding of API V2. + // This field should be generated from the client instead of the server, + // since the message will be send to all the replicas of a region. + // Otherwise, the underlying data generated by the server would be inconsistent which is hard to scale + // for other features like MVCC over RawKV. + uint64 ts = 3; } message RawWriteRequest { - oneof chunk { - SSTMeta meta = 1; - RawWriteBatch batch = 2; - } - kvrpcpb.Context context = 3; + oneof chunk { + SSTMeta meta = 1; + RawWriteBatch batch = 2; + } + kvrpcpb.Context context = 3; } message RawWriteResponse { - Error error = 1; - repeated SSTMeta metas = 2; + Error error = 1; + repeated SSTMeta metas = 2; } message DuplicateDetectRequest { - kvrpcpb.Context context = 1; - bytes start_key = 2; - bytes end_key = 3; - // Return only the keys found by scanning, not their values. - bool key_only = 4; - // We only check the data whose timestamp is larger than `min_commit_ts`. `min_commit_ts` is exclueded. - uint64 min_commit_ts = 5; + kvrpcpb.Context context = 1; + bytes start_key = 2; + bytes end_key = 3; + // Return only the keys found by scanning, not their values. + bool key_only = 4; + // We only check the data whose timestamp is larger than `min_commit_ts`. `min_commit_ts` is exclueded. + uint64 min_commit_ts = 5; } message KvPair { - bytes key = 1; - bytes value = 2; - uint64 commit_ts = 3; + bytes key = 1; + bytes value = 2; + uint64 commit_ts = 3; } message DuplicateDetectResponse { - errorpb.Error region_error = 1; - Error key_error = 2; - // The these keys will be in asc order (but commit time is in desc order), - // and the content is just like following: - // [ - // {key: "key1", value: "value11", commit_ts: 1005}, - // {key: "key1", value: "value12", commit_ts: 1004}, - // {key: "key1", value: "value13", commit_ts: 1001}, - // {key: "key2", value: "value21", commit_ts: 1004}, - // {key: "key2", value: "value22", commit_ts: 1002}, - // ... - // ] - repeated KvPair pairs = 3; + errorpb.Error region_error = 1; + Error key_error = 2; + // The these keys will be in asc order (but commit time is in desc order), + // and the content is just like following: + // [ + // {key: "key1", value: "value11", commit_ts: 1005}, + // {key: "key1", value: "value12", commit_ts: 1004}, + // {key: "key1", value: "value13", commit_ts: 1001}, + // {key: "key2", value: "value21", commit_ts: 1004}, + // {key: "key2", value: "value22", commit_ts: 1002}, + // ... + // ] + repeated KvPair pairs = 3; } message KVMeta { - // The file name of the KV file. - string name = 1; + // The file name of the KV file. + string name = 1; - // file offset, sometimes only need to get a part of data from the merged file - uint64 range_offset = 11; + // file offset, sometimes only need to get a part of data from the merged file + uint64 range_offset = 11; - // file length for check. - uint64 length = 2; + // file length for check. + uint64 length = 2; - // range length of the merged file, if it exists. - uint64 range_length = 12; + // range length of the merged file, if it exists. + uint64 range_length = 12; - // tell us which cf should apply. WRITE_CF or DEFAULT_CF e.g. - string cf = 3; + // tell us which cf should apply. WRITE_CF or DEFAULT_CF e.g. + string cf = 3; - // is_delete represents whether we should delete the kv in tikv. - // it may not be too much delete file. only rollBack operation will generate delete kv file. - bool is_delete = 4; + // is_delete represents whether we should delete the kv in tikv. + // it may not be too much delete file. only rollBack operation will generate delete kv file. + bool is_delete = 4; - // the key ts space being smaller than start_ts can be filter. - uint64 start_ts = 10; + // the key ts space being smaller than start_ts can be filter. + uint64 start_ts = 10; - // the key ts space large than restore_ts can be filter. - uint64 restore_ts = 5; + // the key ts space large than restore_ts can be filter. + uint64 restore_ts = 5; - bytes start_key = 6; + bytes start_key = 6; - bytes end_key = 7; + bytes end_key = 7; - // used for checksum when download kv file. - bytes sha256 = 8; + // used for checksum when download kv file. + bytes sha256 = 8; - // the key ts space less than start_snapshot_ts can be filter. - // Deprecated: this field 'start_snapshot_ts' is replaced by the field 'start_ts'. - uint64 start_snapshot_ts = 9; + // the key ts space less than start_snapshot_ts can be filter. + // Deprecated: this field 'start_snapshot_ts' is replaced by the field 'start_ts'. + uint64 start_snapshot_ts = 9; - // the compression type for the file. - backup.CompressionType compression_type = 13; -} + // the compression type for the file. + backup.CompressionType compression_type = 13; + // encryption information of the kv file, not set if encryption is not enabled. + encryptionpb.FileEncryptionInfo file_encryption_info = 14; +} message ApplyRequest { - // The meta of the KV file. - KVMeta meta = 1; - repeated KVMeta metas = 12; - - // Performs a key prefix rewrite after downloading the file. - // All keys in the files will be rewritten as: - // - // new_key = new_key_prefix + old_key[len(old_key_prefix)..] - // - // When used for TiDB, rewriting the prefix changes the table ID. Please - // note that key-rewrite is applied on the origin keys in encoded - // representation. - // - // You need to ensure that the keys before and after rewriting are in the - // same order, otherwise the RPC request will fail. - RewriteRule rewrite_rule = 2 [(gogoproto.nullable) = false]; - repeated RewriteRule rewrite_rules = 13; - - // The identity for the stroage backend. - // When this field presents, the storage would be cached. - // If there is a cached storage, TiKV would use it driectly. - string storage_cache_id = 5; - backup.StorageBackend storage_backend = 3; - - // context represents region info and it used to build raft commands. - kvrpcpb.Context context = 4; - - // cipher_info is used to decrypt kv file when download file. - backup.CipherInfo cipher_info = 11; + // The meta of the KV file. + KVMeta meta = 1; + repeated KVMeta metas = 12; + + // Performs a key prefix rewrite after downloading the file. + // All keys in the files will be rewritten as: + // + // new_key = new_key_prefix + old_key[len(old_key_prefix)..] + // + // When used for TiDB, rewriting the prefix changes the table ID. Please + // note that key-rewrite is applied on the origin keys in encoded + // representation. + // + // You need to ensure that the keys before and after rewriting are in the + // same order, otherwise the RPC request will fail. + RewriteRule rewrite_rule = 2 [(gogoproto.nullable) = false]; + repeated RewriteRule rewrite_rules = 13; + + // The identity for the stroage backend. + // When this field presents, the storage would be cached. + // If there is a cached storage, TiKV would use it driectly. + string storage_cache_id = 5; + backup.StorageBackend storage_backend = 3; + + // context represents region info and it used to build raft commands. + kvrpcpb.Context context = 4; + + // plaintext data key to decrypt kv file if configured during log backup. + backup.CipherInfo cipher_info = 11; + + // master keys config used to decrypt data keys in restore if configured during log backup. + repeated encryptionpb.MasterKey master_keys = 14; } message ApplyResponse { - // The actual key range (after rewrite) of the downloaded file. The range is - // inclusive in both ends. - Range range = 1 [(gogoproto.nullable) = false]; + // The actual key range (after rewrite) of the downloaded file. The range is + // inclusive in both ends. + Range range = 1 [(gogoproto.nullable) = false]; - Error error = 2; + Error error = 2; } message ClearRequest { - // clear files in import directory with given prefix. - string prefix = 1; + // clear files in import directory with given prefix. + string prefix = 1; } message ClearResponse { - Error error = 1; + Error error = 1; +} + +message AddPartitionRangeRequest { + Range range = 1; + // the number of seconds this range is valid. + //After this duration, if this range is still not removed by + // `RemoveForcePartitionRange`, tikv will automically remove it. So the client + // should set a big enough value to avoid auto cleanup. But in general, 1h + // should be a big enough value. If its value is 0, tikv will auto adjust it to 3600(1h). + uint64 ttl_seconds = 2; +} + +message AddPartitionRangeResponse {} + +message RemovePartitionRangeRequest { + Range range = 1; } +message RemovePartitionRangeResponse {} diff --git a/proto/include/eraftpb.proto b/proto/include/eraftpb.proto index d9ec4b12..b952d81f 100644 --- a/proto/include/eraftpb.proto +++ b/proto/include/eraftpb.proto @@ -2,9 +2,9 @@ syntax = "proto3"; package eraftpb; enum EntryType { - EntryNormal = 0; - EntryConfChange = 1; - EntryConfChangeV2 = 2; + EntryNormal = 0; + EntryConfChange = 1; + EntryConfChangeV2 = 2; } // The entry is a type of change that needs to be applied. It contains two data fields. @@ -18,134 +18,134 @@ enum EntryType { // context will provide anything needed to assist the configuration change. The context // if for the user to set and use in this case. message Entry { - EntryType entry_type = 1; - uint64 term = 2; - uint64 index = 3; - bytes data = 4; - bytes context = 6; - - // Deprecated! It is kept for backward compatibility. - // TODO: remove it in the next major release. - bool sync_log = 5; + EntryType entry_type = 1; + uint64 term = 2; + uint64 index = 3; + bytes data = 4; + bytes context = 6; + + // Deprecated! It is kept for backward compatibility. + // TODO: remove it in the next major release. + bool sync_log = 5; } message SnapshotMetadata { - // The current `ConfState`. - ConfState conf_state = 1; - // The applied index. - uint64 index = 2; - // The term of the applied index. - uint64 term = 3; + // The current `ConfState`. + ConfState conf_state = 1; + // The applied index. + uint64 index = 2; + // The term of the applied index. + uint64 term = 3; } message Snapshot { - bytes data = 1; - SnapshotMetadata metadata = 2; + bytes data = 1; + SnapshotMetadata metadata = 2; } enum MessageType { - MsgHup = 0; - MsgBeat = 1; - MsgPropose = 2; - MsgAppend = 3; - MsgAppendResponse = 4; - MsgRequestVote = 5; - MsgRequestVoteResponse = 6; - MsgSnapshot = 7; - MsgHeartbeat = 8; - MsgHeartbeatResponse = 9; - MsgUnreachable = 10; - MsgSnapStatus = 11; - MsgCheckQuorum = 12; - MsgTransferLeader = 13; - MsgTimeoutNow = 14; - MsgReadIndex = 15; - MsgReadIndexResp = 16; - MsgRequestPreVote = 17; - MsgRequestPreVoteResponse = 18; + MsgHup = 0; + MsgBeat = 1; + MsgPropose = 2; + MsgAppend = 3; + MsgAppendResponse = 4; + MsgRequestVote = 5; + MsgRequestVoteResponse = 6; + MsgSnapshot = 7; + MsgHeartbeat = 8; + MsgHeartbeatResponse = 9; + MsgUnreachable = 10; + MsgSnapStatus = 11; + MsgCheckQuorum = 12; + MsgTransferLeader = 13; + MsgTimeoutNow = 14; + MsgReadIndex = 15; + MsgReadIndexResp = 16; + MsgRequestPreVote = 17; + MsgRequestPreVoteResponse = 18; } message Message { - MessageType msg_type = 1; - uint64 to = 2; - uint64 from = 3; - uint64 term = 4; - uint64 log_term = 5; - uint64 index = 6; - repeated Entry entries = 7; - uint64 commit = 8; - Snapshot snapshot = 9; - uint64 request_snapshot = 13; - bool reject = 10; - uint64 reject_hint = 11; - bytes context = 12; - uint64 deprecated_priority = 14; - // If this new field is not set, then use the above old field; otherwise - // use the new field. When broadcasting request vote, both fields are - // set if the priority is larger than 0. This change is not a fully - // compatible change, but it makes minimal impact that only new priority - // is not recognized by the old nodes during rolling update. - int64 priority = 15; + MessageType msg_type = 1; + uint64 to = 2; + uint64 from = 3; + uint64 term = 4; + uint64 log_term = 5; + uint64 index = 6; + repeated Entry entries = 7; + uint64 commit = 8; + Snapshot snapshot = 9; + uint64 request_snapshot = 13; + bool reject = 10; + uint64 reject_hint = 11; + bytes context = 12; + uint64 deprecated_priority = 14; + // If this new field is not set, then use the above old field; otherwise + // use the new field. When broadcasting request vote, both fields are + // set if the priority is larger than 0. This change is not a fully + // compatible change, but it makes minimal impact that only new priority + // is not recognized by the old nodes during rolling update. + int64 priority = 15; } message HardState { - uint64 term = 1; - uint64 vote = 2; - uint64 commit = 3; + uint64 term = 1; + uint64 vote = 2; + uint64 commit = 3; } enum ConfChangeTransition { - // Automatically use the simple protocol if possible, otherwise fall back - // to ConfChangeType::Implicit. Most applications will want to use this. - Auto = 0; - // Use joint consensus unconditionally, and transition out of them - // automatically (by proposing a zero configuration change). - // - // This option is suitable for applications that want to minimize the time - // spent in the joint configuration and do not store the joint configuration - // in the state machine (outside of InitialState). - Implicit = 1; - // Use joint consensus and remain in the joint configuration until the - // application proposes a no-op configuration change. This is suitable for - // applications that want to explicitly control the transitions, for example - // to use a custom payload (via the Context field). - Explicit = 2; + // Automatically use the simple protocol if possible, otherwise fall back + // to ConfChangeType::Implicit. Most applications will want to use this. + Auto = 0; + // Use joint consensus unconditionally, and transition out of them + // automatically (by proposing a zero configuration change). + // + // This option is suitable for applications that want to minimize the time + // spent in the joint configuration and do not store the joint configuration + // in the state machine (outside of InitialState). + Implicit = 1; + // Use joint consensus and remain in the joint configuration until the + // application proposes a no-op configuration change. This is suitable for + // applications that want to explicitly control the transitions, for example + // to use a custom payload (via the Context field). + Explicit = 2; } message ConfState { - repeated uint64 voters = 1; - repeated uint64 learners = 2; - - // The voters in the outgoing config. If not empty the node is in joint consensus. - repeated uint64 voters_outgoing = 3; - // The nodes that will become learners when the outgoing config is removed. - // These nodes are necessarily currently in nodes_joint (or they would have - // been added to the incoming config right away). - repeated uint64 learners_next = 4; - // If set, the config is joint and Raft will automatically transition into - // the final config (i.e. remove the outgoing config) when this is safe. - bool auto_leave = 5; + repeated uint64 voters = 1; + repeated uint64 learners = 2; + + // The voters in the outgoing config. If not empty the node is in joint consensus. + repeated uint64 voters_outgoing = 3; + // The nodes that will become learners when the outgoing config is removed. + // These nodes are necessarily currently in nodes_joint (or they would have + // been added to the incoming config right away). + repeated uint64 learners_next = 4; + // If set, the config is joint and Raft will automatically transition into + // the final config (i.e. remove the outgoing config) when this is safe. + bool auto_leave = 5; } enum ConfChangeType { - AddNode = 0; - RemoveNode = 1; - AddLearnerNode = 2; + AddNode = 0; + RemoveNode = 1; + AddLearnerNode = 2; } message ConfChange { - ConfChangeType change_type = 2; - uint64 node_id = 3; - bytes context = 4; + ConfChangeType change_type = 2; + uint64 node_id = 3; + bytes context = 4; - uint64 id = 1; + uint64 id = 1; } // ConfChangeSingle is an individual configuration change operation. Multiple // such operations can be carried out atomically via a ConfChangeV2. message ConfChangeSingle { - ConfChangeType change_type = 1; - uint64 node_id = 2; + ConfChangeType change_type = 1; + uint64 node_id = 2; } // ConfChangeV2 messages initiate configuration changes. They support both the @@ -181,7 +181,7 @@ message ConfChangeSingle { // // [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf message ConfChangeV2 { - ConfChangeTransition transition = 1; - repeated ConfChangeSingle changes = 2; - bytes context = 3; + ConfChangeTransition transition = 1; + repeated ConfChangeSingle changes = 2; + bytes context = 3; } diff --git a/proto/include/gogoproto/gogo.proto b/proto/include/gogoproto/gogo.proto index bc8d889f..b80c8565 100644 --- a/proto/include/gogoproto/gogo.proto +++ b/proto/include/gogoproto/gogo.proto @@ -84,6 +84,9 @@ extend google.protobuf.FileOptions { optional bool goproto_registration = 63032; optional bool messagename_all = 63033; + + optional bool goproto_sizecache_all = 63034; + optional bool goproto_unkeyed_all = 63035; } extend google.protobuf.MessageOptions { @@ -118,6 +121,9 @@ extend google.protobuf.MessageOptions { optional bool typedecl = 64030; optional bool messagename = 64033; + + optional bool goproto_sizecache = 64034; + optional bool goproto_unkeyed = 64035; } extend google.protobuf.FieldOptions { @@ -133,4 +139,6 @@ extend google.protobuf.FieldOptions { optional bool stdtime = 65010; optional bool stdduration = 65011; + optional bool wktpointer = 65012; + } diff --git a/proto/include/rustproto.proto b/proto/include/rustproto.proto index 83e76fdf..1bc53345 100644 --- a/proto/include/rustproto.proto +++ b/proto/include/rustproto.proto @@ -1,47 +1,47 @@ syntax = "proto2"; -import "google/protobuf/descriptor.proto"; - // see https://github.com/gogo/protobuf/blob/master/gogoproto/gogo.proto // for the original idea package rustproto; +import "google/protobuf/descriptor.proto"; + extend google.protobuf.FileOptions { - // When true, oneof field is generated public - optional bool expose_oneof_all = 17001; - // When true all fields are public, and not accessors generated - optional bool expose_fields_all = 17003; - // When false, `get_`, `set_`, `mut_` etc. accessors are not generated - optional bool generate_accessors_all = 17004; - // Use `bytes::Bytes` for `bytes` fields - optional bool carllerche_bytes_for_bytes_all = 17011; - // Use `bytes::Bytes` for `string` fields - optional bool carllerche_bytes_for_string_all = 17012; - // When true, will only generate codes that works with lite runtime. - optional bool lite_runtime_all = 17035; + // When true, oneof field is generated public + optional bool expose_oneof_all = 17001; + // When true all fields are public, and not accessors generated + optional bool expose_fields_all = 17003; + // When false, `get_`, `set_`, `mut_` etc. accessors are not generated + optional bool generate_accessors_all = 17004; + // Use `bytes::Bytes` for `bytes` fields + optional bool carllerche_bytes_for_bytes_all = 17011; + // Use `bytes::Bytes` for `string` fields + optional bool carllerche_bytes_for_string_all = 17012; + // When true, will only generate codes that works with lite runtime. + optional bool lite_runtime_all = 17035; } extend google.protobuf.MessageOptions { - // When true, oneof field is generated public - optional bool expose_oneof = 17001; - // When true all fields are public, and not accessors generated - optional bool expose_fields = 17003; - // When false, `get_`, `set_`, `mut_` etc. accessors are not generated - optional bool generate_accessors = 17004; - // Use `bytes::Bytes` for `bytes` fields - optional bool carllerche_bytes_for_bytes = 17011; - // Use `bytes::Bytes` for `string` fields - optional bool carllerche_bytes_for_string = 17012; + // When true, oneof field is generated public + optional bool expose_oneof = 17001; + // When true all fields are public, and not accessors generated + optional bool expose_fields = 17003; + // When false, `get_`, `set_`, `mut_` etc. accessors are not generated + optional bool generate_accessors = 17004; + // Use `bytes::Bytes` for `bytes` fields + optional bool carllerche_bytes_for_bytes = 17011; + // Use `bytes::Bytes` for `string` fields + optional bool carllerche_bytes_for_string = 17012; } extend google.protobuf.FieldOptions { - // When true all fields are public, and not accessors generated - optional bool expose_fields_field = 17003; - // When false, `get_`, `set_`, `mut_` etc. accessors are not generated - optional bool generate_accessors_field = 17004; - // Use `bytes::Bytes` for `bytes` fields - optional bool carllerche_bytes_for_bytes_field = 17011; - // Use `bytes::Bytes` for `string` fields - optional bool carllerche_bytes_for_string_field = 17012; -} \ No newline at end of file + // When true all fields are public, and not accessors generated + optional bool expose_fields_field = 17003; + // When false, `get_`, `set_`, `mut_` etc. accessors are not generated + optional bool generate_accessors_field = 17004; + // Use `bytes::Bytes` for `bytes` fields + optional bool carllerche_bytes_for_bytes_field = 17011; + // Use `bytes::Bytes` for `string` fields + optional bool carllerche_bytes_for_string_field = 17012; +} diff --git a/proto/keyspacepb.proto b/proto/keyspacepb.proto index e817e980..033c01e5 100644 --- a/proto/keyspacepb.proto +++ b/proto/keyspacepb.proto @@ -1,24 +1,26 @@ syntax = "proto3"; package keyspacepb; -import "pdpb.proto"; - import "gogoproto/gogo.proto"; +import "pdpb.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // Keyspace provides services to manage keyspaces. service Keyspace { - rpc LoadKeyspace (LoadKeyspaceRequest) returns (LoadKeyspaceResponse) {} + rpc LoadKeyspace(LoadKeyspaceRequest) returns (LoadKeyspaceResponse) {} + rpc LoadKeyspaceByID(LoadKeyspaceByIDRequest) returns (LoadKeyspaceResponse) {} // WatchKeyspaces first return all current keyspaces' metadata as its first response. // Then, it returns responses containing keyspaces that had their metadata changed. - rpc WatchKeyspaces (WatchKeyspacesRequest) returns (stream WatchKeyspacesResponse) {} + rpc WatchKeyspaces(WatchKeyspacesRequest) returns (stream WatchKeyspacesResponse) {} rpc UpdateKeyspaceState(UpdateKeyspaceStateRequest) returns (UpdateKeyspaceStateResponse) {} rpc GetAllKeyspaces(GetAllKeyspacesRequest) returns (GetAllKeyspacesResponse) {} } @@ -44,6 +46,11 @@ message LoadKeyspaceRequest { string name = 2; } +message LoadKeyspaceByIDRequest { + pdpb.RequestHeader header = 1; + uint32 id = 2; +} + message LoadKeyspaceResponse { pdpb.ResponseHeader header = 1; KeyspaceMeta keyspace = 2; @@ -58,24 +65,24 @@ message WatchKeyspacesResponse { repeated KeyspaceMeta keyspaces = 2; } -message UpdateKeyspaceStateRequest{ +message UpdateKeyspaceStateRequest { pdpb.RequestHeader header = 1; uint32 id = 2; KeyspaceState state = 3; } -message UpdateKeyspaceStateResponse{ +message UpdateKeyspaceStateResponse { pdpb.ResponseHeader header = 1; KeyspaceMeta keyspace = 2; } -message GetAllKeyspacesRequest{ +message GetAllKeyspacesRequest { pdpb.RequestHeader header = 1; uint32 start_id = 2; uint32 limit = 3; } -message GetAllKeyspacesResponse{ +message GetAllKeyspacesResponse { pdpb.ResponseHeader header = 1; repeated KeyspaceMeta keyspaces = 2; } diff --git a/proto/kvrpcpb.proto b/proto/kvrpcpb.proto index 288ff39e..d2398e8d 100644 --- a/proto/kvrpcpb.proto +++ b/proto/kvrpcpb.proto @@ -1,21 +1,23 @@ syntax = "proto3"; package kvrpcpb; -import "metapb.proto"; +import "deadlock.proto"; import "errorpb.proto"; import "gogoproto/gogo.proto"; +import "metapb.proto"; +import "resource_manager.proto"; import "rustproto.proto"; -import "deadlock.proto"; import "tracepb.proto"; -import "resource_manager.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // This proto file defines requests, responses, and helper messages for KV and raw // APIs of TiKV (see tikvpb.proto). @@ -24,268 +26,280 @@ option java_package = "org.tikv.kvproto"; // A transactional get command. Lookup a value for `key` in the transaction with // starting timestamp = `version`. message GetRequest { - Context context = 1; - bytes key = 2; - uint64 version = 3; + Context context = 1; + bytes key = 2; + uint64 version = 3; + // If true, the response will include the commit ts of the key. + bool need_commit_ts = 4; } message GetResponse { - // A region error indicates that the request was sent to the wrong TiKV node - // (or other, similar errors). - errorpb.Error region_error = 1; - // A value could not be retrieved due to the state of the database for the requested key. - KeyError error = 2; - // A successful result. - bytes value = 3; - // True if the key does not exist in the database. - bool not_found = 4; - reserved 5; - // Time and scan details when processing the request. - ExecDetailsV2 exec_details_v2 = 6; + // A region error indicates that the request was sent to the wrong TiKV node + // (or other, similar errors). + errorpb.Error region_error = 1; + // A value could not be retrieved due to the state of the database for the requested key. + KeyError error = 2; + // A successful result. + bytes value = 3; + // True if the key does not exist in the database. + bool not_found = 4; + reserved 5; + // Time and scan details when processing the request. + ExecDetailsV2 exec_details_v2 = 6; + // The commit timestamp of the key. + // If it is zero, it means the commit timestamp is unknown. + uint64 commit_ts = 7; } // Scan fetches values for a range of keys; it is part of the transaction with // starting timestamp = `version`. message ScanRequest { - Context context = 1; - bytes start_key = 2; - // The maximum number of results to return. - uint32 limit = 3; - uint64 version = 4; - // Return only the keys found by scanning, not their values. - bool key_only = 5; - bool reverse = 6; - // For compatibility, when scanning forward, the range to scan is [start_key, end_key), where start_key < end_key; - // and when scanning backward, it scans [end_key, start_key) in descending order, where end_key < start_key. - bytes end_key = 7; - // If sample_step > 0, skips 'sample_step - 1' number of keys after each returned key. - // locks are not checked. - uint32 sample_step = 8; + Context context = 1; + bytes start_key = 2; + // The maximum number of results to return. + uint32 limit = 3; + uint64 version = 4; + // Return only the keys found by scanning, not their values. + bool key_only = 5; + bool reverse = 6; + // For compatibility, when scanning forward, the range to scan is [start_key, end_key), where start_key < end_key; + // and when scanning backward, it scans [end_key, start_key) in descending order, where end_key < start_key. + bytes end_key = 7; + // If sample_step > 0, skips 'sample_step - 1' number of keys after each returned key. + // locks are not checked. + uint32 sample_step = 8; } message ScanResponse { - errorpb.Error region_error = 1; - // Each KvPair may contain a key error. - repeated KvPair pairs = 2; - // This KeyError exists when some key is locked but we cannot check locks of all keys. - // In this case, `pairs` should be empty and the client should redo scanning all the keys - // after resolving the lock. - KeyError error = 3; + errorpb.Error region_error = 1; + // Each KvPair may contain a key error. + repeated KvPair pairs = 2; + // This KeyError exists when some key is locked but we cannot check locks of all keys. + // In this case, `pairs` should be empty and the client should redo scanning all the keys + // after resolving the lock. + KeyError error = 3; } // A prewrite is the first phase of writing to TiKV. It contains all data to be written in a transaction. // TiKV will write the data in a preliminary state. Data cannot be read until it has been committed. // The client should only commit a transaction once all prewrites succeed. message PrewriteRequest { - // What kind of checks need to be performed for keys in a pessimistic transaction. - enum PessimisticAction { - // The key needn't be locked and no extra write conflict checks are needed. - SKIP_PESSIMISTIC_CHECK = 0; - // The key should have been locked at the time of prewrite. - DO_PESSIMISTIC_CHECK = 1; - // The key doesn't need a pessimistic lock. But we need to do data constraint checks. - DO_CONSTRAINT_CHECK = 2; - } - - // for_update_ts constriants that should be checked when prewriting a pessimistic transaction. - message ForUpdateTSConstraint { - // The index of key in the prewrite request that should be checked. - uint32 index = 1; - // The expected for_update_ts of the pessimistic lock of the key. - uint64 expected_for_update_ts = 2; - } - - Context context = 1; - // The data to be written to the database. - repeated Mutation mutations = 2; - // The client picks one key to be primary (unrelated to the primary key concept in SQL). This - // key's lock is the source of truth for the state of a transaction. All other locks due to a - // transaction will point to the primary lock. - bytes primary_lock = 3; - // Identifies the transaction being written. - uint64 start_version = 4; - uint64 lock_ttl = 5; - // TiKV can skip some checks, used for speeding up data migration. - bool skip_constraint_check = 6; - // For pessimistic transaction, some mutations don't need to be locked, for example, non-unique index key. - // Keys with deferred constraint checks are not locked. - repeated PessimisticAction pessimistic_actions = 7; - // How many keys this transaction involves in this region. - uint64 txn_size = 8; - // For pessimistic transactions only; used to check if a conflict lock is already committed. - uint64 for_update_ts = 9; - // If min_commit_ts > 0, this is a large transaction request, the final commit_ts - // will be inferred from `min_commit_ts`. - uint64 min_commit_ts = 10; - // When async commit is enabled, `secondaries` should be set as the key list of all secondary - // locks if the request prewrites the primary lock. - bool use_async_commit = 11; - repeated bytes secondaries = 12; - // When the transaction involves only one region, it's possible to commit the transaction - // directly with 1PC protocol. - bool try_one_pc = 13; - // The max commit ts is reserved for limiting the commit ts of 1PC or async commit, which can be used to avoid - // inconsistency with schema change. - uint64 max_commit_ts = 14; - // The level of assertion to use on this prewrte request. - AssertionLevel assertion_level = 15; - // for_update_ts constriants that should be checked when prewriting a pessimistic transaction. - // See https://github.com/tikv/tikv/issues/14311 - repeated ForUpdateTSConstraint for_update_ts_constraints = 16; - - // Reserved for file based transaction. - repeated uint64 txn_file_chunks = 100; + // What kind of checks need to be performed for keys in a pessimistic transaction. + enum PessimisticAction { + // The key needn't be locked and no extra write conflict checks are needed. + // Deprecated in next-gen (cloud-storage-engine). + SKIP_PESSIMISTIC_CHECK = 0; + // The key should have been locked at the time of prewrite. If the lock is missing, + // the lock will be amended. This is the normal case for pessimistic transactions. + DO_PESSIMISTIC_CHECK = 1; + // The key does not acquire a pessimistic lock for performance optimization. + // Constraint checking (write conflicts + data constraints) is deferred to prewrite. + DO_CONSTRAINT_CHECK = 2; + } + + // for_update_ts constriants that should be checked when prewriting a pessimistic transaction. + message ForUpdateTSConstraint { + // The index of key in the prewrite request that should be checked. + uint32 index = 1; + // The expected for_update_ts of the pessimistic lock of the key. + uint64 expected_for_update_ts = 2; + } + + Context context = 1; + // The data to be written to the database. + repeated Mutation mutations = 2; + // The client picks one key to be primary (unrelated to the primary key concept in SQL). This + // key's lock is the source of truth for the state of a transaction. All other locks due to a + // transaction will point to the primary lock. + bytes primary_lock = 3; + // Identifies the transaction being written. + uint64 start_version = 4; + uint64 lock_ttl = 5; + // TiKV can skip some checks, used for speeding up data migration. + bool skip_constraint_check = 6; + // For pessimistic transaction, some mutations don't need to be locked, for example, non-unique index key. + // Keys with deferred constraint checks are not locked. + repeated PessimisticAction pessimistic_actions = 7; + // How many keys this transaction involves in this region. + uint64 txn_size = 8; + // For pessimistic transactions only; used to check if a conflict lock is already committed. + uint64 for_update_ts = 9; + // If min_commit_ts > 0, this is a large transaction request, the final commit_ts + // will be inferred from `min_commit_ts`. + uint64 min_commit_ts = 10; + // When async commit is enabled, `secondaries` should be set as the key list of all secondary + // locks if the request prewrites the primary lock. + bool use_async_commit = 11; + repeated bytes secondaries = 12; + // When the transaction involves only one region, it's possible to commit the transaction + // directly with 1PC protocol. + bool try_one_pc = 13; + // The max commit ts is reserved for limiting the commit ts of 1PC or async commit, which can be used to avoid + // inconsistency with schema change. + uint64 max_commit_ts = 14; + // The level of assertion to use on this prewrte request. + AssertionLevel assertion_level = 15; + // for_update_ts constriants that should be checked when prewriting a pessimistic transaction. + // See https://github.com/tikv/tikv/issues/14311 + repeated ForUpdateTSConstraint for_update_ts_constraints = 16; + + // Reserved for file based transaction. + repeated uint64 txn_file_chunks = 100; } message PrewriteResponse { - errorpb.Error region_error = 1; - repeated KeyError errors = 2; - // 0 if the min_commit_ts is not ready or any other reason that async - // commit cannot proceed. The client can then fallback to normal way to - // continue committing the transaction if prewrite are all finished. - uint64 min_commit_ts = 3; - // When the transaction is successfully committed with 1PC protocol, this field will be set to - // the commit ts of the transaction. Otherwise, if TiKV failed to commit it with 1PC or the - // transaction is not 1PC, the value will be 0. - uint64 one_pc_commit_ts = 4; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 5; + errorpb.Error region_error = 1; + repeated KeyError errors = 2; + // 0 if the min_commit_ts is not ready or any other reason that async + // commit cannot proceed. The client can then fallback to normal way to + // continue committing the transaction if prewrite are all finished. + uint64 min_commit_ts = 3; + // When the transaction is successfully committed with 1PC protocol, this field will be set to + // the commit ts of the transaction. Otherwise, if TiKV failed to commit it with 1PC or the + // transaction is not 1PC, the value will be 0. + uint64 one_pc_commit_ts = 4; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 5; } // Used to specify the behavior when a pessimistic lock request is woken up after waiting for another // lock. enum PessimisticLockWakeUpMode { - // When woken up, returns WriteConflict error to the client and the client should retry if necessary. - // In this mode, results of `return_values` or `check_existence` will be set to `values` and `not_founds` - // fields of the PessimisticLockResponse, which is compatible with old versions. - WakeUpModeNormal = 0; - // When woken up, continue trying to lock the key. This implicitly enables the `allow_lock_with_conflict` - // behavior, which means, allow acquiring the lock even if there is WriteConflict on the key. - // In this mode, `return_values` or `check_existence` fields of PessimisticLockResponse won't be used, and - // all results are carried in the `results` field. - WakeUpModeForceLock = 1; + // When woken up, returns WriteConflict error to the client and the client should retry if necessary. + // In this mode, results of `return_values` or `check_existence` will be set to `values` and `not_founds` + // fields of the PessimisticLockResponse, which is compatible with old versions. + WakeUpModeNormal = 0; + // When woken up, continue trying to lock the key. This implicitly enables the `allow_lock_with_conflict` + // behavior, which means, allow acquiring the lock even if there is WriteConflict on the key. + // In this mode, `return_values` or `check_existence` fields of PessimisticLockResponse won't be used, and + // all results are carried in the `results` field. + WakeUpModeForceLock = 1; } // Lock a set of keys to prepare to write to them. message PessimisticLockRequest { - Context context = 1; - // In this case every `Op` of the mutations must be `PessimisticLock`. - repeated Mutation mutations = 2; - bytes primary_lock = 3; - uint64 start_version = 4; - uint64 lock_ttl = 5; - // Each locking command in a pessimistic transaction has its own timestamp. If locking fails, then - // the corresponding SQL statement can be retried with a later timestamp, TiDB does not need to - // retry the whole transaction. The name comes from the `SELECT ... FOR UPDATE` SQL statement which - // is a locking read. Each `SELECT ... FOR UPDATE` in a transaction will be assigned its own - // timestamp. - uint64 for_update_ts = 6; - // If the request is the first lock request, we don't need to detect deadlock. - bool is_first_lock = 7; - // Time to wait for lock released in milliseconds when encountering locks. - // 0 means using default timeout in TiKV. Negative means no wait. - int64 wait_timeout = 8; - // If it is true, TiKV will acquire the pessimistic lock regardless of write conflict - // and return the latest value. It's only supported for single mutation. - bool force = 9 [deprecated = true]; - // If it is true, TiKV will return values of the keys if no error, so TiDB can cache the values for - // later read in the same transaction. - // When 'force' is set to true, this field is ignored. - bool return_values = 10; - // If min_commit_ts > 0, this is large transaction proto, the final commit_ts - // would be infered from min_commit_ts. - uint64 min_commit_ts = 11; - // If set to true, it means TiKV need to check if the key exists, and return the result in - // the `not_founds` feild in the response. This works no matter if `return_values` is set. If - // `return_values` is set, it simply makes no difference; otherwise, the `value` field of the - // repsonse will be empty while the `not_founds` field still indicates the keys' existence. - bool check_existence = 12; - // TiKV lock the record only when it exists - bool lock_only_if_exists = 13; - // Specifies the behavior when the request is woken up after wating for lock of another transaction. - PessimisticLockWakeUpMode wake_up_mode = 14; + Context context = 1; + // In this case every `Op` of the mutations must be `PessimisticLock`. + repeated Mutation mutations = 2; + bytes primary_lock = 3; + uint64 start_version = 4; + uint64 lock_ttl = 5; + // Each locking command in a pessimistic transaction has its own timestamp. If locking fails, then + // the corresponding SQL statement can be retried with a later timestamp, TiDB does not need to + // retry the whole transaction. The name comes from the `SELECT ... FOR UPDATE` SQL statement which + // is a locking read. Each `SELECT ... FOR UPDATE` in a transaction will be assigned its own + // timestamp. + uint64 for_update_ts = 6; + // If the request is the first lock request, we don't need to detect deadlock. + bool is_first_lock = 7; + // Time to wait for lock released in milliseconds when encountering locks. + // 0 means using default timeout in TiKV. Negative means no wait. + int64 wait_timeout = 8; + // If it is true, TiKV will acquire the pessimistic lock regardless of write conflict + // and return the latest value. It's only supported for single mutation. + bool force = 9 [deprecated = true]; + // If it is true, TiKV will return values of the keys if no error, so TiDB can cache the values for + // later read in the same transaction. + // When 'force' is set to true, this field is ignored. + bool return_values = 10; + // If min_commit_ts > 0, this is large transaction proto, the final commit_ts + // would be infered from min_commit_ts. + uint64 min_commit_ts = 11; + // If set to true, it means TiKV need to check if the key exists, and return the result in + // the `not_founds` feild in the response. This works no matter if `return_values` is set. If + // `return_values` is set, it simply makes no difference; otherwise, the `value` field of the + // repsonse will be empty while the `not_founds` field still indicates the keys' existence. + bool check_existence = 12; + // TiKV lock the record only when it exists + bool lock_only_if_exists = 13; + // Specifies the behavior when the request is woken up after wating for lock of another transaction. + PessimisticLockWakeUpMode wake_up_mode = 14; } enum PessimisticLockKeyResultType { - LockResultNormal = 0; - LockResultLockedWithConflict = 1; - LockResultFailed = 2; + LockResultNormal = 0; + LockResultLockedWithConflict = 1; + LockResultFailed = 2; } message PessimisticLockKeyResult { - PessimisticLockKeyResultType type = 1; - bytes value = 2; - bool existence = 3; - // We allow a key be locked when there is write conflict (latest commit_ts > for_update_ts). - // In this case, the key is semantically locked by a newer for_update_ts. - // For each requested key, the field is non-zero if the key is locked with write conflict, and it - // equals to the commit_ts of the latest version of the specified key. The for_update_ts field - // of the lock that's actually written to TiKV will also be this value. At the same time, - // `value` and `existence` will be returned regardless to how `return_values` and - // `check_existence` are set. - uint64 locked_with_conflict_ts = 4; - // Hint the client that resolving lock is not needed for this lock. For `PessimisticLock` - // requests only. - bool skip_resolving_lock = 11; + PessimisticLockKeyResultType type = 1; + bytes value = 2; + bool existence = 3; + // We allow a key be locked when there is write conflict (latest commit_ts > for_update_ts). + // In this case, the key is semantically locked by a newer for_update_ts. + // For each requested key, the field is non-zero if the key is locked with write conflict, and it + // equals to the commit_ts of the latest version of the specified key. The for_update_ts field + // of the lock that's actually written to TiKV will also be this value. At the same time, + // `value` and `existence` will be returned regardless to how `return_values` and + // `check_existence` are set. + uint64 locked_with_conflict_ts = 4; + // Hint the client that resolving lock is not needed for this lock. For `PessimisticLock` + // requests only. + bool skip_resolving_lock = 11; } message PessimisticLockResponse { - errorpb.Error region_error = 1; - repeated KeyError errors = 2; - // It carries the latest value and its commit ts if force in PessimisticLockRequest is true. - uint64 commit_ts = 3 [deprecated = true]; - bytes value = 4 [deprecated = true]; - // The values is set if 'return_values' is true in the request and no error. - // If 'force' is true, this field is not used. - // Only used when `wake_up_mode` is `WakeUpModeNormal`. - repeated bytes values = 5; - // Indicates whether the values at the same index is correspond to an existing key. - // In legacy TiKV, this field is not used even 'force' is false. In that case, an empty value indicates - // two possible situations: (1) the key does not exist. (2) the key exists but the value is empty. - // Only used when `wake_up_mode` is `WakeUpModeNormal`. - repeated bool not_founds = 6; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 7; - // Results of the request. Only used when `wake_up_mode` is `WakeUpModeForceLock`. - repeated PessimisticLockKeyResult results = 8; + errorpb.Error region_error = 1; + repeated KeyError errors = 2; + // It carries the latest value and its commit ts if force in PessimisticLockRequest is true. + uint64 commit_ts = 3 [deprecated = true]; + bytes value = 4 [deprecated = true]; + // The values is set if 'return_values' is true in the request and no error. + // If 'force' is true, this field is not used. + // Only used when `wake_up_mode` is `WakeUpModeNormal`. + repeated bytes values = 5; + // Indicates whether the values at the same index is correspond to an existing key. + // In legacy TiKV, this field is not used even 'force' is false. In that case, an empty value indicates + // two possible situations: (1) the key does not exist. (2) the key exists but the value is empty. + // Only used when `wake_up_mode` is `WakeUpModeNormal`. + repeated bool not_founds = 6; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 7; + // Results of the request. Only used when `wake_up_mode` is `WakeUpModeForceLock`. + repeated PessimisticLockKeyResult results = 8; } // Unlock keys locked using `PessimisticLockRequest`. message PessimisticRollbackRequest { - Context context = 1; - uint64 start_version = 2; - uint64 for_update_ts = 3; - repeated bytes keys = 4; + Context context = 1; + uint64 start_version = 2; + uint64 for_update_ts = 3; + repeated bytes keys = 4; } message PessimisticRollbackResponse { - errorpb.Error region_error = 1; - repeated KeyError errors = 2; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 3; + errorpb.Error region_error = 1; + repeated KeyError errors = 2; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 3; } // Used to update the lock_ttl of a psessimistic and/or large transaction to prevent it from been killed. message TxnHeartBeatRequest { - Context context = 1; - // The key of the lock to update. - bytes primary_lock = 2; - // Start timestamp of the large transaction. - uint64 start_version = 3; - // The new TTL the sender would like. - uint64 advise_lock_ttl = 4; - - // Reserved for file based transaction. - bool is_txn_file = 100; + Context context = 1; + // The key of the lock to update. + bytes primary_lock = 2; + // Start timestamp of the large transaction. + uint64 start_version = 3; + // The new TTL the sender would like. + uint64 advise_lock_ttl = 4; + // Optionally update PK's min_commit_ts. + // Only for non-async-commmit and non-1PC transactions. + // If it is 0, ignore this field. + uint64 min_commit_ts = 5; + + // Reserved for file based transaction. + bool is_txn_file = 100; } message TxnHeartBeatResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // The TTL actually set on the requested lock. - uint64 lock_ttl = 3; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 4; + errorpb.Error region_error = 1; + KeyError error = 2; + // The TTL actually set on the requested lock. + uint64 lock_ttl = 3; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 4; } // CheckTxnStatusRequest checks the status of a transaction. @@ -294,1211 +308,1366 @@ message TxnHeartBeatResponse { // Otherwise, returns the TTL information for the transaction. // CheckTxnStatusRequest may also push forward the minCommitTS of a large transaction. message CheckTxnStatusRequest { - Context context = 1; - // Primary key and lock ts together to locate the primary lock of a transaction. - bytes primary_key = 2; - // Starting timestamp of the transaction being checked. - uint64 lock_ts = 3; - // The start timestamp of the transaction which this request is part of. - uint64 caller_start_ts = 4; - // The client must specify the current time to TiKV using this timestamp. It is used to check TTL - // timeouts. It may be inaccurate. - uint64 current_ts = 5; - // If true, then TiKV will leave a rollback tombstone in the write CF for `primary_key`, even if - // that key is not locked. - bool rollback_if_not_exist = 6; - // This field is set to true only if the transaction is known to fall back from async commit. - // Then, CheckTxnStatus treats the transaction as non-async-commit even if the use_async_commit - // field in the primary lock is true. - bool force_sync_commit = 7; - // If the check request is used to resolve or decide the transaction status for a input pessimistic - // lock, the transaction status could not be decided if the primary lock is pessimistic too and - // it's still uncertain. - bool resolving_pessimistic_lock = 8; - // Whether it's needed to check if the lock on the key (if any) is the primary lock. - // This is for handling some corner cases when a pessimistic transaction changes its primary - // (see https://github.com/pingcap/tidb/issues/42937 for details). This field is necessary - // because the old versions of clients cannot handle some results returned from TiKV correctly. - // For new versions, this field should always be set to true. - bool verify_is_primary = 9; - - // Reserved for file based transaction. - bool is_txn_file = 100; + Context context = 1; + // Primary key and lock ts together to locate the primary lock of a transaction. + bytes primary_key = 2; + // Starting timestamp of the transaction being checked. + uint64 lock_ts = 3; + // The start timestamp of the transaction which this request is part of. + uint64 caller_start_ts = 4; + // The client must specify the current time to TiKV using this timestamp. It is used to check TTL + // timeouts. It may be inaccurate. + uint64 current_ts = 5; + // If true, then TiKV will leave a rollback tombstone in the write CF for `primary_key`, even if + // that key is not locked. + bool rollback_if_not_exist = 6; + // This field is set to true only if the transaction is known to fall back from async commit. + // Then, CheckTxnStatus treats the transaction as non-async-commit even if the use_async_commit + // field in the primary lock is true. + bool force_sync_commit = 7; + // If the check request is used to resolve or decide the transaction status for a input pessimistic + // lock, the transaction status could not be decided if the primary lock is pessimistic too and + // it's still uncertain. + bool resolving_pessimistic_lock = 8; + // Whether it's needed to check if the lock on the key (if any) is the primary lock. + // This is for handling some corner cases when a pessimistic transaction changes its primary + // (see https://github.com/pingcap/tidb/issues/42937 for details). This field is necessary + // because the old versions of clients cannot handle some results returned from TiKV correctly. + // For new versions, this field should always be set to true. + bool verify_is_primary = 9; + + // Reserved for file based transaction. + bool is_txn_file = 100; } message CheckTxnStatusResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Three kinds of transaction status: - // locked: lock_ttl > 0 - // committed: commit_version > 0 - // rollbacked: lock_ttl = 0 && commit_version = 0 - uint64 lock_ttl = 3; - uint64 commit_version = 4; - // The action performed by TiKV (and why if the action is to rollback). - Action action = 5; - LockInfo lock_info = 6; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 7; + errorpb.Error region_error = 1; + KeyError error = 2; + // Three kinds of transaction status: + // locked: lock_ttl > 0 + // committed: commit_version > 0 + // rollbacked: lock_ttl = 0 && commit_version = 0 + uint64 lock_ttl = 3; + uint64 commit_version = 4; + // The action performed by TiKV (and why if the action is to rollback). + Action action = 5; + LockInfo lock_info = 6; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 7; } // Part of the async commit protocol, checks for locks on all supplied keys. If a lock is missing, // does not have a successful status, or belongs to another transaction, TiKV will leave a rollback // tombstone for that key. message CheckSecondaryLocksRequest { - Context context = 1; - repeated bytes keys = 2; - // Identifies the transaction we are investigating. - uint64 start_version = 3; + Context context = 1; + repeated bytes keys = 2; + // Identifies the transaction we are investigating. + uint64 start_version = 3; } message CheckSecondaryLocksResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // For each key in `keys` in `CheckSecondaryLocks`, there will be a lock in - // this list if there is a lock present and belonging to the correct transaction, - // nil otherwise. - repeated LockInfo locks = 3; - // If any of the locks have been committed, this is the commit ts used. If no - // locks have been committed, it will be zero. - uint64 commit_ts = 4; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 5; + errorpb.Error region_error = 1; + KeyError error = 2; + // For each key in `keys` in `CheckSecondaryLocks`, there will be a lock in + // this list if there is a lock present and belonging to the correct transaction, + // nil otherwise. + repeated LockInfo locks = 3; + // If any of the locks have been committed, this is the commit ts used. If no + // locks have been committed, it will be zero. + uint64 commit_ts = 4; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 5; +} + +// CommitRole indicates the current commit request is a primary commit or a secondary commit. +enum CommitRole { + // Unknown about the current request is a primary commit or a secondary commit; + Unknown = 0; + // Current request is a primary commit; + Primary = 1; + // Current request is a secondary commit; + Secondary = 2; } // The second phase of writing to TiKV. If there are no errors or conflicts, then this request // commits a transaction so that its data can be read by other transactions. message CommitRequest { - reserved 5; - reserved "binlog"; - Context context = 1; - // Identifies the transaction. - uint64 start_version = 2; - // All keys in the transaction (to be committed). - repeated bytes keys = 3; - // Timestamp for the end of the transaction. Must be greater than `start_version`. - uint64 commit_version = 4; - - // Reserved for file based transaction. - bool is_txn_file = 100; + reserved 5; + reserved "binlog"; + Context context = 1; + // Identifies the transaction. + uint64 start_version = 2; + // All keys in the transaction (to be committed). + repeated bytes keys = 3; + // Timestamp for the end of the transaction. Must be greater than `start_version`. + uint64 commit_version = 4; + // commit_role indicates the current commit request is a primary commit or a secondary commit. + // It's value maybe `Unknown` when using a client with an old version. + CommitRole commit_role = 6; + // primary_key indicates the primary key of the transaction. + // Its value may be empty when using an old version client. + bytes primary_key = 7; + // Whether committing an `use_async_commit` (i.e. should be treated as committed) prewrite. + bool use_async_commit = 8; + + // Reserved for file based transaction. + bool is_txn_file = 100; } message CommitResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // If the commit ts is derived from min_commit_ts, this field should be set. - uint64 commit_version = 3; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 4; + errorpb.Error region_error = 1; + KeyError error = 2; + // If the commit ts is derived from min_commit_ts, this field should be set. + uint64 commit_version = 3; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 4; } // Not yet implemented. message ImportRequest { - repeated Mutation mutations = 1; - uint64 commit_version = 2; + repeated Mutation mutations = 1; + uint64 commit_version = 2; } message ImportResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } // Cleanup a key by possibly unlocking it. // From 4.0 onwards, this message is no longer used. message CleanupRequest { - Context context = 1; - bytes key = 2; - uint64 start_version = 3; - // The current timestamp, used in combination with a lock's TTL to determine - // if the lock has expired. If `current_ts == 0`, then the key will be unlocked - // irrespective of its TTL. - uint64 current_ts = 4; + Context context = 1; + bytes key = 2; + uint64 start_version = 3; + // The current timestamp, used in combination with a lock's TTL to determine + // if the lock has expired. If `current_ts == 0`, then the key will be unlocked + // irrespective of its TTL. + uint64 current_ts = 4; } message CleanupResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Set if the key is already committed. - uint64 commit_version = 3; + errorpb.Error region_error = 1; + KeyError error = 2; + // Set if the key is already committed. + uint64 commit_version = 3; } // Similar to a `Get` request. message BatchGetRequest { - Context context = 1; - repeated bytes keys = 2; - uint64 version = 3; + Context context = 1; + repeated bytes keys = 2; + uint64 version = 3; + // If true, the response will include the commit ts of the key. + bool need_commit_ts = 4; } message BatchGetResponse { - errorpb.Error region_error = 1; - repeated KvPair pairs = 2; - reserved 3; - // Time and scan details when processing the request. - ExecDetailsV2 exec_details_v2 = 4; - // This KeyError exists when some key is locked but we cannot check locks of all keys. - // In this case, `pairs` should be empty and the client should redo batch get all the keys - // after resolving the lock. - KeyError error = 5; + errorpb.Error region_error = 1; + repeated KvPair pairs = 2; + reserved 3; + // Time and scan details when processing the request. + ExecDetailsV2 exec_details_v2 = 4; + // This KeyError exists when some key is locked but we cannot check locks of all keys. + // In this case, `pairs` should be empty and the client should redo batch get all the keys + // after resolving the lock. + KeyError error = 5; } // Rollback a prewritten transaction. This will remove the preliminary data from the database, // unlock locks, and leave a rollback tombstone. message BatchRollbackRequest { - Context context = 1; - // Identify the transaction to be rolled back. - uint64 start_version = 2; - // The keys to rollback. - repeated bytes keys = 3; + Context context = 1; + // Identify the transaction to be rolled back. + uint64 start_version = 2; + // The keys to rollback. + repeated bytes keys = 3; - // Reserved for file based transaction. - bool is_txn_file = 100; + // Reserved for file based transaction. + bool is_txn_file = 100; } message BatchRollbackResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 3; + errorpb.Error region_error = 1; + KeyError error = 2; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 3; } // Scan the database for locks. Used at the start of the GC process to find all // old locks. message ScanLockRequest { - Context context = 1; - // Returns all locks with a start timestamp before `max_version`. - uint64 max_version = 2; - // Start scanning from this key. - bytes start_key = 3; - // The maximum number of locks to return. - uint32 limit = 4; - // The exclusive upperbound for scanning. - bytes end_key = 5; + Context context = 1; + // Returns all locks with a start timestamp before `max_version`. + uint64 max_version = 2; + // Start scanning from this key. + bytes start_key = 3; + // The maximum number of locks to return. + uint32 limit = 4; + // The exclusive upperbound for scanning. + bytes end_key = 5; } message ScanLockResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Info on all locks found by the scan. - repeated LockInfo locks = 3; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 4; + errorpb.Error region_error = 1; + KeyError error = 2; + // Info on all locks found by the scan. + repeated LockInfo locks = 3; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 4; } // For all keys locked by the transaction identified by `start_version`, either // commit or rollback the transaction and unlock the key. message ResolveLockRequest { - Context context = 1; - uint64 start_version = 2; - // `commit_version == 0` means the transaction was rolled back. - // `commit_version > 0` means the transaction was committed at the given timestamp. - uint64 commit_version = 3; - repeated TxnInfo txn_infos = 4; - // Only resolve specified keys. - repeated bytes keys = 5; + Context context = 1; + uint64 start_version = 2; + // `commit_version == 0` means the transaction was rolled back. + // `commit_version > 0` means the transaction was committed at the given timestamp. + uint64 commit_version = 3; + repeated TxnInfo txn_infos = 4; + // Only resolve specified keys. + repeated bytes keys = 5; + + // If true, the resolve lock action is performed asynchronously and the response may be returned before it's finished. + bool is_async = 6; - // Reserved for file based transaction. - bool is_txn_file = 100; + // Reserved for file based transaction. + bool is_txn_file = 100; } message ResolveLockResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 3; + errorpb.Error region_error = 1; + KeyError error = 2; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 3; } // Request TiKV to garbage collect all non-current data older than `safe_point`. message GCRequest { - Context context = 1; - uint64 safe_point = 2; + Context context = 1; + uint64 safe_point = 2; } message GCResponse { - errorpb.Error region_error = 1; - KeyError error = 2; + errorpb.Error region_error = 1; + KeyError error = 2; } // Delete a range of data from TiKV. message DeleteRangeRequest { - Context context = 1; - bytes start_key = 2; - bytes end_key = 3; - // If true, the data will not be immediately deleted, but the operation will - // still be replicated via Raft. This is used to notify TiKV that the data - // will be deleted using `unsafe_destroy_range` soon. - bool notify_only = 4; + Context context = 1; + bytes start_key = 2; + bytes end_key = 3; + // If true, the data will not be immediately deleted, but the operation will + // still be replicated via Raft. This is used to notify TiKV that the data + // will be deleted using `unsafe_destroy_range` soon. + bool notify_only = 4; } message DeleteRangeResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } // Preparing the flashback for a region/key range will "lock" the region // so that there is no any read, write or schedule operation could be proposed before // the actual flashback operation. message PrepareFlashbackToVersionRequest { - Context context = 1; - bytes start_key = 2; - bytes end_key = 3; - // The `start_ts` which we will use to write a lock to prevent - // the `resolved_ts` from advancing during the whole process. - uint64 start_ts = 4; - // The TS version which the data will flashback to later. - uint64 version = 5; + Context context = 1; + bytes start_key = 2; + bytes end_key = 3; + // The `start_ts` which we will use to write a lock to prevent + // the `resolved_ts` from advancing during the whole process. + uint64 start_ts = 4; + // The TS version which the data will flashback to later. + uint64 version = 5; } message PrepareFlashbackToVersionResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } // Flashback the region to a specific point with the given `version`, please // make sure the region is "locked" by `PrepareFlashbackToVersionRequest` first, // otherwise this request will fail. message FlashbackToVersionRequest { - Context context = 1; - // The TS version which the data should flashback to. - uint64 version = 2; - bytes start_key = 3; - bytes end_key = 4; - // The `start_ts` and `commit_ts` which the newly written MVCC version will use. - // Please make sure the `start_ts` is the same one in `PrepareFlashbackToVersionRequest`. - uint64 start_ts = 5; - uint64 commit_ts = 6; + Context context = 1; + // The TS version which the data should flashback to. + uint64 version = 2; + bytes start_key = 3; + bytes end_key = 4; + // The `start_ts` and `commit_ts` which the newly written MVCC version will use. + // Please make sure the `start_ts` is the same one in `PrepareFlashbackToVersionRequest`. + uint64 start_ts = 5; + uint64 commit_ts = 6; } message FlashbackToVersionResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } // Raw commands. message RawGetRequest { - Context context = 1; - bytes key = 2; - string cf = 3; + Context context = 1; + bytes key = 2; + string cf = 3; } message RawGetResponse { - errorpb.Error region_error = 1; - string error = 2; - bytes value = 3; - bool not_found = 4; + errorpb.Error region_error = 1; + string error = 2; + bytes value = 3; + bool not_found = 4; } message RawBatchGetRequest { - Context context = 1; - repeated bytes keys = 2; - string cf = 3; + Context context = 1; + repeated bytes keys = 2; + string cf = 3; } message RawBatchGetResponse { - errorpb.Error region_error = 1; - repeated KvPair pairs = 2; + errorpb.Error region_error = 1; + repeated KvPair pairs = 2; } message RawPutRequest { - Context context = 1; - bytes key = 2; - bytes value = 3; - string cf = 4; - uint64 ttl = 5; - bool for_cas = 6; + Context context = 1; + bytes key = 2; + bytes value = 3; + string cf = 4; + uint64 ttl = 5; + bool for_cas = 6; } message RawPutResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawBatchPutRequest { - Context context = 1; - repeated KvPair pairs = 2; - string cf = 3; - uint64 ttl = 4 [deprecated=true]; - bool for_cas = 5; + Context context = 1; + repeated KvPair pairs = 2; + string cf = 3; + uint64 ttl = 4 [deprecated = true]; + bool for_cas = 5; - // The time-to-live for each keys in seconds, and if the length of `ttls` - // is exactly one, the ttl will be applied to all keys. Otherwise, the length - // mismatch between `ttls` and `pairs` will return an error. - repeated uint64 ttls = 6; + // The time-to-live for each keys in seconds, and if the length of `ttls` + // is exactly one, the ttl will be applied to all keys. Otherwise, the length + // mismatch between `ttls` and `pairs` will return an error. + repeated uint64 ttls = 6; } message RawBatchPutResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawDeleteRequest { - Context context = 1; - bytes key = 2; - string cf = 3; - bool for_cas = 4; + Context context = 1; + bytes key = 2; + string cf = 3; + bool for_cas = 4; } message RawDeleteResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawBatchDeleteRequest { - Context context = 1; - repeated bytes keys = 2; - string cf = 3; - bool for_cas = 4; + Context context = 1; + repeated bytes keys = 2; + string cf = 3; + bool for_cas = 4; } message RawBatchDeleteResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawScanRequest { - Context context = 1; - bytes start_key = 2; - uint32 limit = 3; - bool key_only = 4; - string cf = 5; - bool reverse = 6; - // For compatibility, when scanning forward, the range to scan is [start_key, end_key), where start_key < end_key; - // and when scanning backward, it scans [end_key, start_key) in descending order, where end_key < start_key. - bytes end_key = 7; + Context context = 1; + bytes start_key = 2; + uint32 limit = 3; + bool key_only = 4; + string cf = 5; + bool reverse = 6; + // For compatibility, when scanning forward, the range to scan is [start_key, end_key), where start_key < end_key; + // and when scanning backward, it scans [end_key, start_key) in descending order, where end_key < start_key. + bytes end_key = 7; } message RawScanResponse { - errorpb.Error region_error = 1; - repeated KvPair kvs = 2; + errorpb.Error region_error = 1; + repeated KvPair kvs = 2; } message RawDeleteRangeRequest { - Context context = 1; - bytes start_key = 2; - bytes end_key = 3; - string cf = 4; + Context context = 1; + bytes start_key = 2; + bytes end_key = 3; + string cf = 4; } message RawDeleteRangeResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawBatchScanRequest { - Context context = 1; - repeated KeyRange ranges = 2; // scanning range - uint32 each_limit = 3; // max number of returning kv pairs for each scanning range - bool key_only = 4; - string cf = 5; - bool reverse = 6; + Context context = 1; + repeated KeyRange ranges = 2; // scanning range + uint32 each_limit = 3; // max number of returning kv pairs for each scanning range + bool key_only = 4; + string cf = 5; + bool reverse = 6; } message RawBatchScanResponse { - errorpb.Error region_error = 1; - repeated KvPair kvs = 2; + errorpb.Error region_error = 1; + repeated KvPair kvs = 2; } // Store commands (sent to a whole TiKV cluster, rather than a certain region). message UnsafeDestroyRangeRequest { - Context context = 1; - bytes start_key = 2; - bytes end_key = 3; + Context context = 1; + bytes start_key = 2; + bytes end_key = 3; } message UnsafeDestroyRangeResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RegisterLockObserverRequest { - Context context = 1; - uint64 max_ts = 2; + Context context = 1; + uint64 max_ts = 2; } message RegisterLockObserverResponse { - string error = 1; + string error = 1; } message CheckLockObserverRequest { - Context context = 1; - uint64 max_ts = 2; + Context context = 1; + uint64 max_ts = 2; } message CheckLockObserverResponse { - string error = 1; - bool is_clean = 2; - repeated LockInfo locks = 3; + string error = 1; + bool is_clean = 2; + repeated LockInfo locks = 3; } message RemoveLockObserverRequest { - Context context = 1; - uint64 max_ts = 2; + Context context = 1; + uint64 max_ts = 2; } message RemoveLockObserverResponse { - string error = 1; + string error = 1; } message PhysicalScanLockRequest { - Context context = 1; - uint64 max_ts = 2; - bytes start_key = 3; - uint32 limit = 4; + Context context = 1; + uint64 max_ts = 2; + bytes start_key = 3; + uint32 limit = 4; } message PhysicalScanLockResponse { - string error = 1; - repeated LockInfo locks = 2; + string error = 1; + repeated LockInfo locks = 2; } // Sent from PD to a TiKV node. message SplitRegionRequest { - Context context = 1; - bytes split_key = 2 [deprecated=true]; - repeated bytes split_keys = 3; // when use it to do batch split, `split_key` should be empty. - // Once enabled, the split_key will not be encoded. - bool is_raw_kv = 4; + Context context = 1; + bytes split_key = 2 [deprecated = true]; + repeated bytes split_keys = 3; // when use it to do batch split, `split_key` should be empty. + // Once enabled, the split_key will not be encoded. + bool is_raw_kv = 4; } message SplitRegionResponse { - errorpb.Error region_error = 1; - metapb.Region left = 2 [deprecated=true]; // set when there are only 2 result regions. - metapb.Region right = 3 [deprecated=true]; // set when there are only 2 result regions. - repeated metapb.Region regions = 4; // include all result regions. + errorpb.Error region_error = 1; + metapb.Region left = 2 [deprecated = true]; // set when there are only 2 result regions. + metapb.Region right = 3 [deprecated = true]; // set when there are only 2 result regions. + repeated metapb.Region regions = 4; // include all result regions. + + // Reserved for file based transaction. + repeated KeyError errors = 100; } // Sent from TiFlash to a TiKV node. -message ReadIndexRequest{ - Context context = 1; +message ReadIndexRequest { + Context context = 1; - // TiKV checks the given range if there is any unapplied lock - // blocking the read request. - uint64 start_ts = 2; - repeated KeyRange ranges = 3; + // TiKV checks the given range if there is any unapplied lock + // blocking the read request. + uint64 start_ts = 2; + repeated KeyRange ranges = 3; } -message ReadIndexResponse{ - errorpb.Error region_error = 1; - uint64 read_index = 2; - // If `locked` is set, this read request is blocked by a lock. - // The lock should be returned to the client. - kvrpcpb.LockInfo locked = 3; +message ReadIndexResponse { + errorpb.Error region_error = 1; + uint64 read_index = 2; + // If `locked` is set, this read request is blocked by a lock. + // The lock should be returned to the client. + kvrpcpb.LockInfo locked = 3; } // Commands for debugging transactions. message MvccGetByKeyRequest { - Context context = 1; - bytes key = 2; + Context context = 1; + bytes key = 2; } message MvccGetByKeyResponse { - errorpb.Error region_error = 1; - string error = 2; - MvccInfo info = 3; + errorpb.Error region_error = 1; + string error = 2; + MvccInfo info = 3; } message MvccGetByStartTsRequest { - Context context = 1; - uint64 start_ts = 2; + Context context = 1; + uint64 start_ts = 2; } message MvccGetByStartTsResponse { - errorpb.Error region_error = 1; - string error = 2; - bytes key = 3; - MvccInfo info = 4; + errorpb.Error region_error = 1; + string error = 2; + bytes key = 3; + MvccInfo info = 4; } // Helper messages. +enum RequestOrigin { + RequestOriginUnknown = 0; + RequestOriginTiDB = 1; +} + // Miscellaneous metadata attached to most requests. message Context { - reserved 4; - reserved "read_quorum"; - uint64 region_id = 1; - metapb.RegionEpoch region_epoch = 2; - metapb.Peer peer = 3; - uint64 term = 5; - CommandPri priority = 6; - IsolationLevel isolation_level = 7; - bool not_fill_cache = 8; - bool sync_log = 9; - - // True means execution time statistics should be recorded and returned. - bool record_time_stat = 10; - // True means RocksDB scan statistics should be recorded and returned. - bool record_scan_stat = 11; - - bool replica_read = 12; - // Read requests can ignore locks belonging to these transactions because either - // these transactions are rolled back or theirs commit_ts > read request's start_ts. - repeated uint64 resolved_locks = 13; - uint64 max_execution_duration_ms = 14; - - // After a region applies to `applied_index`, we can get a - // snapshot for the region even if the peer is a follower. - uint64 applied_index = 15; - // A hint for TiKV to schedule tasks more fairly. Query with same task ID - // may share same priority and resource quota. - uint64 task_id = 16; - - // Not required to read the most up-to-date data, replicas with `safe_ts` >= `start_ts` - // can handle read request directly - bool stale_read = 17; - - // Any additional serialized information about the request. - bytes resource_group_tag = 18; - - // Used to tell TiKV whether operations are allowed or not on different disk usages. - DiskFullOpt disk_full_opt = 19; - - // Indicates the request is a retry request and the same request may have been sent before. - bool is_retry_request = 20; - - // API version implies the encode of the key and value. - APIVersion api_version = 21; - - // Read request should read through locks belonging to these transactions because these - // transactions are committed and theirs commit_ts <= read request's start_ts. - repeated uint64 committed_locks = 22; - - // The informantion to trace a request sent to TiKV. - tracepb.TraceContext trace_context = 23; - - // The source of the request, will be used as the tag of the metrics reporting. - // This field can be set for any requests that require to report metrics with any extra labels. - string request_source = 24; - - // The source of the current transaction. - uint64 txn_source = 25; - - reserved 26; // Used for resource_group_name, now it's moved into ResourceControlContext. - - // If `busy_threshold_ms` is given, TiKV can reject the request and return a `ServerIsBusy` - // error before processing if the estimated waiting duration exceeds the threshold. - uint32 busy_threshold_ms = 27; - - // Some information used for resource control. - ResourceControlContext resource_control_context = 28; - - // The keyspace that the request is sent to. - // NOTE: This field is only meaningful while the api_version is V2. - uint32 keyspace_id = 32; - - // The buckets version that the request is sent to. - // NOTE: This field is only meaningful while enable buckets. - uint64 buckets_version = 33; - - // It tells us where the request comes from in TiDB. If it isn't from TiDB, leave it blank. - // This is for tests only and thus can be safely changed/removed without affecting compatibility. - SourceStmt source_stmt = 34; + reserved 4; + reserved "read_quorum"; + uint64 region_id = 1; + metapb.RegionEpoch region_epoch = 2; + metapb.Peer peer = 3; + uint64 term = 5; + CommandPri priority = 6; + IsolationLevel isolation_level = 7; + bool not_fill_cache = 8; + bool sync_log = 9; + + // True means execution time statistics should be recorded and returned. + bool record_time_stat = 10; + // True means RocksDB scan statistics should be recorded and returned. + bool record_scan_stat = 11; + + bool replica_read = 12; + // Read requests can ignore locks belonging to these transactions because either + // these transactions are rolled back or theirs commit_ts > read request's start_ts. + repeated uint64 resolved_locks = 13; + uint64 max_execution_duration_ms = 14; + + // After a region applies to `applied_index`, we can get a + // snapshot for the region even if the peer is a follower. + uint64 applied_index = 15; + // A hint for TiKV to schedule tasks more fairly. Query with same task ID + // may share same priority and resource quota. + uint64 task_id = 16; + + // Not required to read the most up-to-date data, replicas with `safe_ts` >= `start_ts` + // can handle read request directly + bool stale_read = 17; + + // Any additional serialized information about the request. + bytes resource_group_tag = 18; + + // Used to tell TiKV whether operations are allowed or not on different disk usages. + DiskFullOpt disk_full_opt = 19; + + // Indicates the request is a retry request and the same request may have been sent before. + bool is_retry_request = 20; + + // API version implies the encode of the key and value. + APIVersion api_version = 21; + + // Read request should read through locks belonging to these transactions because these + // transactions are committed and theirs commit_ts <= read request's start_ts. + repeated uint64 committed_locks = 22; + + // The informantion to trace a request sent to TiKV. + tracepb.TraceContext trace_context = 23; + + // The source of the request, will be used as the tag of the metrics reporting. + // This field can be set for any requests that require to report metrics with any extra labels. + string request_source = 24; + + // The source of the current transaction. + uint64 txn_source = 25; + + reserved 26; // Used for resource_group_name, now it's moved into ResourceControlContext. + + // If `busy_threshold_ms` is given, TiKV can reject the request and return a `ServerIsBusy` + // error before processing if the estimated waiting duration exceeds the threshold. + uint32 busy_threshold_ms = 27; + + // Some information used for resource control. + ResourceControlContext resource_control_context = 28; + + // The declared origin of the request. TiDB server requests set this to TIDB. + // This field is client-provided metadata; consumers should rely on it only when + // the caller is authenticated/authorized by the transport or deployment boundary. + // UNKNOWN means the origin is unset or not recognized by this protocol version. + // Other components should add dedicated enum variants when they need origin-specific behavior. + RequestOrigin request_origin = 29; + + // The keyspace that the request is sent to. + // NOTE: This field is only meaningful while the api_version is V2. + string keyspace_name = 31; + + // The keyspace that the request is sent to. + // NOTE: This field is only meaningful while the api_version is V2. + uint32 keyspace_id = 32; + + // The buckets version that the request is sent to. + // NOTE: This field is only meaningful while enable buckets. + uint64 buckets_version = 33; + + // It tells us where the request comes from in TiDB. If it isn't from TiDB, leave it blank. + // This is for tests only and thus can be safely changed/removed without affecting compatibility. + SourceStmt source_stmt = 34; + + // The cluster id of the request + uint64 cluster_id = 35; + + // The trace id of the request, will be used for tracing the request's execution's inner steps. + bytes trace_id = 36; + + // Control flags for trace logging behavior. + // Bit 0: immediate_log - Force immediate logging without buffering + // Bit 1: category_req_resp - Enable request/response tracing + // Bit 2: category_write_details - Enable detailed write tracing + // Bit 3: category_read_details - Enable detailed read tracing + // Bits 4-63: Reserved for future use + // This field is set by client-go based on an extractor function provided by TiDB. + uint64 trace_control_flags = 37; + + // TODO(slock): tidb should send requests with a feature flag to indicate whether it + // supports shared locks, so that tikv can fail the requests from old tidb versions + // when needed. } message ResourceControlContext { - // It's used to identify which resource group the request belongs to. - string resource_group_name = 1; + // It's used to identify which resource group the request belongs to. + string resource_group_name = 1; - // The resource consumption of the resource group that have completed at all TiKVs between the previous request to this TiKV and current request. - // It's used as penalty to make the local resource scheduling on one TiKV takes the gloabl resource consumption into consideration. - resource_manager.Consumption penalty = 2; + // The resource consumption of the resource group that have completed at all TiKVs between the previous request to this TiKV and current request. + // It's used as penalty to make the local resource scheduling on one TiKV takes the gloabl resource consumption into consideration. + resource_manager.Consumption penalty = 2; - // This priority would override the original priority of the resource group for the request. - // Used to deprioritize the runaway queries. - uint64 override_priority = 3; + // This priority would override the original priority of the resource group for the request. + // Used to deprioritize the runaway queries. + uint64 override_priority = 3; } // The API version the server and the client is using. // See more details in https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md. enum APIVersion { - // `V1` is mainly for TiDB & TxnKV, and is not safe to use RawKV along with the others. - // V1 server only accepts V1 requests. V1 raw requests with TTL will be rejected. - V1 = 0; - // - // `V1TTL` is only available to RawKV, and 8 bytes representing the unix timestamp in - // seconds for expiring time will be append to the value of all RawKV entries. For example: - // ------------------------------------------------------------ - // | User value | Expire Ts | - // ------------------------------------------------------------ - // | 0x12 0x34 0x56 | 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff | - // ------------------------------------------------------------ - // V1TTL server only accepts V1 raw requests. - // V1 client should not use `V1TTL` in request. V1 client should always send `V1`. - V1TTL = 1; - // - // `V2` use new encoding for RawKV & TxnKV to support more features. - // - // Key Encoding: - // TiDB: start with `m` or `t`, the same as `V1`. - // TxnKV: prefix with `x`, encoded as `MCE( x{keyspace id} + {user key} ) + timestamp`. - // RawKV: prefix with `r`, encoded as `MCE( r{keyspace id} + {user key} ) + timestamp`. - // Where the `{keyspace id}` is fixed-length of 3 bytes in network byte order. - // Besides, RawKV entires must be in `default` CF. - // - // Value Encoding: - // TiDB & TxnKV: the same as `V1`. - // RawKV: `{user value} + {optional fields} + {meta flag}`. The last byte in the - // raw value must be meta flags. For example: - // -------------------------------------- - // | User value | Meta flags | - // -------------------------------------- - // | 0x12 0x34 0x56 | 0x00 (0b00000000) | - // -------------------------------------- - // Bit 0 of meta flags is for TTL. If set, the value contains 8 bytes expiring time as - // unix timestamp in seconds at the very left to the meta flags. - // -------------------------------------------------------------------------------- - // | User value | Expiring time | Meta flags | - // -------------------------------------------------------------------------------- - // | 0x12 0x34 0x56 | 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff | 0x01 (0b00000001) | - // -------------------------------------------------------------------------------- - // Bit 1 is for deletion. If set, the entry is logical deleted. - // --------------------- - // | Meta flags | - // --------------------- - // | 0x02 (0b00000010) | - // --------------------- - // - // V2 server accpets V2 requests and V1 transactional requests that statrts with TiDB key - // prefix (`m` and `t`). - V2 = 2; + // `V1` is mainly for TiDB & TxnKV, and is not safe to use RawKV along with the others. + // V1 server only accepts V1 requests. V1 raw requests with TTL will be rejected. + V1 = 0; + // + // `V1TTL` is only available to RawKV, and 8 bytes representing the unix timestamp in + // seconds for expiring time will be append to the value of all RawKV entries. For example: + // ------------------------------------------------------------ + // | User value | Expire Ts | + // ------------------------------------------------------------ + // | 0x12 0x34 0x56 | 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff | + // ------------------------------------------------------------ + // V1TTL server only accepts V1 raw requests. + // V1 client should not use `V1TTL` in request. V1 client should always send `V1`. + V1TTL = 1; + // + // `V2` use new encoding for RawKV & TxnKV to support more features. + // + // Key Encoding: + // TiDB: start with `m` or `t`, the same as `V1`. + // TxnKV: prefix with `x`, encoded as `MCE( x{keyspace id} + {user key} ) + timestamp`. + // RawKV: prefix with `r`, encoded as `MCE( r{keyspace id} + {user key} ) + timestamp`. + // Where the `{keyspace id}` is fixed-length of 3 bytes in network byte order. + // Besides, RawKV entires must be in `default` CF. + // + // Value Encoding: + // TiDB & TxnKV: the same as `V1`. + // RawKV: `{user value} + {optional fields} + {meta flag}`. The last byte in the + // raw value must be meta flags. For example: + // -------------------------------------- + // | User value | Meta flags | + // -------------------------------------- + // | 0x12 0x34 0x56 | 0x00 (0b00000000) | + // -------------------------------------- + // Bit 0 of meta flags is for TTL. If set, the value contains 8 bytes expiring time as + // unix timestamp in seconds at the very left to the meta flags. + // -------------------------------------------------------------------------------- + // | User value | Expiring time | Meta flags | + // -------------------------------------------------------------------------------- + // | 0x12 0x34 0x56 | 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff | 0x01 (0b00000001) | + // -------------------------------------------------------------------------------- + // Bit 1 is for deletion. If set, the entry is logical deleted. + // --------------------- + // | Meta flags | + // --------------------- + // | 0x02 (0b00000010) | + // --------------------- + // + // V2 server accpets V2 requests and V1 transactional requests that statrts with TiDB key + // prefix (`m` and `t`). + V2 = 2; } message SourceStmt { - uint64 start_ts = 1; - uint64 connection_id = 2; - uint64 stmt_id = 3; - // session alias set by user - string session_alias = 4; + uint64 start_ts = 1; + uint64 connection_id = 2; + uint64 stmt_id = 3; + // session alias set by user + string session_alias = 4; } message LockInfo { - bytes primary_lock = 1; - uint64 lock_version = 2; - bytes key = 3; - uint64 lock_ttl = 4; - // How many keys this transaction involves in this region. - uint64 txn_size = 5; - Op lock_type = 6; - uint64 lock_for_update_ts = 7; - // Fields for transactions that are using Async Commit. - bool use_async_commit = 8; - uint64 min_commit_ts = 9; - repeated bytes secondaries = 10; - // The time elapsed since last update of lock wait info when waiting. - // It's used in timeout errors. 0 means unknown or not applicable. - // It can be used to help the client decide whether to try resolving the lock. - uint64 duration_to_last_update_ms = 11; - - // Reserved for file based transaction. - bool is_txn_file = 100; + bytes primary_lock = 1; + uint64 lock_version = 2; + bytes key = 3; + uint64 lock_ttl = 4; + // How many keys this transaction involves in this region. + uint64 txn_size = 5; + Op lock_type = 6; + uint64 lock_for_update_ts = 7; + // Fields for transactions that are using Async Commit. + bool use_async_commit = 8; + uint64 min_commit_ts = 9; + repeated bytes secondaries = 10; + // The time elapsed since last update of lock wait info when waiting. + // It's used in timeout errors. 0 means unknown or not applicable. + // It can be used to help the client decide whether to try resolving the lock. + uint64 duration_to_last_update_ms = 11; + // When lock_type is SharedLock, this describes transactions holding the shared lock. + // Important: when lock_type is SharedLock, all shared locks must use shared_lock_infos; + // DO NOT read from the wrapper LockInfo. + // TODO(slock): tidb should send requests with a feature flag to indicate whether it + // supports shared locks, so that tikv can fail the requests from old tidb versions + // when needed. + repeated LockInfo shared_lock_infos = 12; + + // Reserved for file based transaction. + bool is_txn_file = 100; } message KeyError { - LockInfo locked = 1; // Client should backoff or cleanup the lock then retry. - string retryable = 2; // Client may restart the txn. e.g write conflict. - string abort = 3; // Client should abort the txn. - WriteConflict conflict = 4; // Write conflict is moved from retryable to here. - AlreadyExist already_exist = 5; // Key already exists - Deadlock deadlock = 6; // Deadlock is used in pessimistic transaction for single statement rollback. - CommitTsExpired commit_ts_expired = 7; // Commit ts is earlier than min commit ts of a transaction. - TxnNotFound txn_not_found = 8; // Txn not found when checking txn status. - CommitTsTooLarge commit_ts_too_large = 9; // Calculated commit TS exceeds the limit given by the user. - AssertionFailed assertion_failed = 10; // Assertion of a `Mutation` is evaluated as a failure. - PrimaryMismatch primary_mismatch = 11; // CheckTxnStatus is sent to a lock that's not the primary. + LockInfo locked = 1; // Client should backoff or cleanup the lock then retry. + string retryable = 2; // Client may restart the txn. e.g write conflict. + string abort = 3; // Client should abort the txn. + WriteConflict conflict = 4; // Write conflict is moved from retryable to here. + AlreadyExist already_exist = 5; // Key already exists + Deadlock deadlock = 6; // Deadlock is used in pessimistic transaction for single statement rollback. + CommitTsExpired commit_ts_expired = 7; // Commit ts is earlier than min commit ts of a transaction. + TxnNotFound txn_not_found = 8; // Txn not found when checking txn status. + CommitTsTooLarge commit_ts_too_large = 9; // Calculated commit TS exceeds the limit given by the user. + AssertionFailed assertion_failed = 10; // Assertion of a `Mutation` is evaluated as a failure. + PrimaryMismatch primary_mismatch = 11; // CheckTxnStatus is sent to a lock that's not the primary. + TxnLockNotFound txn_lock_not_found = 12; // TxnLockNotFound indicates the txn lock is not found. + + // Extra information for error debugging + DebugInfo debug_info = 100; } message WriteConflict { - enum Reason { - Unknown = 0; - Optimistic = 1; // in optimistic transactions. - PessimisticRetry = 2; // a lock acquisition request waits for a lock and awakes, or meets a newer version of data, let TiDB retry. - SelfRolledBack = 3; // the transaction itself has been rolled back when it tries to prewrite. - RcCheckTs = 4; // RcCheckTs failure by meeting a newer version, let TiDB retry. - LazyUniquenessCheck = 5; // write conflict found in lazy uniqueness check in pessimistic transactions. - } - - uint64 start_ts = 1; - uint64 conflict_ts = 2; - bytes key = 3; - bytes primary = 4; - uint64 conflict_commit_ts = 5; - Reason reason = 6; + enum Reason { + Unknown = 0; + Optimistic = 1; // in optimistic transactions. + PessimisticRetry = 2; // a lock acquisition request waits for a lock and awakes, or meets a newer version of data, let TiDB retry. + SelfRolledBack = 3; // the transaction itself has been rolled back when it tries to prewrite. + RcCheckTs = 4; // RcCheckTs failure by meeting a newer version, let TiDB retry. + LazyUniquenessCheck = 5; // write conflict found when deferring constraint checks in pessimistic transactions. Deprecated in next-gen (cloud-storage-engine). + NotLockedKeyConflict = 6; // write conflict found on keys that do not acquire pessimistic locks in pessimistic transactions. + } + + uint64 start_ts = 1; + uint64 conflict_ts = 2; + bytes key = 3; + bytes primary = 4; + uint64 conflict_commit_ts = 5; + Reason reason = 6; } message AlreadyExist { - bytes key = 1; + bytes key = 1; } message Deadlock { - uint64 lock_ts = 1; - bytes lock_key = 2; - uint64 deadlock_key_hash = 3; - repeated deadlock.WaitForEntry wait_chain = 4; + uint64 lock_ts = 1; + bytes lock_key = 2; + + // The hash of `deadlock_key` field. + uint64 deadlock_key_hash = 3; + repeated deadlock.WaitForEntry wait_chain = 4; + + // The key that the current transaction has already acquired and blocks another transaction to form the deadlock. + bytes deadlock_key = 5; } message CommitTsExpired { - uint64 start_ts = 1; - uint64 attempted_commit_ts = 2; - bytes key = 3; - uint64 min_commit_ts = 4; + uint64 start_ts = 1; + uint64 attempted_commit_ts = 2; + bytes key = 3; + uint64 min_commit_ts = 4; } message TxnNotFound { - uint64 start_ts = 1; - bytes primary_key = 2; + uint64 start_ts = 1; + bytes primary_key = 2; } message CommitTsTooLarge { - uint64 commit_ts = 1; // The calculated commit TS. + uint64 commit_ts = 1; // The calculated commit TS. } message AssertionFailed { - uint64 start_ts = 1; - bytes key = 2; - Assertion assertion = 3; - uint64 existing_start_ts = 4; - uint64 existing_commit_ts = 5; + uint64 start_ts = 1; + bytes key = 2; + Assertion assertion = 3; + uint64 existing_start_ts = 4; + uint64 existing_commit_ts = 5; } message PrimaryMismatch { - LockInfo lock_info = 1; + LockInfo lock_info = 1; +} + +message TxnLockNotFound { + bytes key = 1; +} + +message MvccDebugInfo { + bytes key = 1; + MvccInfo mvcc = 2; +} + +message DebugInfo { + repeated MvccDebugInfo mvcc_info = 1; } enum CommandPri { - Normal = 0; // Normal is the default value. - Low = 1; - High = 2; + Normal = 0; // Normal is the default value. + Low = 1; + High = 2; } enum IsolationLevel { - SI = 0; // SI = snapshot isolation - RC = 1; // RC = read committed - RCCheckTS = 2; // RC read and it's needed to check if there exists more recent versions. + SI = 0; // SI = snapshot isolation + RC = 1; // RC = read committed + RCCheckTS = 2; // RC read and it's needed to check if there exists more recent versions. } // Operation allowed info during each TiKV storage threshold. enum DiskFullOpt { - NotAllowedOnFull = 0; // The default value, means operations are not allowed either under almost full or already full. - AllowedOnAlmostFull = 1; // Means operations will be allowed when disk is almost full. - AllowedOnAlreadyFull = 2; // Means operations will be allowed when disk is already full. + NotAllowedOnFull = 0; // The default value, means operations are not allowed either under almost full or already full. + AllowedOnAlmostFull = 1; // Means operations will be allowed when disk is almost full. + AllowedOnAlreadyFull = 2; // Means operations will be allowed when disk is already full. } message TimeDetail { - // Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and - // other kind of waitings in series. (Wait time in the raftstore is not included.) - uint64 wait_wall_time_ms = 1; - // Off-cpu and on-cpu wall time elapsed to actually process the request payload. It does not - // include `wait_wall_time`. - // This field is very close to the CPU time in most cases. Some wait time spend in RocksDB - // cannot be excluded for now, like Mutex wait time, which is included in this field, so that - // this field is called wall time instead of CPU time. - uint64 process_wall_time_ms = 2; - // KV read wall Time means the time used in key/value scan and get. - uint64 kv_read_wall_time_ms = 3; - // Total wall clock time spent on this RPC in TiKV . - uint64 total_rpc_wall_time_ns = 4; + // Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and + // other kind of waitings in series. (Wait time in the raftstore is not included.) + uint64 wait_wall_time_ms = 1; + // Off-cpu and on-cpu wall time elapsed to actually process the request payload. It does not + // include `wait_wall_time`. + // This field is very close to the CPU time in most cases. Some wait time spend in RocksDB + // cannot be excluded for now, like Mutex wait time, which is included in this field, so that + // this field is called wall time instead of CPU time. + uint64 process_wall_time_ms = 2; + // KV read wall Time means the time used in key/value scan and get. + uint64 kv_read_wall_time_ms = 3; + // Total wall clock time spent on this RPC in TiKV . + uint64 total_rpc_wall_time_ns = 4; } message TimeDetailV2 { - // Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and - // other kind of waitings in series. (Wait time in the raftstore is not included.) - uint64 wait_wall_time_ns = 1; - // Off-cpu and on-cpu wall time elapsed to actually process the request payload. It does not - // include `wait_wall_time` and `suspend_wall_time`. - // This field is very close to the CPU time in most cases. Some wait time spend in RocksDB - // cannot be excluded for now, like Mutex wait time, which is included in this field, so that - // this field is called wall time instead of CPU time. - uint64 process_wall_time_ns = 2; - // Cpu wall time elapsed that task is waiting in queue. - uint64 process_suspend_wall_time_ns = 3; - // KV read wall Time means the time used in key/value scan and get. - uint64 kv_read_wall_time_ns = 4; - // Total wall clock time spent on this RPC in TiKV . - uint64 total_rpc_wall_time_ns = 5; + // Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and + // other kind of waitings in series. (Wait time in the raftstore is not included.) + uint64 wait_wall_time_ns = 1; + // Off-cpu and on-cpu wall time elapsed to actually process the request payload. It does not + // include `wait_wall_time` and `suspend_wall_time`. + // This field is very close to the CPU time in most cases. Some wait time spend in RocksDB + // cannot be excluded for now, like Mutex wait time, which is included in this field, so that + // this field is called wall time instead of CPU time. + uint64 process_wall_time_ns = 2; + // Cpu wall time elapsed that task is waiting in queue. + uint64 process_suspend_wall_time_ns = 3; + // KV read wall Time means the time used in key/value scan and get. + uint64 kv_read_wall_time_ns = 4; + // Total wall clock time spent on this RPC in TiKV . + uint64 total_rpc_wall_time_ns = 5; + // Time spent on the gRPC layer. + uint64 kv_grpc_process_time_ns = 6; + // Time spent on waiting for run again in grpc pool from other executor pool. + uint64 kv_grpc_wait_time_ns = 7; } message ScanInfo { - int64 total = 1; - int64 processed = 2; - int64 read_bytes = 3; + int64 total = 1; + int64 processed = 2; + int64 read_bytes = 3; } // Only reserved for compatibility. message ScanDetail { - ScanInfo write = 1; - ScanInfo lock = 2; - ScanInfo data = 3; + ScanInfo write = 1; + ScanInfo lock = 2; + ScanInfo data = 3; } message ScanDetailV2 { - // Number of user keys scanned from the storage. - // It does not include deleted version or RocksDB tombstone keys. - // For Coprocessor requests, it includes keys that has been filtered out by - // Selection. - uint64 processed_versions = 1; + // Number of user keys scanned from the storage. + // It does not include deleted version or RocksDB tombstone keys. + // For Coprocessor requests, it includes keys that has been filtered out by + // Selection. + uint64 processed_versions = 1; - // Number of bytes of user key-value pairs scanned from the storage, i.e. - // total size of data returned from MVCC layer. - uint64 processed_versions_size = 8; + // Number of bytes of user key-value pairs scanned from the storage, i.e. + // total size of data returned from MVCC layer. + uint64 processed_versions_size = 8; - // Approximate number of MVCC keys meet during scanning. It includes - // deleted versions, but does not include RocksDB tombstone keys. - // - // When this field is notably larger than `processed_versions`, it means - // there are a lot of deleted MVCC keys. - uint64 total_versions = 2; + // Approximate number of MVCC keys meet during scanning. It includes + // deleted versions, but does not include RocksDB tombstone keys. + // + // When this field is notably larger than `processed_versions`, it means + // there are a lot of deleted MVCC keys. + uint64 total_versions = 2; - // Total number of deletes and single deletes skipped over during - // iteration, i.e. how many RocksDB tombstones are skipped. - uint64 rocksdb_delete_skipped_count = 3; + // Total number of deletes and single deletes skipped over during + // iteration, i.e. how many RocksDB tombstones are skipped. + uint64 rocksdb_delete_skipped_count = 3; - // Total number of internal keys skipped over during iteration. - // See https://github.com/facebook/rocksdb/blob/9f1c84ca471d8b1ad7be9f3eebfc2c7e07dfd7a7/include/rocksdb/perf_context.h#L84 for details. - uint64 rocksdb_key_skipped_count = 4; + // Total number of internal keys skipped over during iteration. + // See https://github.com/facebook/rocksdb/blob/9f1c84ca471d8b1ad7be9f3eebfc2c7e07dfd7a7/include/rocksdb/perf_context.h#L84 for details. + uint64 rocksdb_key_skipped_count = 4; - // Total number of RocksDB block cache hits. - uint64 rocksdb_block_cache_hit_count = 5; + // Total number of RocksDB block cache hits. + uint64 rocksdb_block_cache_hit_count = 5; - // Total number of block reads (with IO). - uint64 rocksdb_block_read_count = 6; + // Total number of block reads (with IO). + uint64 rocksdb_block_read_count = 6; - // Total number of bytes from block reads. - uint64 rocksdb_block_read_byte = 7; + // Total number of bytes from block reads. + uint64 rocksdb_block_read_byte = 7; - // Total time used for block reads. - uint64 rocksdb_block_read_nanos = 9; + // Total time used for block reads. + uint64 rocksdb_block_read_nanos = 9; - // Time used for getting a raftstore snapshot (including proposing read index, leader confirmation and getting the RocksDB snapshot). - uint64 get_snapshot_nanos = 10; + // Time used for getting a raftstore snapshot (including proposing read index, leader confirmation and getting the RocksDB snapshot). + uint64 get_snapshot_nanos = 10; - // Time used for proposing read index from read pool to store pool, equals 0 when performing lease read. - uint64 read_index_propose_wait_nanos = 11; + // Time used for proposing read index from read pool to store pool, equals 0 when performing lease read. + uint64 read_index_propose_wait_nanos = 11; - // Time used for leader confirmation, equals 0 when performing lease read. - uint64 read_index_confirm_wait_nanos = 12; + // Time used for leader confirmation, equals 0 when performing lease read. + uint64 read_index_confirm_wait_nanos = 12; - // Time used for read pool scheduling. - uint64 read_pool_schedule_wait_nanos = 13; + // Time used for read pool scheduling. + uint64 read_pool_schedule_wait_nanos = 13; + + // The total MVCC key-value size meet during scanning. It includes + // deleted versions, but does not include RocksDB tombstone keys. + // + // When this field is notably larger than `processed_versions_size`, it means + // there are a lot of deleted MVCC keys. + uint64 total_versions_size = 14; + + // Total number of IA segment cache hits for this request. + uint64 ia_cache_hit_count = 15; + + // Total number of IA remote segment reads for this request. + // + // IA segments are only read on cache misses, so this is also the IA segment + // cache miss count for the request. + uint64 ia_remote_read_segment_count = 16; + + // Total number of logical bytes returned from IA remote segment reads for this + // request. + uint64 ia_remote_read_segment_bytes = 17; + + // Total time spent serving IA remote segment reads for this request. + uint64 ia_remote_read_segment_nanos = 18; } message ExecDetails { - // Available when ctx.record_time_stat = true or meet slow query. - TimeDetail time_detail = 1; + // Available when ctx.record_time_stat = true or meet slow query. + TimeDetail time_detail = 1; - // Available when ctx.record_scan_stat = true or meet slow query. - ScanDetail scan_detail = 2; + // Available when ctx.record_scan_stat = true or meet slow query. + ScanDetail scan_detail = 2; - // See https://github.com/pingcap/kvproto/pull/689 - reserved 3; - reserved 4; + // See https://github.com/pingcap/kvproto/pull/689 + reserved 3; + reserved 4; } message ExecDetailsV2 { - // Available when ctx.record_time_stat = true or meet slow query. - // deprecated. Should use `time_detail_v2` instead. - TimeDetail time_detail = 1; - - // Available when ctx.record_scan_stat = true or meet slow query. - ScanDetailV2 scan_detail_v2 = 2; - - // Raftstore writing durations of the request. Only available for some write requests. - WriteDetail write_detail = 3; - // Available when ctx.record_time_stat = true or meet slow query. - TimeDetailV2 time_detail_v2 = 4; + // Available when ctx.record_time_stat = true or meet slow query. + // deprecated. Should use `time_detail_v2` instead. + TimeDetail time_detail = 1; + + // Available when ctx.record_scan_stat = true or meet slow query. + ScanDetailV2 scan_detail_v2 = 2; + + // Raftstore writing durations of the request. Only available for some write requests. + WriteDetail write_detail = 3; + // Available when ctx.record_time_stat = true or meet slow query. + TimeDetailV2 time_detail_v2 = 4; + + // RU (Request Unit) consumption details. + RUV2 ru_v2 = 5; +} + +message RUV2 { + uint64 kv_engine_cache_miss = 1; + ExecutorInputs executor_inputs = 2; + uint64 coprocessor_executor_iterations = 3; + uint64 coprocessor_response_bytes = 4; + uint64 raftstore_store_write_trigger_wb_bytes = 5; + uint64 storage_processed_keys_batch_get = 6; + uint64 storage_processed_keys_get = 7; + // Filled by client-go. + uint64 read_rpc_count = 8; + // Filled by client-go. + uint64 write_rpc_count = 9; +} + +message ExecutorInputs { + uint64 tikv_coprocessor_executor_work_total_batch_index_scan = 1; + uint64 tikv_coprocessor_executor_work_total_batch_table_scan = 2; + uint64 tikv_coprocessor_executor_work_total_batch_selection = 3; + uint64 tikv_coprocessor_executor_work_total_batch_top_n = 4; + uint64 tikv_coprocessor_executor_work_total_batch_limit = 5; + uint64 tikv_coprocessor_executor_work_total_batch_simple_aggr = 6; + uint64 tikv_coprocessor_executor_work_total_batch_fast_hash_aggr = 7; } message WriteDetail { - // Wait duration in the store loop. - uint64 store_batch_wait_nanos = 1; + // Wait duration in the store loop. + uint64 store_batch_wait_nanos = 1; - // Wait duration before sending proposal to peers. - uint64 propose_send_wait_nanos = 2; + // Wait duration before sending proposal to peers. + uint64 propose_send_wait_nanos = 2; - // Total time spent on persisting the log. - uint64 persist_log_nanos = 3; + // Total time spent on persisting the log. + uint64 persist_log_nanos = 3; - // Wait time until the Raft log write leader begins to write. - uint64 raft_db_write_leader_wait_nanos = 4; + // Wait time until the Raft log write leader begins to write. + uint64 raft_db_write_leader_wait_nanos = 4; - // Time spent on synchronizing the Raft log to the disk. - uint64 raft_db_sync_log_nanos = 5; + // Time spent on synchronizing the Raft log to the disk. + uint64 raft_db_sync_log_nanos = 5; - // Time spent on writing the Raft log to the Raft memtable. - uint64 raft_db_write_memtable_nanos = 6; + // Time spent on writing the Raft log to the Raft memtable. + uint64 raft_db_write_memtable_nanos = 6; - // Time waiting for peers to confirm the proposal (counting from the instant when the leader sends the proposal message). - uint64 commit_log_nanos = 7; + // Time waiting for peers to confirm the proposal (counting from the instant when the leader sends the proposal message). + uint64 commit_log_nanos = 7; - // Wait duration in the apply loop. - uint64 apply_batch_wait_nanos = 8; + // Wait duration in the apply loop. + uint64 apply_batch_wait_nanos = 8; - // Total time spend to applying the log. - uint64 apply_log_nanos = 9; + // Total time spend to applying the log. + uint64 apply_log_nanos = 9; - // Wait time until the KV RocksDB lock is acquired. - uint64 apply_mutex_lock_nanos = 10; + // Wait time until the KV RocksDB lock is acquired. + uint64 apply_mutex_lock_nanos = 10; - // Wait time until becoming the KV RocksDB write leader. - uint64 apply_write_leader_wait_nanos = 11; + // Wait time until becoming the KV RocksDB write leader. + uint64 apply_write_leader_wait_nanos = 11; - // Time spent on writing the KV DB WAL to the disk. - uint64 apply_write_wal_nanos = 12; + // Time spent on writing the KV DB WAL to the disk. + uint64 apply_write_wal_nanos = 12; - // Time spent on writing to the memtable of the KV RocksDB. - uint64 apply_write_memtable_nanos = 13; + // Time spent on writing to the memtable of the KV RocksDB. + uint64 apply_write_memtable_nanos = 13; - // Time spent on waiting in the latch. - uint64 latch_wait_nanos = 14; + // Time spent on waiting in the latch. + uint64 latch_wait_nanos = 14; - // Processing time in the transaction layer. - uint64 process_nanos = 15; + // Processing time in the transaction layer. + uint64 process_nanos = 15; - // Wait time because of the scheduler flow control or quota limiter throttling. - uint64 throttle_nanos = 16; + // Wait time because of the scheduler flow control or quota limiter throttling. + uint64 throttle_nanos = 16; - // Wait time in the waiter manager for pessimistic locking. - uint64 pessimistic_lock_wait_nanos = 17; + // Wait time in the waiter manager for pessimistic locking. + uint64 pessimistic_lock_wait_nanos = 17; } message KvPair { - KeyError error = 1; - bytes key = 2; - bytes value = 3; + KeyError error = 1; + bytes key = 2; + bytes value = 3; + // The commit timestamp of the key. + // If it is zero, it means the commit timestamp is unknown. + uint64 commit_ts = 4; } enum Op { - Put = 0; - Del = 1; - Lock = 2; - Rollback = 3; - // insert operation has a constraint that key should not exist before. - Insert = 4; - PessimisticLock = 5; - CheckNotExists = 6; + Put = 0; + Del = 1; + Lock = 2; + Rollback = 3; + // insert operation has a constraint that key should not exist before. + Insert = 4; + // PessimisticLock is exclusive lock acquired in pessimistic transaction. + PessimisticLock = 5; + CheckNotExists = 6; + // SharedLock likes Lock but in shared mode. + SharedLock = 7; + // SharedPessimisticLock is shared lock acquired in pessimistic transaction. + SharedPessimisticLock = 8; } enum Assertion { - None = 0; - Exist = 1; - NotExist = 2; + None = 0; + Exist = 1; + NotExist = 2; } enum AssertionLevel { - // No assertion. - Off = 0; - // Assertion is enabled, but not enforced when it might affect performance. - Fast = 1; - // Assertion is enabled and enforced. - Strict = 2; + // No assertion. + Off = 0; + // Assertion is enabled, but not enforced when it might affect performance. + Fast = 1; + // Assertion is enabled and enforced. + Strict = 2; } message Mutation { - Op op = 1; - bytes key = 2; - bytes value = 3; - Assertion assertion = 4; + Op op = 1; + bytes key = 2; + bytes value = 3; + Assertion assertion = 4; } message MvccWrite { - Op type = 1; - uint64 start_ts = 2; - uint64 commit_ts = 3; - bytes short_value = 4; - bool has_overlapped_rollback = 5; - bool has_gc_fence = 6; - uint64 gc_fence = 7; - uint64 last_change_ts = 8; - uint64 versions_to_last_change = 9; + Op type = 1; + uint64 start_ts = 2; + uint64 commit_ts = 3; + bytes short_value = 4; + bool has_overlapped_rollback = 5; + bool has_gc_fence = 6; + uint64 gc_fence = 7; + uint64 last_change_ts = 8; + uint64 versions_to_last_change = 9; } message MvccValue { - uint64 start_ts = 1; - bytes value = 2; + uint64 start_ts = 1; + bytes value = 2; } message MvccLock { - Op type = 1; - uint64 start_ts = 2; - bytes primary = 3; - bytes short_value = 4; - uint64 ttl = 5; - uint64 for_update_ts = 6; - uint64 txn_size = 7; - bool use_async_commit = 8; - repeated bytes secondaries = 9; - repeated uint64 rollback_ts = 10; - uint64 last_change_ts = 11; - uint64 versions_to_last_change = 12; + Op type = 1; + uint64 start_ts = 2; + bytes primary = 3; + bytes short_value = 4; + uint64 ttl = 5; + uint64 for_update_ts = 6; + uint64 txn_size = 7; + bool use_async_commit = 8; + repeated bytes secondaries = 9; + repeated uint64 rollback_ts = 10; + uint64 last_change_ts = 11; + uint64 versions_to_last_change = 12; } message MvccInfo { - MvccLock lock = 1; - repeated MvccWrite writes = 2; - repeated MvccValue values = 3; + MvccLock lock = 1; + repeated MvccWrite writes = 2; + repeated MvccValue values = 3; } message TxnInfo { - uint64 txn = 1; - uint64 status = 2; + uint64 txn = 1; + uint64 status = 2; - // Reserved for file based transaction. - bool is_txn_file = 100; + // Reserved for file based transaction. + bool is_txn_file = 100; } enum Action { - NoAction = 0; - TTLExpireRollback = 1; - LockNotExistRollback = 2; - MinCommitTSPushed = 3; - TTLExpirePessimisticRollback = 4; - LockNotExistDoNothing = 5; + NoAction = 0; + TTLExpireRollback = 1; + LockNotExistRollback = 2; + MinCommitTSPushed = 3; + TTLExpirePessimisticRollback = 4; + LockNotExistDoNothing = 5; } message KeyRange { - bytes start_key = 1; - bytes end_key = 2; + bytes start_key = 1; + bytes end_key = 2; } enum ExtraOp { - Noop = 0; - // ReadOldValue represents to output the previous value for delete/update operations. - ReadOldValue = 1; + Noop = 0; + // ReadOldValue represents to output the previous value for delete/update operations. + ReadOldValue = 1; } message LeaderInfo { - uint64 region_id = 1; - uint64 peer_id = 2; - uint64 term = 3; - metapb.RegionEpoch region_epoch = 4; - ReadState read_state = 5; + uint64 region_id = 1; + uint64 peer_id = 2; + uint64 term = 3; + metapb.RegionEpoch region_epoch = 4; + ReadState read_state = 5; } message ReadState { - uint64 applied_index = 1; - uint64 safe_ts = 2; + uint64 applied_index = 1; + uint64 safe_ts = 2; } message CheckLeaderRequest { - repeated LeaderInfo regions = 1; - uint64 ts = 2; + repeated LeaderInfo regions = 1; + uint64 ts = 2; } message CheckLeaderResponse { - repeated uint64 regions = 1; - uint64 ts = 2; + repeated uint64 regions = 1; + uint64 ts = 2; } message StoreSafeTSRequest { - // Get the minimal `safe_ts` from regions that overlap with the key range [`start_key`, `end_key`) - // An empty key range means all regions in the store - KeyRange key_range = 1; + // Get the minimal `safe_ts` from regions that overlap with the key range [`start_key`, `end_key`) + // An empty key range means all regions in the store + KeyRange key_range = 1; } message StoreSafeTSResponse { - uint64 safe_ts = 1; + uint64 safe_ts = 1; } message RawGetKeyTTLRequest { - Context context = 1; - bytes key = 2; - string cf = 3; + Context context = 1; + bytes key = 2; + string cf = 3; } message RawGetKeyTTLResponse { - errorpb.Error region_error = 1; - string error = 2; - uint64 ttl = 3; - bool not_found = 4; + errorpb.Error region_error = 1; + string error = 2; + uint64 ttl = 3; + bool not_found = 4; } message RawCASRequest { - Context context = 1; - bytes key = 2; - bytes value = 3; - bool previous_not_exist = 4; - bytes previous_value = 5; - string cf = 6; - uint64 ttl = 7; + Context context = 1; + bytes key = 2; + bytes value = 3; + bool previous_not_exist = 4; + bytes previous_value = 5; + string cf = 6; + uint64 ttl = 7; + // If `delete` is true, the request is to delete the key when the comparison succeeds + bool delete = 8; } message RawCASResponse { - errorpb.Error region_error = 1; - string error = 2; - bool succeed = 3; - // The previous value regardless of whether the comparison is succeed. - bool previous_not_exist = 4; - bytes previous_value = 5; + errorpb.Error region_error = 1; + string error = 2; + bool succeed = 3; + // The previous value regardless of whether the comparison is succeed. + bool previous_not_exist = 4; + bytes previous_value = 5; } message GetLockWaitInfoRequest { - Context context = 1; - // TODO: There may need some filter options to be used on conditional querying, e.g., finding - // the lock waiting status for some specified transaction. + Context context = 1; + // TODO: There may need some filter options to be used on conditional querying, e.g., finding + // the lock waiting status for some specified transaction. } message GetLockWaitInfoResponse { - errorpb.Error region_error = 1; - string error = 2; - repeated deadlock.WaitForEntry entries = 3; + errorpb.Error region_error = 1; + string error = 2; + repeated deadlock.WaitForEntry entries = 3; } message GetLockWaitHistoryRequest { - Context context = 1; - // TODO: There may need some filter options to be used on conditional querying, e.g., finding - // the lock waiting status for some specified transaction. + Context context = 1; + // TODO: There may need some filter options to be used on conditional querying, e.g., finding + // the lock waiting status for some specified transaction. } message GetLockWaitHistoryResponse { - errorpb.Error region_error = 1; - string error = 2; - repeated deadlock.WaitForEntry entries = 3; + errorpb.Error region_error = 1; + string error = 2; + repeated deadlock.WaitForEntry entries = 3; } message RawCoprocessorRequest { - kvrpcpb.Context context = 1; + kvrpcpb.Context context = 1; - string copr_name = 2; - // Coprorcessor version constraint following SEMVER definition. - string copr_version_req = 3; + string copr_name = 2; + // Coprorcessor version constraint following SEMVER definition. + string copr_version_req = 3; - repeated KeyRange ranges = 4; - bytes data = 5; + repeated KeyRange ranges = 4; + bytes data = 5; } message RawCoprocessorResponse { - errorpb.Error region_error = 1; - // Error message for cases like if no coprocessor with a matching name is found - // or on a version mismatch between plugin_api and the coprocessor. - string error = 2; - bytes data = 3; + errorpb.Error region_error = 1; + // Error message for cases like if no coprocessor with a matching name is found + // or on a version mismatch between plugin_api and the coprocessor. + string error = 2; + bytes data = 3; } enum ChecksumAlgorithm { - Crc64_Xor = 0; + Crc64_Xor = 0; } message RawChecksumRequest { - Context context = 1; - ChecksumAlgorithm algorithm = 2; - repeated KeyRange ranges = 3; + Context context = 1; + ChecksumAlgorithm algorithm = 2; + repeated KeyRange ranges = 3; } message RawChecksumResponse { - errorpb.Error region_error = 1; - string error = 2; - uint64 checksum = 3; - uint64 total_kvs = 4; - uint64 total_bytes = 5; + errorpb.Error region_error = 1; + string error = 2; + uint64 checksum = 3; + uint64 total_kvs = 4; + uint64 total_bytes = 5; } message CompactError { - oneof error { - CompactErrorInvalidStartKey err_invalid_start_key = 1; - CompactErrorPhysicalTableNotExist err_physical_table_not_exist = 2; - CompactErrorCompactInProgress err_compact_in_progress = 3; - CompactErrorTooManyPendingTasks err_too_many_pending_tasks = 4; - } + oneof error { + CompactErrorInvalidStartKey err_invalid_start_key = 1; + CompactErrorPhysicalTableNotExist err_physical_table_not_exist = 2; + CompactErrorCompactInProgress err_compact_in_progress = 3; + CompactErrorTooManyPendingTasks err_too_many_pending_tasks = 4; + } } message CompactErrorInvalidStartKey {} @@ -1510,47 +1679,132 @@ message CompactErrorCompactInProgress {} message CompactErrorTooManyPendingTasks {} message CompactRequest { - // If specified, the compaction will start from this start key. - // If unspecified, the compaction will start from beginning. - // NOTE 1: The start key should be never manually constructed. You should always use a key - // returned in CompactResponse. - // NOTE 2: the compaction range will be always restricted by physical_table_id. - bytes start_key = 1; - - // The physical table that will be compacted. - // - // TODO: this is information that TiKV doesn't need to know. - // See https://github.com/pingcap/kvproto/issues/912 - int64 physical_table_id = 2; - - // The logical table id of the compaction. When receiving parallel requests with the same - // logical table id, err_compact_in_progress will be returned. - // - // TODO: this is information that TiKV doesn't need to know. - // See https://github.com/pingcap/kvproto/issues/912 - int64 logical_table_id = 3; - - // API version of the request - APIVersion api_version = 7; - // Keyspace of the table located in. - uint32 keyspace_id = 8; + // If specified, the compaction will start from this start key. + // If unspecified, the compaction will start from beginning. + // NOTE 1: The start key should be never manually constructed. You should always use a key + // returned in CompactResponse. + // NOTE 2: the compaction range will be always restricted by physical_table_id. + bytes start_key = 1; + + // The physical table that will be compacted. + // + // TODO: this is information that TiKV doesn't need to know. + // See https://github.com/pingcap/kvproto/issues/912 + int64 physical_table_id = 2; + + // The logical table id of the compaction. When receiving parallel requests with the same + // logical table id, err_compact_in_progress will be returned. + // + // TODO: this is information that TiKV doesn't need to know. + // See https://github.com/pingcap/kvproto/issues/912 + int64 logical_table_id = 3; + + // API version of the request + APIVersion api_version = 7; + // Keyspace of the table located in. + uint32 keyspace_id = 8; } message CompactResponse { - CompactError error = 1; + CompactError error = 1; - // The compaction is done incrementally. If there are more data to compact, this field - // will be set. The client can request to compact more data according to the `compacted_end_key`. - bool has_remaining = 2; + // The compaction is done incrementally. If there are more data to compact, this field + // will be set. The client can request to compact more data according to the `compacted_end_key`. + bool has_remaining = 2; - bytes compacted_start_key = 3; - bytes compacted_end_key = 4; + bytes compacted_start_key = 3; + bytes compacted_end_key = 4; } message TiFlashSystemTableRequest { - string sql = 1; + string sql = 1; } message TiFlashSystemTableResponse { - bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; -} + bytes data = 1 [ + (gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", + (gogoproto.nullable) = false + ]; +} + +// Flush is introduced from the pipelined DML protocol. +// A Flush request writes some keys and values to TiKV, storing in LOCK and DEFAULT CF, just like a Prewrite request. +message FlushRequest { + Context context = 1; + repeated Mutation mutations = 2; + bytes primary_key = 3; + uint64 start_ts = 4; + uint64 min_commit_ts = 5; + // generation of the flush request. It is a monotonically increasing number in each transaction. + uint64 generation = 6; + uint64 lock_ttl = 7; + AssertionLevel assertion_level = 8; +} + +message FlushResponse { + errorpb.Error region_error = 1; + repeated KeyError errors = 2; + ExecDetailsV2 exec_details_v2 = 3; +} + +// BufferBatchGet is introduced from the pipelined DML protocol. +// It is similar to a BatchGet request, except that it can only read the data that has been flushed by itself. +message BufferBatchGetRequest { + Context context = 1; + repeated bytes keys = 2; + uint64 version = 3; +} + +message BufferBatchGetResponse { + errorpb.Error region_error = 1; + KeyError error = 2; + repeated KvPair pairs = 3; + // Time and scan details when processing the request. + ExecDetailsV2 exec_details_v2 = 4; +} + +// Actively request TiKV to report health feedback information. TiKV won't omit the health feedback information when sending the +// `BatchCommandsResponse` that contains this response. +// The health feedback information won't be replied in the response, but will be attached to `BatchCommandsResponse.health_feedback` field as usual. +// Only works when batch RPC is enabled. +message GetHealthFeedbackRequest { + Context context = 1; +} + +message GetHealthFeedbackResponse { + // The error field is added for keeping consistent. This request won't meet any region error as it's store level rather than region level. + errorpb.Error region_error = 1; + HealthFeedback health_feedback = 2; +} + +message HealthFeedback { + uint64 store_id = 1; + // The sequence number of the feedback message. + // It's defined as an incrementing integer, starting from the unix timestamp (milliseconds) at + // the time that the TiKV node is started. + // This can be useful for filtering out out-of-order feedback messages. + // Note that considering the possibility of system clock changing, this field doesn't guarantee + // uniqueness and monotonic if the TiKV node is restarted. + uint64 feedback_seq_no = 2; + // The slow_score calculated in raftstore module. Due to some limitations of slow score, this would + // be replaced by `SlowTrend` in the future. + int32 slow_score = 3; +} + +message BroadcastTxnStatusRequest { + Context context = 1; + repeated TxnStatus txn_status = 2; +} + +message TxnStatus { + uint64 start_ts = 1; + // a non-zero min_commit_ts indicates the transaction is ongoing + uint64 min_commit_ts = 2; + // a non-zero commit_ts indicates the transaction is committed + uint64 commit_ts = 3; + bool rolled_back = 4; + // The txn has unlocked all keys, implying that it can be removed from txn_status_cache. + bool is_completed = 5; +} + +message BroadcastTxnStatusResponse {} diff --git a/proto/logbackuppb.proto b/proto/logbackuppb.proto index 17e9820f..ff92667c 100644 --- a/proto/logbackuppb.proto +++ b/proto/logbackuppb.proto @@ -1,58 +1,73 @@ syntax = "proto3"; package logbackup; +import "errorpb.proto"; import "gogoproto/gogo.proto"; import "rustproto.proto"; -import "errorpb.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // The minimal information for identify a region. message RegionIdentity { - uint64 id = 1; - uint64 epoch_version = 2; - // We omitted epoch_conf_version because config change won't make range change. + uint64 id = 1; + uint64 epoch_version = 2; + // We omitted epoch_conf_version because config change won't make range change. } // The last flush ts with region information. message RegionCheckpoint { - errorpb.Error err = 1; - RegionIdentity region = 2; - uint64 checkpoint = 3; + errorpb.Error err = 1; + RegionIdentity region = 2; + uint64 checkpoint = 3; } message GetLastFlushTSOfRegionRequest { - repeated RegionIdentity regions = 1; + repeated RegionIdentity regions = 1; } message GetLastFlushTSOfRegionResponse { - repeated RegionCheckpoint checkpoints = 1; + repeated RegionCheckpoint checkpoints = 1; } message SubscribeFlushEventRequest { - string client_id = 1; + string client_id = 1; } message SubscribeFlushEventResponse { - repeated FlushEvent events = 1; + repeated FlushEvent events = 1; } message FlushEvent { - bytes start_key = 1; - bytes end_key = 2; - uint64 checkpoint = 3; + bytes start_key = 1; + bytes end_key = 2; + uint64 checkpoint = 3; +} + +message FlushNowRequest {} + +message FlushNowResponse { + repeated FlushResult results = 1; +} + +message FlushResult { + string task_name = 1; + bool success = 2; + string error_message = 3; } // The log backup service. -// Generally, most essential interfaces of log backup (say, checkpoint management, task management) are +// Generally, most essential interfaces of log backup (say, checkpoint management, task management) are // provided by adding some key in the embed etcd of PD. // This interface is mainly provided for the checkpoint advancer and debug usage. service LogBackup { - rpc GetLastFlushTSOfRegion(GetLastFlushTSOfRegionRequest) returns (GetLastFlushTSOfRegionResponse) {} - rpc SubscribeFlushEvent(SubscribeFlushEventRequest) returns (stream SubscribeFlushEventResponse) {} + rpc GetLastFlushTSOfRegion(GetLastFlushTSOfRegionRequest) returns (GetLastFlushTSOfRegionResponse) {} + rpc SubscribeFlushEvent(SubscribeFlushEventRequest) returns (stream SubscribeFlushEventResponse) {} + rpc FlushNow(FlushNowRequest) returns (FlushNowResponse) {} } diff --git a/proto/meta_storagepb.proto b/proto/meta_storagepb.proto index bae7ae57..40a8d2aa 100644 --- a/proto/meta_storagepb.proto +++ b/proto/meta_storagepb.proto @@ -4,96 +4,125 @@ package meta_storagepb; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // MetaStorage is the meta storage service. service MetaStorage { - rpc Watch(WatchRequest) returns (stream WatchResponse) {} + rpc Watch(WatchRequest) returns (stream WatchResponse) {} + + // Get is the same as etcd Range which might be implemented in a more common way + // so that we can use other storages to replace etcd in the future. + rpc Get(GetRequest) returns (GetResponse) {} - // Get is the same as etcd Range which might be implemented in a more common way - // so that we can use other storages to replace etcd in the future. - rpc Get(GetRequest) returns (GetResponse) {} + rpc Put(PutRequest) returns (PutResponse) {} - rpc Put(PutRequest) returns (PutResponse) {} + // Delete is the same as etcd Range which might be implemented in a more common way + // so that we can use other storages to replace etcd in the future. + rpc Delete(DeleteRequest) returns (DeleteResponse) {} } enum ErrorType { - OK = 0; - UNKNOWN = 1; - // required watch revision is smaller than current compact/min revision. - DATA_COMPACTED = 2; + OK = 0; + UNKNOWN = 1; + // required watch revision is smaller than current compact/min revision. + DATA_COMPACTED = 2; } message Error { - ErrorType type = 1; - string message = 2; + ErrorType type = 1; + string message = 2; } message RequestHeader { - // cluster_id is the ID of the cluster which be sent to. - uint64 cluster_id = 1; - // source is the source of the request. - string source = 2; + // cluster_id is the ID of the cluster which be sent to. + uint64 cluster_id = 1; + // source is the source of the request. + string source = 2; } message ResponseHeader { - // cluster_id is the ID of the cluster which sent the response. - uint64 cluster_id = 1; - Error error = 2; - int64 revision = 3; + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; + int64 revision = 3; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message WatchRequest { - RequestHeader header = 1; - bytes key = 2; - bytes range_end = 3; - int64 start_revision = 4; - bool prev_kv = 5; + RequestHeader header = 1; + bytes key = 2; + bytes range_end = 3; + int64 start_revision = 4; + bool prev_kv = 5; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message WatchResponse { - ResponseHeader header = 1; - int64 compact_revision = 2; - repeated Event events = 3; + ResponseHeader header = 1; + int64 compact_revision = 2; + repeated Event events = 3; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message GetRequest { - RequestHeader header = 1; - bytes key = 2; - bytes range_end = 3; - int64 limit = 4; - int64 revision = 5; + RequestHeader header = 1; + bytes key = 2; + bytes range_end = 3; + int64 limit = 4; + int64 revision = 5; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message GetResponse { - ResponseHeader header = 1; - repeated KeyValue kvs = 2; - bool more = 3; - int64 count = 4; + ResponseHeader header = 1; + repeated KeyValue kvs = 2; + bool more = 3; + int64 count = 4; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message PutRequest { - RequestHeader header = 1; - bytes key = 2; - bytes value = 3; - int64 lease = 4; - bool prev_kv = 5; + RequestHeader header = 1; + bytes key = 2; + bytes value = 3; + int64 lease = 4; + bool prev_kv = 5; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message PutResponse { - ResponseHeader header = 1; - KeyValue prev_kv = 2; + ResponseHeader header = 1; + KeyValue prev_kv = 2; +} + +// copied from etcd https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/mvccpb/kv.proto +message DeleteRequest { + RequestHeader header = 1; + bytes key = 2; + // range_end is the key following the last key to delete for the range [key, range_end). + // If range_end is not given, the range is defined to contain only the key argument. + // If range_end is one bit larger than the given key, then the range is all the keys + // with the prefix (the given key). + // If range_end is '\0', the range is all keys greater than or equal to the key argument. + bytes range_end = 3; + + // If prev_kv is set, etcd gets the previous key-value pairs before deleting it. + // The previous key-value pairs will be returned in the delete response. + bool prev_kv = 4; +} + +message DeleteResponse { + ResponseHeader header = 1; + // deleted is the number of keys deleted by the delete range request. + int64 deleted = 2; + repeated KeyValue prev_kvs = 3; } // copied from etcd https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/mvccpb/kv.proto diff --git a/proto/metapb.proto b/proto/metapb.proto index ef3c8d45..4e38559e 100644 --- a/proto/metapb.proto +++ b/proto/metapb.proto @@ -5,25 +5,27 @@ import "encryptionpb.proto"; import "gogoproto/gogo.proto"; import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - message Cluster { - uint64 id = 1; - // max peer count for a region. - // pd will do the auto-balance if region peer count mismatches. - uint32 max_peer_count = 2; - // more attributes...... + uint64 id = 1; + // max peer count for a region. + // pd will do the auto-balance if region peer count mismatches. + uint32 max_peer_count = 2; + // more attributes...... } enum StoreState { - Up = 0; - Offline = 1; - Tombstone = 2; + Up = 0; + Offline = 1; + Tombstone = 2; } // NodeState is going to replace StoreState to make the state concept more clear. @@ -31,122 +33,129 @@ enum StoreState { // "Removing" is just like previous `Offline` which is more accurate. // "Removed" has the same meaning with `Tombstone`. enum NodeState { - Preparing = 0; - Serving = 1; - Removing = 2; - Removed = 3; + Preparing = 0; + Serving = 1; + Removing = 2; + Removed = 3; } // Case insensitive key/value for replica constraints. message StoreLabel { - string key = 1; - string value = 2; + string key = 1; + string value = 2; } message Store { - uint64 id = 1; - // Address to handle client requests (kv, cop, etc.) - string address = 2; - StoreState state = 3; - repeated StoreLabel labels = 4; - string version = 5; - // Address to handle peer requests (raft messages from other store). - // Empty means same as address. - string peer_address = 6; - // Status address provides the HTTP service for external components - string status_address = 7; - string git_hash = 8; - // The start timestamp of the current store - int64 start_timestamp = 9; - string deploy_path = 10; - // The last heartbeat timestamp of the store. - int64 last_heartbeat = 11; - // If the store is physically destroyed, which means it can never up again. - bool physically_destroyed = 12; - // NodeState is used to replace StoreState which will be deprecated in the future. - NodeState node_state = 13; + uint64 id = 1; + // Address to handle client requests (kv, cop, etc.) + string address = 2; + StoreState state = 3; + repeated StoreLabel labels = 4; + string version = 5; + // Address to handle peer requests (raft messages from other store). + // Empty means same as address. + string peer_address = 6; + // Status address provides the HTTP service for external components + string status_address = 7; + string git_hash = 8; + // The start timestamp of the current store + int64 start_timestamp = 9; + string deploy_path = 10; + // The last heartbeat timestamp of the store. + int64 last_heartbeat = 11; + // If the store is physically destroyed, which means it can never up again. + bool physically_destroyed = 12; + // NodeState is used to replace StoreState which will be deprecated in the future. + NodeState node_state = 13; } message RegionEpoch { - // Conf change version, auto increment when add or remove peer - uint64 conf_ver = 1; - // Region version, auto increment when split or merge - uint64 version = 2; + // Conf change version, auto increment when add or remove peer + uint64 conf_ver = 1; + // Region version, auto increment when split or merge + uint64 version = 2; } message BucketStats { - // total read in bytes of each bucket - repeated uint64 read_bytes = 1; + // total read in bytes of each bucket + repeated uint64 read_bytes = 1; - // total write in bytes of each bucket - repeated uint64 write_bytes = 2; + // total write in bytes of each bucket + repeated uint64 write_bytes = 2; - // total read qps of each bucket - repeated uint64 read_qps = 3; + // total read qps of each bucket + repeated uint64 read_qps = 3; - // total write qps of each bucket - repeated uint64 write_qps = 4; + // total write qps of each bucket + repeated uint64 write_qps = 4; - // total read keys of each bucket - repeated uint64 read_keys = 5; + // total read keys of each bucket + repeated uint64 read_keys = 5; - // total write keys of each bucket - repeated uint64 write_keys = 6; + // total write keys of each bucket + repeated uint64 write_keys = 6; } message Buckets { - uint64 region_id = 1; + uint64 region_id = 1; - // A hint indicate if keys have changed. - uint64 version = 2; + // A hint indicate if keys have changed. + uint64 version = 2; - // keys of buckets, include start/end key of region - repeated bytes keys = 3; + // keys of buckets, include start/end key of region + repeated bytes keys = 3; - // bucket stats - BucketStats stats = 4; + // bucket stats + BucketStats stats = 4; + + // The period in milliseconds that stats are collected with in + uint64 period_in_ms = 5; +} - // The period in milliseconds that stats are collected with in - uint64 period_in_ms = 5; +message BucketMeta { + // A hint indicate if keys have changed. + uint64 version = 1; + // keys of buckets, include start/end key of region + repeated bytes keys = 2; } message Region { - uint64 id = 1; - // Region key range [start_key, end_key). - bytes start_key = 2; - bytes end_key = 3; - RegionEpoch region_epoch = 4; - repeated Peer peers = 5; - // Encryption metadata for start_key and end_key. encryption_meta.iv is IV for start_key. - // IV for end_key is calculated from (encryption_meta.iv + len(start_key)). - // The field is only used by PD and should be ignored otherwise. - // If encryption_meta is empty (i.e. nil), it means start_key and end_key are unencrypted. - encryptionpb.EncryptionMeta encryption_meta = 6; - // The flashback state indicates whether this region is in the flashback state. - // TODO: only check by `flashback_start_ts` in the future. Keep for compatibility now. - bool is_in_flashback = 7; - // The start_ts that the current flashback progress is using. - uint64 flashback_start_ts = 8; + uint64 id = 1; + // Region key range [start_key, end_key). + bytes start_key = 2; + bytes end_key = 3; + RegionEpoch region_epoch = 4; + repeated Peer peers = 5; + // Encryption metadata for start_key and end_key. encryption_meta.iv is IV for start_key. + // IV for end_key is calculated from (encryption_meta.iv + len(start_key)). + // The field is only used by PD and should be ignored otherwise. + // If encryption_meta is empty (i.e. nil), it means start_key and end_key are unencrypted. + encryptionpb.EncryptionMeta encryption_meta = 6; + // The flashback state indicates whether this region is in the flashback state. + // TODO: only check by `flashback_start_ts` in the future. Keep for compatibility now. + bool is_in_flashback = 7; + // The start_ts that the current flashback progress is using. + uint64 flashback_start_ts = 8; } enum PeerRole { - // Voter -> Voter - Voter = 0; - // Learner/None -> Learner - Learner = 1; - // Learner/None -> Voter - IncomingVoter = 2; - // Voter -> Learner - DemotingVoter = 3; - // We forbid Voter -> None, it can introduce unavailability as discussed in - // etcd-io/etcd#7625 - // Learner -> None can be apply directly, doesn't need to be stored as - // joint state. + // Voter -> Voter + Voter = 0; + // Learner/None -> Learner + Learner = 1; + // Learner/None -> Voter + IncomingVoter = 2; + // Voter -> Learner + DemotingVoter = 3; + // We forbid Voter -> None, it can introduce unavailability as discussed in + // etcd-io/etcd#7625 + // Learner -> None can be apply directly, doesn't need to be stored as + // joint state. } message Peer { - uint64 id = 1; - uint64 store_id = 2; - PeerRole role = 3; - bool is_witness = 4; + uint64 id = 1; + uint64 store_id = 2; + PeerRole role = 3; + bool is_witness = 4; } diff --git a/proto/mpp.proto b/proto/mpp.proto index 268e218d..1035ac00 100644 --- a/proto/mpp.proto +++ b/proto/mpp.proto @@ -1,103 +1,110 @@ syntax = "proto3"; package mpp; -import "gogoproto/gogo.proto"; import "coprocessor.proto"; -import "metapb.proto"; +import "gogoproto/gogo.proto"; import "kvrpcpb.proto"; +import "metapb.proto"; +import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; - -option java_package = "org.tikv.kvproto"; +option (rustproto.lite_runtime_all) = true; // TaskMeta contains meta of a mpp plan, including query's ts and task address. message TaskMeta { - uint64 start_ts = 1; // start ts of a query - int64 task_id = 2; // if task id is -1 , it indicates a tidb task. - int64 partition_id = 3; // Only used for hash partition - string address = 4; // target address of this task. - uint64 gather_id = 5; // used to distinguish different gathers in the mpp query. - uint64 query_ts = 6; // timestamp when start to execute query, used for TiFlash miniTSO schedule. - uint64 local_query_id = 7; // unique local query_id if tidb don't restart. So we can use gather_id + query_ts + local_query_id + server_id to represent a global unique query. - uint64 server_id = 8; // TiDB server id - int64 mpp_version = 9; // mpp version - uint32 keyspace_id = 10; // keyspace id of the request - string coordinator_address = 11; // coordinator_address of this query - bool report_execution_summary = 12; // Only when coordinator_address is not empty, this flag can be true. When set to true, TiFlash only report execution summary through ReportMPPTaskStatus service, don't include summaries in MppDataPacket - kvrpcpb.APIVersion api_version = 16; // API version of the request - string resource_group_name = 17; - uint64 connection_id = 18; // This is the session id between a client and tidb - string connection_alias = 19; // This is the session alias between a client and tidb + uint64 start_ts = 1; // start ts of a query + int64 task_id = 2; // if task id is -1 , it indicates a tidb task. + int64 partition_id = 3; // Only used for hash partition + string address = 4; // target address of this task. + uint64 gather_id = 5; // used to distinguish different gathers in the mpp query. + uint64 query_ts = 6; // timestamp when start to execute query, used for TiFlash miniTSO schedule. + uint64 local_query_id = 7; // unique local query_id if tidb don't restart. So we can use gather_id + query_ts + local_query_id + server_id to represent a global unique query. + uint64 server_id = 8; // TiDB server id + int64 mpp_version = 9; // mpp version + uint32 keyspace_id = 10; // keyspace id of the request + string coordinator_address = 11; // coordinator_address of this query + bool report_execution_summary = 12; // Only when coordinator_address is not empty, this flag can be true. When set to true, TiFlash only report execution summary through ReportMPPTaskStatus service, don't include summaries in MppDataPacket + kvrpcpb.APIVersion api_version = 16; // API version of the request + string resource_group_name = 17; + uint64 connection_id = 18; // This is the session id between a client and tidb + string connection_alias = 19; // This is the session alias between a client and tidb + string sql_digest = 20; + string plan_digest = 21; } -message IsAliveRequest { -} +message IsAliveRequest {} message IsAliveResponse { - bool available = 1; - int64 mpp_version = 2; + bool available = 1; + int64 mpp_version = 2; } // Dipsatch the task request to different tiflash servers. message DispatchTaskRequest { - TaskMeta meta = 1; - bytes encoded_plan = 2; - int64 timeout = 3; - repeated coprocessor.RegionInfo regions = 4; - // If this task contains table scan, we still need their region info. - int64 schema_ver = 5; - // Used for partition table scan - repeated coprocessor.TableRegions table_regions = 6; + TaskMeta meta = 1; + bytes encoded_plan = 2; + int64 timeout = 3; + repeated coprocessor.RegionInfo regions = 4; + // If this task contains table scan, we still need their region info. + int64 schema_ver = 5; + // Used for partition table scan + repeated coprocessor.TableRegions table_regions = 6; + // Shard infos for TiCI/FTS routing in MPP dispatch path. + repeated coprocessor.TableShardInfos table_shard_infos = 7; } // Get response of DispatchTaskRequest. message DispatchTaskResponse { - Error error = 1; - repeated metapb.Region retry_regions = 2; + Error error = 1; + repeated metapb.Region retry_regions = 2; } // CancelTaskRequest closes the execution of a task. message CancelTaskRequest { - TaskMeta meta = 1; - Error error = 2; + TaskMeta meta = 1; + Error error = 2; } message CancelTaskResponse { - Error error = 1; + Error error = 1; } // ReportTaskStatus reports the execution status of a task. // when TiFlash reports status to TiDB, ReportTaskStatusRequest serialize tipb.TiFlashExecutionInfo into data; message ReportTaskStatusRequest { - TaskMeta meta = 1; - bytes data = 2; - Error error = 3; + TaskMeta meta = 1; + bytes data = 2; + Error error = 3; } message ReportTaskStatusResponse { - Error error = 1; + Error error = 1; } // build connection between different tasks. Data is sent by the tasks that are closer to the data sources. message EstablishMPPConnectionRequest { - TaskMeta sender_meta = 1; // node closer to the source - TaskMeta receiver_meta = 2; // node closer to the tidb mpp gather. + TaskMeta sender_meta = 1; // node closer to the source + TaskMeta receiver_meta = 2; // node closer to the tidb mpp gather. } // when TiFlash sends data to TiDB, Data packets wrap tipb.SelectResponse, i.e., serialize tipb.SelectResponse into data; // when TiFlash sends data to TiFlash, data blocks are serialized into chunks, and the execution_summaries in tipb.SelectResponse are serialized into data only for the last packet. message MPPDataPacket { - bytes data = 1; - Error error = 2; - repeated bytes chunks = 3; - repeated uint64 stream_ids = 4; - int64 version = 5; // version of data packet format + bytes data = 1; + Error error = 2; + repeated bytes chunks = 3; + repeated uint64 stream_ids = 4; + int64 version = 5; // version of data packet format } message Error { - int32 code = 1; - string msg = 2; - int64 mpp_version = 3; + int32 code = 1; + string msg = 2; + int64 mpp_version = 3; } diff --git a/proto/pdpb.proto b/proto/pdpb.proto index c8fec849..5142625e 100644 --- a/proto/pdpb.proto +++ b/proto/pdpb.proto @@ -1,1127 +1,1423 @@ syntax = "proto3"; package pdpb; -import "metapb.proto"; import "eraftpb.proto"; +import "gogoproto/gogo.proto"; +import "metapb.proto"; import "raft_serverpb.proto"; import "replication_modepb.proto"; - -import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - service PD { - // GetClusterInfo get the information of this cluster. It does not require - // the cluster_id in request matchs the id of this cluster. - rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {} + // GetClusterInfo get the information of this cluster. It does not require + // the cluster_id in request matchs the id of this cluster. + rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {} + + // GetMembers get the member list of this cluster. It does not require + // the cluster_id in request matchs the id of this cluster. + rpc GetMembers(GetMembersRequest) returns (GetMembersResponse) {} + + rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} + + rpc Bootstrap(BootstrapRequest) returns (BootstrapResponse) {} - // GetMembers get the member list of this cluster. It does not require - // the cluster_id in request matchs the id of this cluster. - rpc GetMembers(GetMembersRequest) returns (GetMembersResponse) {} + rpc IsBootstrapped(IsBootstrappedRequest) returns (IsBootstrappedResponse) {} - rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} + rpc AllocID(AllocIDRequest) returns (AllocIDResponse) {} - rpc Bootstrap(BootstrapRequest) returns (BootstrapResponse) {} + rpc IsSnapshotRecovering(IsSnapshotRecoveringRequest) returns (IsSnapshotRecoveringResponse) {} - rpc IsBootstrapped(IsBootstrappedRequest) returns (IsBootstrappedResponse) {} + rpc GetStore(GetStoreRequest) returns (GetStoreResponse) {} - rpc AllocID(AllocIDRequest) returns (AllocIDResponse) {} + rpc PutStore(PutStoreRequest) returns (PutStoreResponse) {} - rpc IsSnapshotRecovering(IsSnapshotRecoveringRequest) returns (IsSnapshotRecoveringResponse) {} + rpc GetAllStores(GetAllStoresRequest) returns (GetAllStoresResponse) {} - rpc GetStore(GetStoreRequest) returns (GetStoreResponse) {} + rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} - rpc PutStore(PutStoreRequest) returns (PutStoreResponse) {} + rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {} - rpc GetAllStores(GetAllStoresRequest) returns (GetAllStoresResponse) {} + rpc GetRegion(GetRegionRequest) returns (GetRegionResponse) {} - rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} + rpc GetPrevRegion(GetRegionRequest) returns (GetRegionResponse) {} - rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {} + rpc GetRegionByID(GetRegionByIDRequest) returns (GetRegionResponse) {} - rpc GetRegion(GetRegionRequest) returns (GetRegionResponse) {} + rpc QueryRegion(stream QueryRegionRequest) returns (stream QueryRegionResponse) {} - rpc GetPrevRegion(GetRegionRequest) returns (GetRegionResponse) {} + // Deprecated: use BatchScanRegions instead. + rpc ScanRegions(ScanRegionsRequest) returns (ScanRegionsResponse) {} - rpc GetRegionByID(GetRegionByIDRequest) returns (GetRegionResponse) {} + rpc BatchScanRegions(BatchScanRegionsRequest) returns (BatchScanRegionsResponse) {} - rpc ScanRegions(ScanRegionsRequest) returns (ScanRegionsResponse) {} + rpc AskSplit(AskSplitRequest) returns (AskSplitResponse) { + // Use AskBatchSplit instead. + option deprecated = true; + } - rpc AskSplit(AskSplitRequest) returns (AskSplitResponse) { - // Use AskBatchSplit instead. - option deprecated = true; - } + rpc ReportSplit(ReportSplitRequest) returns (ReportSplitResponse) { + // Use ResportBatchSplit instead. + option deprecated = true; + } - rpc ReportSplit(ReportSplitRequest) returns (ReportSplitResponse) { - // Use ResportBatchSplit instead. - option deprecated = true; - } + rpc AskBatchSplit(AskBatchSplitRequest) returns (AskBatchSplitResponse) {} - rpc AskBatchSplit(AskBatchSplitRequest) returns (AskBatchSplitResponse) {} + rpc ReportBatchSplit(ReportBatchSplitRequest) returns (ReportBatchSplitResponse) {} - rpc ReportBatchSplit(ReportBatchSplitRequest) returns (ReportBatchSplitResponse) {} + rpc GetClusterConfig(GetClusterConfigRequest) returns (GetClusterConfigResponse) {} - rpc GetClusterConfig(GetClusterConfigRequest) returns (GetClusterConfigResponse) {} + rpc PutClusterConfig(PutClusterConfigRequest) returns (PutClusterConfigResponse) {} - rpc PutClusterConfig(PutClusterConfigRequest) returns (PutClusterConfigResponse) {} + rpc ScatterRegion(ScatterRegionRequest) returns (ScatterRegionResponse) {} - rpc ScatterRegion(ScatterRegionRequest) returns (ScatterRegionResponse) {} + rpc GetGCSafePoint(GetGCSafePointRequest) returns (GetGCSafePointResponse) {} - rpc GetGCSafePoint(GetGCSafePointRequest) returns (GetGCSafePointResponse) {} + rpc UpdateGCSafePoint(UpdateGCSafePointRequest) returns (UpdateGCSafePointResponse) {} - rpc UpdateGCSafePoint(UpdateGCSafePointRequest) returns (UpdateGCSafePointResponse) {} + rpc UpdateServiceGCSafePoint(UpdateServiceGCSafePointRequest) returns (UpdateServiceGCSafePointResponse) {} - rpc UpdateServiceGCSafePoint(UpdateServiceGCSafePointRequest) returns (UpdateServiceGCSafePointResponse) {} + rpc GetGCSafePointV2(GetGCSafePointV2Request) returns (GetGCSafePointV2Response) {} - rpc GetGCSafePointV2(GetGCSafePointV2Request) returns (GetGCSafePointV2Response) {} + rpc WatchGCSafePointV2(WatchGCSafePointV2Request) returns (stream WatchGCSafePointV2Response) {} - rpc WatchGCSafePointV2(WatchGCSafePointV2Request) returns (stream WatchGCSafePointV2Response) {} + rpc UpdateGCSafePointV2(UpdateGCSafePointV2Request) returns (UpdateGCSafePointV2Response) {} - rpc UpdateGCSafePointV2(UpdateGCSafePointV2Request) returns (UpdateGCSafePointV2Response) {} + rpc UpdateServiceSafePointV2(UpdateServiceSafePointV2Request) returns (UpdateServiceSafePointV2Response) {} - rpc UpdateServiceSafePointV2(UpdateServiceSafePointV2Request) returns (UpdateServiceSafePointV2Response) {} + rpc GetAllGCSafePointV2(GetAllGCSafePointV2Request) returns (GetAllGCSafePointV2Response) {} - rpc GetAllGCSafePointV2(GetAllGCSafePointV2Request) returns (GetAllGCSafePointV2Response) {} + rpc AdvanceGCSafePoint(AdvanceGCSafePointRequest) returns (AdvanceGCSafePointResponse) {} - rpc SyncRegions(stream SyncRegionRequest) returns (stream SyncRegionResponse) {} + rpc AdvanceTxnSafePoint(AdvanceTxnSafePointRequest) returns (AdvanceTxnSafePointResponse) {} - rpc GetOperator(GetOperatorRequest) returns (GetOperatorResponse) {} + rpc SetGCBarrier(SetGCBarrierRequest) returns (SetGCBarrierResponse) {} - rpc SyncMaxTS(SyncMaxTSRequest) returns (SyncMaxTSResponse) {} + rpc DeleteGCBarrier(DeleteGCBarrierRequest) returns (DeleteGCBarrierResponse) {} - rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} + rpc SetGlobalGCBarrier(SetGlobalGCBarrierRequest) returns (SetGlobalGCBarrierResponse) {} - rpc SplitAndScatterRegions(SplitAndScatterRegionsRequest) returns (SplitAndScatterRegionsResponse) {} + rpc DeleteGlobalGCBarrier(DeleteGlobalGCBarrierRequest) returns (DeleteGlobalGCBarrierResponse) {} - rpc GetDCLocationInfo(GetDCLocationInfoRequest) returns (GetDCLocationInfoResponse) {} + rpc GetGCState(GetGCStateRequest) returns (GetGCStateResponse) {} - rpc StoreGlobalConfig(StoreGlobalConfigRequest) returns (StoreGlobalConfigResponse) {} + rpc GetAllKeyspacesGCStates(GetAllKeyspacesGCStatesRequest) returns (GetAllKeyspacesGCStatesResponse) {} - rpc LoadGlobalConfig(LoadGlobalConfigRequest) returns (LoadGlobalConfigResponse) {} + rpc SyncRegions(stream SyncRegionRequest) returns (stream SyncRegionResponse) {} - rpc WatchGlobalConfig(WatchGlobalConfigRequest) returns (stream WatchGlobalConfigResponse) {} + rpc GetOperator(GetOperatorRequest) returns (GetOperatorResponse) {} - rpc ReportBuckets(stream ReportBucketsRequest) returns (ReportBucketsResponse) {} + rpc SyncMaxTS(SyncMaxTSRequest) returns (SyncMaxTSResponse) {} - rpc ReportMinResolvedTS(ReportMinResolvedTsRequest) returns (ReportMinResolvedTsResponse) {} + rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} - rpc SetExternalTimestamp(SetExternalTimestampRequest) returns (SetExternalTimestampResponse) {} + rpc SplitAndScatterRegions(SplitAndScatterRegionsRequest) returns (SplitAndScatterRegionsResponse) {} - rpc GetExternalTimestamp(GetExternalTimestampRequest) returns (GetExternalTimestampResponse) {} + rpc GetDCLocationInfo(GetDCLocationInfoRequest) returns (GetDCLocationInfoResponse) {} - // Get the minimum timestamp across all keyspace groups from API server - // TODO: Currently, we need to ask API server to get the minimum timestamp. - // Once we support service discovery, we can remove it. - rpc GetMinTS (GetMinTSRequest) returns (GetMinTSResponse) {} + rpc StoreGlobalConfig(StoreGlobalConfigRequest) returns (StoreGlobalConfigResponse) {} + + rpc LoadGlobalConfig(LoadGlobalConfigRequest) returns (LoadGlobalConfigResponse) {} + + rpc WatchGlobalConfig(WatchGlobalConfigRequest) returns (stream WatchGlobalConfigResponse) {} + + rpc ReportBuckets(stream ReportBucketsRequest) returns (ReportBucketsResponse) {} + + rpc ReportMinResolvedTS(ReportMinResolvedTsRequest) returns (ReportMinResolvedTsResponse) {} + + rpc SetExternalTimestamp(SetExternalTimestampRequest) returns (SetExternalTimestampResponse) {} + + rpc GetExternalTimestamp(GetExternalTimestampRequest) returns (GetExternalTimestampResponse) {} + + // Get the minimum timestamp across all keyspace groups from API server + // TODO: Currently, we need to ask API server to get the minimum timestamp. + // Once we support service discovery, we can remove it. + rpc GetMinTS(GetMinTSRequest) returns (GetMinTSResponse) {} } message WatchGlobalConfigRequest { - string config_path = 1; - int64 revision = 2; + string config_path = 1; + int64 revision = 2; } message WatchGlobalConfigResponse { - repeated GlobalConfigItem changes = 1; - int64 revision = 2; - ResponseHeader header = 3; + repeated GlobalConfigItem changes = 1; + int64 revision = 2; + ResponseHeader header = 3; } message StoreGlobalConfigRequest { - repeated GlobalConfigItem changes = 1; - string config_path = 2; + repeated GlobalConfigItem changes = 1; + string config_path = 2; } message StoreGlobalConfigResponse { - Error error = 1; + Error error = 1; } message LoadGlobalConfigRequest { - repeated string names = 1; - string config_path = 2; + repeated string names = 1; + string config_path = 2; } message LoadGlobalConfigResponse { - repeated GlobalConfigItem items = 1; - int64 revision = 2; + repeated GlobalConfigItem items = 1; + int64 revision = 2; } enum EventType { - PUT = 0; - DELETE = 1; + PUT = 0; + DELETE = 1; } message GlobalConfigItem { - string name = 1; - // this field 'value' is replaced by the field 'payload'. - string value = 2; - Error error = 3; - EventType kind = 4; - // Since item value needs to support marshal of different struct types, - // it should be set to bytes instead of string. - bytes payload = 5; + string name = 1; + // this field 'value' is replaced by the field 'payload'. + string value = 2; + Error error = 3; + EventType kind = 4; + // Since item value needs to support marshal of different struct types, + // it should be set to bytes instead of string. + bytes payload = 5; } message RequestHeader { - // cluster_id is the ID of the cluster which be sent to. - uint64 cluster_id = 1; - // sender_id is the ID of the sender server, also member ID or etcd ID. - uint64 sender_id = 2; + // cluster_id is the ID of the cluster which be sent to. + uint64 cluster_id = 1; + // sender_id is the ID of the sender server, also member ID or etcd ID. + // sender_id is used in PD internal communication. + uint64 sender_id = 2; + // caller_id is the ID of the client which sends the request, such as tikv, + // tidb, cdc, etc. + string caller_id = 3; + // caller_component is the component of the client which sends the request, + // such as ddl, optimizer, etc. + string caller_component = 4; } message ResponseHeader { - // cluster_id is the ID of the cluster which sent the response. - uint64 cluster_id = 1; - Error error = 2; + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; } enum ErrorType { - OK = 0; - UNKNOWN = 1; - NOT_BOOTSTRAPPED = 2; - STORE_TOMBSTONE = 3; - ALREADY_BOOTSTRAPPED = 4; - INCOMPATIBLE_VERSION = 5; - REGION_NOT_FOUND = 6; - GLOBAL_CONFIG_NOT_FOUND = 7; - DUPLICATED_ENTRY = 8; - ENTRY_NOT_FOUND = 9; - INVALID_VALUE = 10; - // required watch revision is smaller than current compact/min revision. - DATA_COMPACTED = 11; + OK = 0; + UNKNOWN = 1; + NOT_BOOTSTRAPPED = 2; + STORE_TOMBSTONE = 3; + ALREADY_BOOTSTRAPPED = 4; + INCOMPATIBLE_VERSION = 5; + REGION_NOT_FOUND = 6; + GLOBAL_CONFIG_NOT_FOUND = 7; + DUPLICATED_ENTRY = 8; + ENTRY_NOT_FOUND = 9; + INVALID_VALUE = 10; + // required watch revision is smaller than current compact/min revision. + DATA_COMPACTED = 11; + REGIONS_NOT_CONTAIN_ALL_KEY_RANGE = 12; } message Error { - ErrorType type = 1; - string message = 2; + ErrorType type = 1; + string message = 2; } message TsoRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 count = 2; - string dc_location = 3; + uint32 count = 2; + string dc_location = 3; } message Timestamp { - int64 physical = 1; - int64 logical = 2; - // Number of suffix bits used for global distinction, - // PD client will use this to compute a TSO's logical part. - uint32 suffix_bits = 3; + int64 physical = 1; + int64 logical = 2; + // Number of suffix bits used for global distinction, + // PD client will use this to compute a TSO's logical part. + uint32 suffix_bits = 3; } message TsoResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint32 count = 2; - Timestamp timestamp = 3; + uint32 count = 2; + Timestamp timestamp = 3; } message BootstrapRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Store store = 2; - metapb.Region region = 3; + metapb.Store store = 2; + metapb.Region region = 3; } message BootstrapResponse { - ResponseHeader header = 1; - replication_modepb.ReplicationStatus replication_status = 2; + ResponseHeader header = 1; + replication_modepb.ReplicationStatus replication_status = 2; } message IsBootstrappedRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message IsBootstrappedResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - bool bootstrapped = 2; + bool bootstrapped = 2; } message AllocIDRequest { - RequestHeader header = 1; + RequestHeader header = 1; + + uint32 count = 2; } message AllocIDResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 id = 2; + uint64 id = 2; + uint32 count = 3; } message IsSnapshotRecoveringRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message IsSnapshotRecoveringResponse { - ResponseHeader header = 1; - bool marked = 2; + ResponseHeader header = 1; + bool marked = 2; } message GetStoreRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 store_id = 2; + uint64 store_id = 2; } message GetStoreResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - metapb.Store store = 2; - StoreStats stats = 3; + metapb.Store store = 2; + StoreStats stats = 3; } message PutStoreRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Store store = 2; + metapb.Store store = 2; } message PutStoreResponse { - ResponseHeader header = 1; - replication_modepb.ReplicationStatus replication_status = 2; + ResponseHeader header = 1; + replication_modepb.ReplicationStatus replication_status = 2; } message GetAllStoresRequest { - RequestHeader header = 1; - // Do NOT return tombstone stores if set to true. - bool exclude_tombstone_stores = 2; + RequestHeader header = 1; + // Do NOT return tombstone stores if set to true. + bool exclude_tombstone_stores = 2; } message GetAllStoresResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - repeated metapb.Store stores = 2; + repeated metapb.Store stores = 2; } message GetRegionRequest { - RequestHeader header = 1; + RequestHeader header = 1; - bytes region_key = 2; - bool need_buckets = 3; + bytes region_key = 2; + bool need_buckets = 3; } message GetRegionResponse { - reserved 4; + reserved 4; - ResponseHeader header = 1; + ResponseHeader header = 1; - metapb.Region region = 2; - metapb.Peer leader = 3; - // Leader considers that these peers are down. - repeated PeerStats down_peers = 5; - // Pending peers are the peers that the leader can't consider as - // working followers. - repeated metapb.Peer pending_peers = 6; - // buckets isn't nil if GetRegion.* requests set need_buckets. - metapb.Buckets buckets = 7; + metapb.Region region = 2; + metapb.Peer leader = 3; + // Leader considers that these peers are down. + repeated PeerStats down_peers = 5; + // Pending peers are the peers that the leader can't consider as + // working followers. + repeated metapb.Peer pending_peers = 6; + // buckets isn't nil if GetRegion.* requests set need_buckets. + metapb.Buckets buckets = 7; } message GetRegionByIDRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 region_id = 2; - bool need_buckets = 3; + uint64 region_id = 2; + bool need_buckets = 3; } -// Use GetRegionResponse as the response of GetRegionByIDRequest. +message QueryRegionRequest { + RequestHeader header = 1; + + // Whether to include the buckets info within the response. + bool need_buckets = 2; + // The region IDs to query. + repeated uint64 ids = 3; + // The region keys to query. + repeated bytes keys = 4; + // The previous region keys to query. + repeated bytes prev_keys = 5; +} + +message QueryRegionResponse { + ResponseHeader header = 1; + // This array functions as a map corresponding to the region IDs, + // preserving the order of the input region keys, if they are present. + repeated uint64 key_id_map = 2; + // This array functions as a map corresponding to the previous region IDs, + // preserving the order of the input previous region keys, if they are present. + repeated uint64 prev_key_id_map = 3; + // RegionID -> RegionResponse + map regions_by_id = 4; +} + +message RegionResponse { + metapb.Region region = 1; + metapb.Peer leader = 2; + repeated PeerStats down_peers = 3; + repeated metapb.Peer pending_peers = 4; + metapb.Buckets buckets = 5; +} + +// Use GetRegionResponse as the response of GetRegionByIDRequest. +// Deprecated: use BatchScanRegionsRequest instead. message ScanRegionsRequest { - RequestHeader header = 1; + RequestHeader header = 1; - bytes start_key = 2; - int32 limit = 3; // no limit when limit <= 0. - bytes end_key = 4; // end_key is +inf when it is empty. + bytes start_key = 2; + int32 limit = 3; // no limit when limit <= 0. + bytes end_key = 4; // end_key is +inf when it is empty. } message Region { - metapb.Region region = 1; - metapb.Peer leader = 2; - // Leader considers that these peers are down. - repeated PeerStats down_peers = 3; - // Pending peers are the peers that the leader can't consider as - // working followers. - repeated metapb.Peer pending_peers = 4; + metapb.Region region = 1; + metapb.Peer leader = 2; + // Leader considers that these peers are down. + repeated PeerStats down_peers = 3; + // Pending peers are the peers that the leader can't consider as + // working followers. + repeated metapb.Peer pending_peers = 4; + // buckets isn't nil only when need_buckets is true. + metapb.Buckets buckets = 5; } message ScanRegionsResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; + + // Keep for backword compatibability. + repeated metapb.Region region_metas = 2; + repeated metapb.Peer leaders = 3; + + // Extended region info with down/pending peers. + repeated Region regions = 4; +} + +message KeyRange { + bytes start_key = 1; + bytes end_key = 2; // end_key is +inf when it is empty. +} + +message BatchScanRegionsRequest { + RequestHeader header = 1; + bool need_buckets = 2; + + repeated KeyRange ranges = 3; // the given ranges must be in order. + int32 limit = 4; // limit the total number of regions to scan. + // If contain_all_key_range is true, the output must contain all + // key ranges in the request. + // If the output does not contain all key ranges, the request is considered + // failed and returns an error(REGIONS_NOT_CONTAIN_ALL_KEY_RANGE). + bool contain_all_key_range = 5; +} - // Keep for backword compatibability. - repeated metapb.Region region_metas = 2; - repeated metapb.Peer leaders = 3; +message BatchScanRegionsResponse { + ResponseHeader header = 1; - // Extended region info with down/pending peers. - repeated Region regions = 4; + // the returned regions are flattened into a list, because the given ranges can located in the same range, we do not return duplicated regions then. + repeated Region regions = 2; } message GetClusterConfigRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetClusterConfigResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - metapb.Cluster cluster = 2; + metapb.Cluster cluster = 2; } message PutClusterConfigRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Cluster cluster = 2; + metapb.Cluster cluster = 2; } message PutClusterConfigResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message Member { - // name is the name of the PD member. - string name = 1; - // member_id is the unique id of the PD member. - uint64 member_id = 2; - repeated string peer_urls = 3; - repeated string client_urls = 4; - int32 leader_priority = 5; - string deploy_path = 6; - string binary_version = 7; - string git_hash = 8; - string dc_location = 9; + // name is the name of the PD member. + string name = 1; + // member_id is the unique id of the PD member. + uint64 member_id = 2; + repeated string peer_urls = 3; + repeated string client_urls = 4; + int32 leader_priority = 5; + string deploy_path = 6; + string binary_version = 7; + string git_hash = 8; + string dc_location = 9; } message GetMembersRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetMembersResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - repeated Member members = 2; - Member leader = 3; - Member etcd_leader = 4; - map tso_allocator_leaders = 5; + repeated Member members = 2; + Member leader = 3; + Member etcd_leader = 4; + map tso_allocator_leaders = 5; } message GetClusterInfoRequest { - ResponseHeader header = 1; + ResponseHeader header = 1; } enum ServiceMode { - UNKNOWN_SVC_MODE = 0; - PD_SVC_MODE = 1; - API_SVC_MODE = 2; + UNKNOWN_SVC_MODE = 0; + PD_SVC_MODE = 1; + API_SVC_MODE = 2; } message GetClusterInfoResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - repeated ServiceMode serviceModes = 2; - // If service mode is API_SVC_MODE, this field will be set to the - // registered tso service addresses. - repeated string tso_urls = 3; + repeated ServiceMode serviceModes = 2; + // If service mode is API_SVC_MODE, this field will be set to the + // registered tso service addresses. + repeated string tso_urls = 3; } message PeerStats { - metapb.Peer peer = 1; - uint64 down_seconds = 2; + metapb.Peer peer = 1; + uint64 down_seconds = 2; } message RegionHeartbeatRequest { - RequestHeader header = 1; - - metapb.Region region = 2; - // Leader Peer sending the heartbeat. - metapb.Peer leader = 3; - // Leader considers that these peers are down. - repeated PeerStats down_peers = 4; - // Pending peers are the peers that the leader can't consider as - // working followers. - repeated metapb.Peer pending_peers = 5; - // Bytes read/written during this period. - uint64 bytes_written = 6; - uint64 bytes_read = 7; - // Keys read/written during this period. - uint64 keys_written = 8; - uint64 keys_read = 9; - // Approximate region size. - uint64 approximate_size = 10; - reserved 11; - // Actually reported time interval - TimeInterval interval = 12; - // Approximate number of keys. - uint64 approximate_keys = 13; - // Term is the term of raft group. - uint64 term = 14; - replication_modepb.RegionReplicationStatus replication_status = 15; - // QueryStats reported write query stats, and there are read query stats in store heartbeat - QueryStats query_stats = 16; - // cpu_usage is the CPU time usage of the leader region since the last heartbeat, - // which is calculated by cpu_time_delta/heartbeat_reported_interval. - uint64 cpu_usage = 17; - // (Serverless) Approximate size of key-value pairs for billing. - // It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. - uint64 approximate_kv_size = 18; + RequestHeader header = 1; + + metapb.Region region = 2; + // Leader Peer sending the heartbeat. + metapb.Peer leader = 3; + // Leader considers that these peers are down. + repeated PeerStats down_peers = 4; + // Pending peers are the peers that the leader can't consider as + // working followers. + repeated metapb.Peer pending_peers = 5; + // Bytes read/written during this period. + uint64 bytes_written = 6; + uint64 bytes_read = 7; + // Keys read/written during this period. + uint64 keys_written = 8; + uint64 keys_read = 9; + // Approximate region size. + uint64 approximate_size = 10; + reserved 11; + // Actually reported time interval + TimeInterval interval = 12; + // Approximate number of keys. + uint64 approximate_keys = 13; + // Term is the term of raft group. + uint64 term = 14; + replication_modepb.RegionReplicationStatus replication_status = 15; + // QueryStats reported write query stats, and there are read query stats in store heartbeat + QueryStats query_stats = 16; + // cpu_usage is the total CPU time usage of the leader region since the last heartbeat, + // which is calculated by cpu_time_delta/heartbeat_reported_interval. + // Deprecated: use cpu_stats instead. + uint64 cpu_usage = 17 [deprecated = true]; + // cpu_stats reports CPU usage breakdown for the leader region by kind + // (e.g. unified read). + CPUStats cpu_stats = 21; + // Approximate size of row-based key-value pairs for billing. + // It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. + uint64 approximate_kv_size = 18; + // Approximate size of column-based key-value pairs for billing. + // It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. + uint64 approximate_columnar_kv_size = 19; + + // BucketMeta is the bucket version and keys of this region if TiKV enabled the bucket feature + metapb.BucketMeta bucket_meta = 20; } message ChangePeer { - metapb.Peer peer = 1; - eraftpb.ConfChangeType change_type = 2; + metapb.Peer peer = 1; + eraftpb.ConfChangeType change_type = 2; } message ChangePeerV2 { - // If changes is empty, it means that to exit joint state. - repeated ChangePeer changes = 1; + // If changes is empty, it means that to exit joint state. + repeated ChangePeer changes = 1; } message TransferLeader { - metapb.Peer peer = 1; - repeated metapb.Peer peers = 2; + metapb.Peer peer = 1; + repeated metapb.Peer peers = 2; } message Merge { - metapb.Region target = 1; + metapb.Region target = 1; } message SplitRegion { - CheckPolicy policy = 1; - repeated bytes keys = 2; + CheckPolicy policy = 1; + repeated bytes keys = 2; } message SwitchWitness { - uint64 peer_id = 1; - bool is_witness = 2; + uint64 peer_id = 1; + bool is_witness = 2; } message BatchSwitchWitness { - repeated SwitchWitness switch_witnesses = 1; + repeated SwitchWitness switch_witnesses = 1; } enum CheckPolicy { - SCAN = 0; - APPROXIMATE = 1; - USEKEY = 2; + SCAN = 0; + APPROXIMATE = 1; + USEKEY = 2; +} + +message ChangeSplit { + // auto_split_enabled configures whether the corresponding Region is allowed to be auto split by size or load. + bool auto_split_enabled = 1; } message RegionHeartbeatResponse { - ResponseHeader header = 1; - - // Notice, Pd only allows handling reported epoch >= current pd's. - // Leader peer reports region status with RegionHeartbeatRequest - // to pd regularly, pd will determine whether this region - // should do ChangePeer or not. - // E,g, max peer number is 3, region A, first only peer 1 in A. - // 1. Pd region state -> Peers (1), ConfVer (1). - // 2. Leader peer 1 reports region state to pd, pd finds the - // peer number is < 3, so first changes its current region - // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2. - // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2), - // pd updates its state -> Peers (1, 2), ConfVer (2). - // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange - // finished, pd stills responses ChangePeer Adding 2, of course, we must - // guarantee the second ChangePeer can't be applied in TiKV. - ChangePeer change_peer = 2; - // Pd can return transfer_leader to let TiKV does leader transfer itself. - TransferLeader transfer_leader = 3; - // ID of the region - uint64 region_id = 4; - metapb.RegionEpoch region_epoch = 5; - // Leader of the region at the moment of the corresponding request was made. - metapb.Peer target_peer = 6; - Merge merge = 7; - // PD sends split_region to let TiKV split a region into two regions. - SplitRegion split_region = 8; - // Multiple change peer operations atomically. - // Note: PD can use both ChangePeer and ChangePeerV2 at the same time - // (not in the same RegionHeartbeatResponse). - // Now, PD use ChangePeerV2 in following scenarios: - // 1. replacing peers - // 2. demoting voter directly - ChangePeerV2 change_peer_v2 = 9; - BatchSwitchWitness switch_witnesses = 10; + ResponseHeader header = 1; + + // Notice, Pd only allows handling reported epoch >= current pd's. + // Leader peer reports region status with RegionHeartbeatRequest + // to pd regularly, pd will determine whether this region + // should do ChangePeer or not. + // E,g, max peer number is 3, region A, first only peer 1 in A. + // 1. Pd region state -> Peers (1), ConfVer (1). + // 2. Leader peer 1 reports region state to pd, pd finds the + // peer number is < 3, so first changes its current region + // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2. + // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2), + // pd updates its state -> Peers (1, 2), ConfVer (2). + // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange + // finished, pd stills responses ChangePeer Adding 2, of course, we must + // guarantee the second ChangePeer can't be applied in TiKV. + ChangePeer change_peer = 2; + // Pd can return transfer_leader to let TiKV does leader transfer itself. + TransferLeader transfer_leader = 3; + // ID of the region + uint64 region_id = 4; + metapb.RegionEpoch region_epoch = 5; + // Leader of the region at the moment of the corresponding request was made. + metapb.Peer target_peer = 6; + Merge merge = 7; + // PD sends split_region to let TiKV split a region into two regions. + SplitRegion split_region = 8; + // Multiple change peer operations atomically. + // Note: PD can use both ChangePeer and ChangePeerV2 at the same time + // (not in the same RegionHeartbeatResponse). + // Now, PD use ChangePeerV2 in following scenarios: + // 1. replacing peers + // 2. demoting voter directly + ChangePeerV2 change_peer_v2 = 9; + BatchSwitchWitness switch_witnesses = 10; + ChangeSplit change_split = 11; } message AskSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Region region = 2; + metapb.Region region = 2; } message AskSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - // We split the region into two, first uses the origin - // parent region id, and the second uses the new_region_id. - // We must guarantee that the new_region_id is global unique. - uint64 new_region_id = 2; - // The peer ids for the new split region. - repeated uint64 new_peer_ids = 3; + // We split the region into two, first uses the origin + // parent region id, and the second uses the new_region_id. + // We must guarantee that the new_region_id is global unique. + uint64 new_region_id = 2; + // The peer ids for the new split region. + repeated uint64 new_peer_ids = 3; } message ReportSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Region left = 2; - metapb.Region right = 3; + metapb.Region left = 2; + metapb.Region right = 3; } message ReportSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; +} + +enum SplitReason { + ADMIN = 0; + SIZE = 1; + LOAD = 2; } message AskBatchSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Region region = 2; - uint32 split_count = 3; + metapb.Region region = 2; + uint32 split_count = 3; + SplitReason reason = 4; } message SplitID { - uint64 new_region_id = 1; - repeated uint64 new_peer_ids = 2; + uint64 new_region_id = 1; + repeated uint64 new_peer_ids = 2; } message AskBatchSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - repeated SplitID ids = 2; + repeated SplitID ids = 2; } message ReportBatchSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - repeated metapb.Region regions = 2; + repeated metapb.Region regions = 2; } message ReportBatchSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message TimeInterval { - // The unix timestamp in seconds of the start of this period. - uint64 start_timestamp = 1; - // The unix timestamp in seconds of the end of this period. - uint64 end_timestamp = 2; + // The unix timestamp in seconds of the start of this period. + uint64 start_timestamp = 1; + // The unix timestamp in seconds of the end of this period. + uint64 end_timestamp = 2; } message RecordPair { - string key = 1; - uint64 value = 2; + string key = 1; + uint64 value = 2; } message PeerStat { - uint64 region_id = 1; - uint64 read_keys = 2; - uint64 read_bytes = 3; - QueryStats query_stats = 4; - uint64 written_keys = 5; - uint64 written_bytes = 6; + uint64 region_id = 1; + uint64 read_keys = 2; + uint64 read_bytes = 3; + QueryStats query_stats = 4; + uint64 written_keys = 5; + uint64 written_bytes = 6; + // cpu_stats is the CPU usage of the region's unified read pool since the last heartbeat, + // which is calculated by cpu_time_delta/heartbeat_reported_interval. + CPUStats cpu_stats = 7; } message StoreStats { - uint64 store_id = 1; - // Capacity for the store. - uint64 capacity = 2; - // Available size for the store. - uint64 available = 3; - // Total region count in this store. - uint32 region_count = 4; - // Current sending snapshot count. - uint32 sending_snap_count = 5; - // Current receiving snapshot count. - uint32 receiving_snap_count = 6; - // When the store is started (unix timestamp in seconds). - uint32 start_time = 7; - // How many region is applying snapshot. - uint32 applying_snap_count = 8; - // If the store is busy - bool is_busy = 9; - // Actually used space by db - uint64 used_size = 10; - // Bytes written for the store during this period. - uint64 bytes_written = 11; - // Keys written for the store during this period. - uint64 keys_written = 12; - // Bytes read for the store during this period. - uint64 bytes_read = 13; - // Keys read for the store during this period. - uint64 keys_read = 14; - // Actually reported time interval - TimeInterval interval = 15; - // Threads' CPU usages in the store - repeated RecordPair cpu_usages = 16; - // Threads' read disk I/O rates in the store - repeated RecordPair read_io_rates = 17; - // Threads' write disk I/O rates in the store - repeated RecordPair write_io_rates = 18; - // Operations' latencies in the store - repeated RecordPair op_latencies = 19; - // Hot peer stat in the store - repeated PeerStat peer_stats = 20; - // Store query stats - QueryStats query_stats = 21; - // Score that represents the speed of the store, ranges in [1, 100], lower is better. - uint64 slow_score = 22; - // Damaged regions on the store that need to be removed by PD. - repeated uint64 damaged_regions_id = 23; - // If the apply worker is busy, namely high apply wait duration - bool is_apply_busy = 24; - // Snapshot stats in the store - repeated SnapshotStat snapshot_stats = 25; - SlowTrend slow_trend = 26; - // If the grpc server is paused. - bool is_grpc_paused = 27; - // Total memory of the store in bytes. - uint64 total_memory = 28; - // Used memory of the store in bytes. - uint64 used_memory = 29; -} - -message SlowTrend{ - double cause_value = 1; - double cause_rate = 2; - double result_value = 3; - double result_rate = 4; -} - -message SnapshotStat{ - uint64 region_id = 1; - // Generate snapshot duration - uint64 generate_duration_sec = 2; - // Send snapshot duration - uint64 send_duration_sec = 3; - // |-- waiting --|-- generate --｜-- send --| - // |-----------total duration---------------| - // Total duration include waiting and executing duration - uint64 total_duration_sec = 4; - // Size is the transport data size - uint64 transport_size = 5; + uint64 store_id = 1; + // Capacity for the store. + uint64 capacity = 2; + // Available size for the store. + uint64 available = 3; + // Total region count in this store. + uint32 region_count = 4; + // Current sending snapshot count. + uint32 sending_snap_count = 5; + // Current receiving snapshot count. + uint32 receiving_snap_count = 6; + // When the store is started (unix timestamp in seconds). + uint32 start_time = 7; + // How many region is applying snapshot. + uint32 applying_snap_count = 8; + // If the store is busy + bool is_busy = 9; + // Actually used space by db + uint64 used_size = 10; + // Bytes written for the store during this period. + uint64 bytes_written = 11; + // Keys written for the store during this period. + uint64 keys_written = 12; + // Bytes read for the store during this period. + uint64 bytes_read = 13; + // Keys read for the store during this period. + uint64 keys_read = 14; + // Actually reported time interval + TimeInterval interval = 15; + // Threads' CPU usages in the store + repeated RecordPair cpu_usages = 16; + // Threads' read disk I/O rates in the store + repeated RecordPair read_io_rates = 17; + // Threads' write disk I/O rates in the store + repeated RecordPair write_io_rates = 18; + // Operations' latencies in the store + repeated RecordPair op_latencies = 19; + // Hot peer stat in the store + repeated PeerStat peer_stats = 20; + // Store query stats + QueryStats query_stats = 21; + // Score that represents the speed of the store, ranges in [1, 100], lower is better. + uint64 slow_score = 22; + // Damaged regions on the store that need to be removed by PD. + repeated uint64 damaged_regions_id = 23; + // If the apply worker is busy, namely high apply wait duration + bool is_apply_busy = 24; + // Snapshot stats in the store + repeated SnapshotStat snapshot_stats = 25; + SlowTrend slow_trend = 26; + // If the grpc server is paused. + bool is_grpc_paused = 27; + // Total memory of the store in bytes. + uint64 total_memory = 28; + // Used memory of the store in bytes. + uint64 used_memory = 29; + // Network_slow_scores indicate the network status between TiKV nodes, ranging from 1 to 100 (lower is better). + // StoreID -> score + map network_slow_scores = 30; + // The statistics about DFS uploads. + repeated DfsStatItem dfs = 31; + // True if the store is undergoing graceful shutdown. + bool is_stopping = 32; + + // Reserved for a downstream fork + reserved 1000 to 1199; +} + +message DfsStatScope { + // When true, the statistic is not tied to any keyspace. + bool is_global = 1; + // The keyspace of this statistic. Ignore when is_global is true. + uint32 keyspace_id = 2; + // The component that provides the statistic. + string component = 3; +} + +message DfsStatItem { + DfsStatScope scope = 1; + // Number of bytes written to DFS. + uint64 written_bytes = 2; + // Number of write requests sent to DFS. + uint64 write_requests = 3; +} + +message SlowTrend { + double cause_value = 1; + double cause_rate = 2; + double result_value = 3; + double result_rate = 4; +} + +message SnapshotStat { + uint64 region_id = 1; + // Generate snapshot duration + uint64 generate_duration_sec = 2; + // Send snapshot duration + uint64 send_duration_sec = 3; + // |-- waiting --|-- generate --｜-- send --| + // |-----------total duration---------------| + // Total duration include waiting and executing duration + uint64 total_duration_sec = 4; + // Size is the transport data size + uint64 transport_size = 5; } message PeerReport { - raft_serverpb.RaftLocalState raft_state = 1; - raft_serverpb.RegionLocalState region_state = 2; - bool is_force_leader = 3; - // The peer has proposed but uncommitted commit merge. - bool has_commit_merge = 4; + raft_serverpb.RaftLocalState raft_state = 1; + raft_serverpb.RegionLocalState region_state = 2; + bool is_force_leader = 3; + // The peer has proposed but uncommitted commit merge. + bool has_commit_merge = 4; + // raft applied index + uint64 applied_index = 5; } message StoreReport { - repeated PeerReport peer_reports = 1; - uint64 step = 2; + repeated PeerReport peer_reports = 1; + uint64 step = 2; } message StoreHeartbeatRequest { - RequestHeader header = 1; + RequestHeader header = 1; - StoreStats stats = 2; - // Detailed store report that is only filled up on PD's demand for online unsafe recovery. - StoreReport store_report = 3; - replication_modepb.StoreDRAutoSyncStatus dr_autosync_status = 4; + StoreStats stats = 2; + // Detailed store report that is only filled up on PD's demand for online unsafe recovery. + StoreReport store_report = 3; + replication_modepb.StoreDRAutoSyncStatus dr_autosync_status = 4; } message DemoteFailedVoters { - uint64 region_id = 1; - repeated metapb.Peer failed_voters = 2; + uint64 region_id = 1; + repeated metapb.Peer failed_voters = 2; } message ForceLeader { - // The store ids of the failed stores, TiKV uses it to decide if a peer is alive. - repeated uint64 failed_stores = 1; - // The region ids of the peer which is to be force leader. - repeated uint64 enter_force_leaders = 2; + // The store ids of the failed stores, TiKV uses it to decide if a peer is alive. + repeated uint64 failed_stores = 1; + // The region ids of the peer which is to be force leader. + repeated uint64 enter_force_leaders = 2; } message RecoveryPlan { - // Create empty regions to fill the key range hole. - repeated metapb.Region creates = 1; - // Update the meta of the regions, including peer lists, epoch and key range. - repeated metapb.Region updates = 2 [deprecated=true]; - // Tombstone the peers on the store locally. - repeated uint64 tombstones = 3; - // Issue conf change that demote voters on failed stores to learners on the regions. - repeated DemoteFailedVoters demotes = 4; - // Make the peers to be force leaders. - ForceLeader force_leader = 5; - // Step is an increasing number to note the round of recovery, - // It should be filled in the corresponding store report. - uint64 step = 6; + // Create empty regions to fill the key range hole. + repeated metapb.Region creates = 1; + // Update the meta of the regions, including peer lists, epoch and key range. + repeated metapb.Region updates = 2 [deprecated = true]; + // Tombstone the peers on the store locally. + repeated uint64 tombstones = 3; + // Issue conf change that demote voters on failed stores to learners on the regions. + repeated DemoteFailedVoters demotes = 4; + // Make the peers to be force leaders. + ForceLeader force_leader = 5; + // Step is an increasing number to note the round of recovery, + // It should be filled in the corresponding store report. + uint64 step = 6; } message AwakenRegions { - // Awake all regions if abnormal_stores is empty. - repeated uint64 abnormal_stores = 1; + // Awake all regions if abnormal_stores is empty. + repeated uint64 abnormal_stores = 1; } enum ControlGrpcEvent { - // Pause TiKV grpc server. - PAUSE = 0; - // Resume TiKV grpc server. - RESUME = 1; + // Pause TiKV grpc server. + PAUSE = 0; + // Resume TiKV grpc server. + RESUME = 1; } message ControlGrpc { - ControlGrpcEvent ctrl_event = 1; + ControlGrpcEvent ctrl_event = 1; } message StoreHeartbeatResponse { - ResponseHeader header = 1; - replication_modepb.ReplicationStatus replication_status = 2; - string cluster_version = 3; - - // Used by online unsafe recovery to request store report. - // Now it's substituted by reusing recovery_plan field. PD will send a empty - // recovery plan instead to request store report. - bool require_detailed_report = 4 [deprecated=true]; - // Operations of recovery. After the plan is executed, TiKV should attach the - // store report in store heartbeat. - RecoveryPlan recovery_plan = 5; - // Pd can return awaken_regions to let TiKV awaken hibernated regions itself. - AwakenRegions awaken_regions = 6; - // Pd can return operations to let TiKV forcely PAUSE | RESUME grpc server. - ControlGrpc control_grpc = 7; + ResponseHeader header = 1; + replication_modepb.ReplicationStatus replication_status = 2; + string cluster_version = 3; + + // Used by online unsafe recovery to request store report. + // Now it's substituted by reusing recovery_plan field. PD will send a empty + // recovery plan instead to request store report. + bool require_detailed_report = 4 [deprecated = true]; + // Operations of recovery. After the plan is executed, TiKV should attach the + // store report in store heartbeat. + RecoveryPlan recovery_plan = 5; + // Pd can return awaken_regions to let TiKV awaken hibernated regions itself. + AwakenRegions awaken_regions = 6; + // Pd can return operations to let TiKV forcely PAUSE | RESUME grpc server. + ControlGrpc control_grpc = 7; + // NodeState is going to mark the state of the store. + metapb.NodeState state = 8; } message ScatterRegionRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 region_id = 2 [deprecated=true]; + uint64 region_id = 2 [deprecated = true]; - // PD will use these region information if it can't find the region. - // For example, the region is just split and hasn't report to PD yet. - metapb.Region region = 3; - metapb.Peer leader = 4; + // PD will use these region information if it can't find the region. + // For example, the region is just split and hasn't report to PD yet. + metapb.Region region = 3; + metapb.Peer leader = 4; - // If group is defined, the regions with the same group would be scattered as a whole group. - // If not defined, the regions would be scattered in a cluster level. - string group = 5; + // If group is defined, the regions with the same group would be scattered as a whole group. + // If not defined, the regions would be scattered in a cluster level. + string group = 5; - // If regions_id is defined, the region_id would be ignored. - repeated uint64 regions_id = 6; - uint64 retry_limit = 7; - bool skip_store_limit = 8; + // If regions_id is defined, the region_id would be ignored. + repeated uint64 regions_id = 6; + uint64 retry_limit = 7; + bool skip_store_limit = 8; } message ScatterRegionResponse { - ResponseHeader header = 1; - uint64 finished_percentage = 2; + ResponseHeader header = 1; + uint64 finished_percentage = 2; + repeated uint64 failed_regions_id = 3; } message GetGCSafePointRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetGCSafePointResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 safe_point = 2; + uint64 safe_point = 2; } message UpdateGCSafePointRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 safe_point = 2; + uint64 safe_point = 2; } message UpdateGCSafePointResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 new_safe_point = 2; + uint64 new_safe_point = 2; } message UpdateServiceGCSafePointRequest { - RequestHeader header = 1; + RequestHeader header = 1; - bytes service_id = 2; - int64 TTL = 3; - uint64 safe_point = 4; + bytes service_id = 2; + int64 TTL = 3; + uint64 safe_point = 4; } message UpdateServiceGCSafePointResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - bytes service_id = 2; - int64 TTL = 3; - uint64 min_safe_point = 4; + bytes service_id = 2; + int64 TTL = 3; + uint64 min_safe_point = 4; } message GetGCSafePointV2Request { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 keyspace_id = 2; + uint32 keyspace_id = 2; } message GetGCSafePointV2Response { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 safe_point = 2; + uint64 safe_point = 2; } message WatchGCSafePointV2Request { - RequestHeader header = 1; - int64 revision = 2; - + RequestHeader header = 1; + int64 revision = 2; } // SafePointEvent is for the rpc WatchGCSafePointV2. message SafePointEvent { - uint32 keyspace_id = 1; - uint64 safe_point = 2; - EventType type = 3; + uint32 keyspace_id = 1; + uint64 safe_point = 2; + EventType type = 3; } message WatchGCSafePointV2Response { - ResponseHeader header = 1; - repeated SafePointEvent events = 2; - int64 revision = 3; + ResponseHeader header = 1; + repeated SafePointEvent events = 2; + int64 revision = 3; } message UpdateGCSafePointV2Request { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 keyspace_id = 2; - uint64 safe_point = 3; + uint32 keyspace_id = 2; + uint64 safe_point = 3; } message UpdateGCSafePointV2Response { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 new_safe_point = 2; + uint64 new_safe_point = 2; } message UpdateServiceSafePointV2Request { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 keyspace_id = 2; - bytes service_id = 3; - uint64 safe_point = 4; - // Safe point will be set to expire on (PD Server time + TTL), - // pass in a ttl < 0 to remove target safe point; - // pass in MAX_INT64 to set a safe point that never expire. - // This should be set by component that may crash unexpectedly so that it doesn't block - // cluster garbage collection. - int64 ttl = 5; + uint32 keyspace_id = 2; + bytes service_id = 3; + uint64 safe_point = 4; + // Safe point will be set to expire on (PD Server time + TTL), + // pass in a ttl < 0 to remove target safe point; + // pass in MAX_INT64 to set a safe point that never expire. + // This should be set by component that may crash unexpectedly so that it doesn't block + // cluster garbage collection. + int64 ttl = 5; } message UpdateServiceSafePointV2Response { - ResponseHeader header = 1; + ResponseHeader header = 1; - bytes service_id = 2; - int64 ttl = 3; - uint64 min_safe_point = 4; + bytes service_id = 2; + int64 ttl = 3; + uint64 min_safe_point = 4; } message GetAllGCSafePointV2Request { - RequestHeader header = 1; + RequestHeader header = 1; } message GCSafePointV2 { - uint32 keyspace_id = 1; - uint64 gc_safe_point = 2; + uint32 keyspace_id = 1; + uint64 gc_safe_point = 2; } message GetAllGCSafePointV2Response { - ResponseHeader header = 1; - repeated GCSafePointV2 gc_safe_points = 2; - int64 revision = 3; + ResponseHeader header = 1; + repeated GCSafePointV2 gc_safe_points = 2; + int64 revision = 3; +} + +// A wrapper over keyspace_id. +// When a field is not specified in proto3, its value will be regarded as 0; however, keyspace_id = 0 is regarded as a valid keyspace (which +// is the "DEFAULT" keyspace). To distinguish unspecified keyspace (NullKeyspace, 0xffffffff) and the default keyspace in some APIs as well +// as preventing potential misuse, we wrap the keyspace_id into a message type which is nullable. +message KeyspaceScope { + uint32 keyspace_id = 1; +} + +message AdvanceGCSafePointRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + uint64 target = 3; +} + +message AdvanceGCSafePointResponse { + ResponseHeader header = 1; + + uint64 old_gc_safe_point = 2; + uint64 new_gc_safe_point = 3; +} + +message AdvanceTxnSafePointRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + uint64 target = 3; +} + +message AdvanceTxnSafePointResponse { + ResponseHeader header = 1; + + uint64 old_txn_safe_point = 2; + uint64 new_txn_safe_point = 3; + string blocker_description = 4; +} + +message SetGCBarrierRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + + string barrier_id = 3; + uint64 barrier_ts = 4; + int64 ttl_seconds = 5; +} + +message GCBarrierInfo { + string barrier_id = 1; + uint64 barrier_ts = 2; + int64 ttl_seconds = 3; +} + +message SetGCBarrierResponse { + ResponseHeader header = 1; + + GCBarrierInfo new_barrier_info = 2; +} + +message DeleteGCBarrierRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + + string barrier_id = 3; +} + +message DeleteGCBarrierResponse { + ResponseHeader header = 1; + + GCBarrierInfo deleted_barrier_info = 2; +} + +message SetGlobalGCBarrierRequest { + RequestHeader header = 1; + string barrier_id = 2; + uint64 barrier_ts = 3; + int64 ttl_seconds = 4; +} + +message SetGlobalGCBarrierResponse { + ResponseHeader header = 1; + GlobalGCBarrierInfo new_barrier_info = 2; +} + +message DeleteGlobalGCBarrierRequest { + RequestHeader header = 1; + string barrier_id = 2; +} + +message DeleteGlobalGCBarrierResponse { + ResponseHeader header = 1; + GlobalGCBarrierInfo deleted_barrier_info = 2; +} + +message GlobalGCBarrierInfo { + string barrier_id = 1; + uint64 barrier_ts = 2; + int64 ttl_seconds = 3; +} + +message GetGCStateRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + bool exclude_gc_barriers = 3; +} + +message GCState { + KeyspaceScope keyspace_scope = 1; + + bool is_keyspace_level_gc = 2; + uint64 txn_safe_point = 3; + uint64 gc_safe_point = 4; + repeated GCBarrierInfo gc_barriers = 5; +} + +message GetGCStateResponse { + ResponseHeader header = 1; + + GCState gc_state = 2; +} + +message GetAllKeyspacesGCStatesRequest { + RequestHeader header = 1; + + bool exclude_gc_barriers = 2; + bool exclude_global_gc_barriers = 3; +} + +message GetAllKeyspacesGCStatesResponse { + ResponseHeader header = 1; + + repeated GCState gc_states = 2; + repeated GlobalGCBarrierInfo global_gc_barriers = 3; } message RegionStat { - // Bytes read/written during this period. - uint64 bytes_written = 1; - uint64 bytes_read = 2; - // Keys read/written during this period. - uint64 keys_written = 3; - uint64 keys_read = 4; + // Bytes read/written during this period. + uint64 bytes_written = 1; + uint64 bytes_read = 2; + // Keys read/written during this period. + uint64 keys_written = 3; + uint64 keys_read = 4; } -message SyncRegionRequest{ - RequestHeader header = 1; - Member member = 2; - // the follower PD will use the start index to locate historical changes - // that require synchronization. - uint64 start_index = 3; +message SyncRegionRequest { + RequestHeader header = 1; + Member member = 2; + // the follower PD will use the start index to locate historical changes + // that require synchronization. + uint64 start_index = 3; } message PeersStats { - repeated PeerStats peers = 1; + repeated PeerStats peers = 1; } message Peers { - repeated metapb.Peer peers = 1; + repeated metapb.Peer peers = 1; } -message SyncRegionResponse{ - ResponseHeader header = 1; - // the leader PD will send the repsonds include - // changed regions records and the index of the first record. - repeated metapb.Region regions = 2; - uint64 start_index = 3; - repeated RegionStat region_stats = 4; - repeated metapb.Peer region_leaders = 5; - // the buckets informations without stats. - repeated metapb.Buckets buckets = 6; - repeated PeersStats down_peers = 16; - repeated Peers pending_peers = 17; +message SyncRegionResponse { + ResponseHeader header = 1; + // the leader PD will send the repsonds include + // changed regions records and the index of the first record. + repeated metapb.Region regions = 2; + uint64 start_index = 3; + repeated RegionStat region_stats = 4; + repeated metapb.Peer region_leaders = 5; + // the buckets informations without stats. + repeated metapb.Buckets buckets = 6; + repeated PeersStats down_peers = 16; + repeated Peers pending_peers = 17; } message GetOperatorRequest { - RequestHeader header = 1; - uint64 region_id = 2; + RequestHeader header = 1; + uint64 region_id = 2; } enum OperatorStatus { - SUCCESS = 0; - TIMEOUT = 1; - CANCEL = 2; - REPLACE = 3; - RUNNING = 4; + SUCCESS = 0; + TIMEOUT = 1; + CANCEL = 2; + REPLACE = 3; + RUNNING = 4; } message GetOperatorResponse { - ResponseHeader header = 1; - uint64 region_id = 2; - bytes desc = 3; - OperatorStatus status = 4; - bytes kind = 5; + ResponseHeader header = 1; + uint64 region_id = 2; + bytes desc = 3; + OperatorStatus status = 4; + bytes kind = 5; } message SyncMaxTSRequest { - RequestHeader header = 1; - Timestamp max_ts = 2; - // If skip_check is true, the sync will try to write the max_ts without checking whether it's bigger. - bool skip_check = 3; + RequestHeader header = 1; + Timestamp max_ts = 2; + // If skip_check is true, the sync will try to write the max_ts without checking whether it's bigger. + bool skip_check = 3; } message SyncMaxTSResponse { - ResponseHeader header = 1; - Timestamp max_local_ts = 2; - repeated string synced_dcs = 3; + ResponseHeader header = 1; + Timestamp max_local_ts = 2; + repeated string synced_dcs = 3; } message SplitRegionsRequest { - RequestHeader header = 1; - repeated bytes split_keys = 2; - uint64 retry_limit = 3; + RequestHeader header = 1; + repeated bytes split_keys = 2; + uint64 retry_limit = 3; } message SplitRegionsResponse { - ResponseHeader header = 1; - uint64 finished_percentage = 2; - repeated uint64 regions_id = 3; + ResponseHeader header = 1; + uint64 finished_percentage = 2; + repeated uint64 regions_id = 3; } message SplitAndScatterRegionsRequest { - RequestHeader header = 1; - repeated bytes split_keys = 2; - string group = 3; - uint64 retry_limit = 4; + RequestHeader header = 1; + repeated bytes split_keys = 2; + string group = 3; + uint64 retry_limit = 4; } message SplitAndScatterRegionsResponse { - ResponseHeader header = 1; - uint64 split_finished_percentage = 2; - uint64 scatter_finished_percentage = 3; - repeated uint64 regions_id = 4; + ResponseHeader header = 1; + uint64 split_finished_percentage = 2; + uint64 scatter_finished_percentage = 3; + repeated uint64 regions_id = 4; } message GetDCLocationInfoRequest { - RequestHeader header = 1; - string dc_location = 2; + RequestHeader header = 1; + string dc_location = 2; } message GetDCLocationInfoResponse { - ResponseHeader header = 1; - // suffix sign - int32 suffix = 2; - // max_ts will be included into this response if PD leader think the receiver needs, - // which it's set when the number of the max suffix bits changes. - Timestamp max_ts = 3; + ResponseHeader header = 1; + // suffix sign + int32 suffix = 2; + // max_ts will be included into this response if PD leader think the receiver needs, + // which it's set when the number of the max suffix bits changes. + Timestamp max_ts = 3; } message QueryStats { - uint64 GC = 1; - uint64 Get = 2; - uint64 Scan = 3; - uint64 Coprocessor = 4; - uint64 Delete = 5; - uint64 DeleteRange = 6; - uint64 Put = 7; - uint64 Prewrite = 8; - uint64 AcquirePessimisticLock = 9; - uint64 Commit = 10; - uint64 Rollback = 11; + uint64 GC = 1; + uint64 Get = 2; + uint64 Scan = 3; + uint64 Coprocessor = 4; + uint64 Delete = 5; + uint64 DeleteRange = 6; + uint64 Put = 7; + uint64 Prewrite = 8; + uint64 AcquirePessimisticLock = 9; + uint64 Commit = 10; + uint64 Rollback = 11; +} + +// CPU usage breakdown by kind. New kinds may be added in the future. +message CPUStats { + // UnifiedRead is the CPU usage of the unified read pool. + uint64 UnifiedRead = 1; + // Scheduler is the CPU usage of the scheduler pool, it contains `sched-pool`, `sched-high`, and `sched-pri`. + uint64 Scheduler = 2; } enum QueryKind { - Others = 0; - GC = 1; - Get = 2; - Scan = 3; - Coprocessor = 4; - Delete = 5; - DeleteRange = 6; - Put = 7; - Prewrite = 8; - AcquirePessimisticLock = 9; - Commit = 10; - Rollback = 11; + Others = 0; + GC = 1; + Get = 2; + Scan = 3; + Coprocessor = 4; + Delete = 5; + DeleteRange = 6; + Put = 7; + Prewrite = 8; + AcquirePessimisticLock = 9; + Commit = 10; + Rollback = 11; } message ReportBucketsRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.RegionEpoch region_epoch = 2; - metapb.Buckets buckets = 3; + metapb.RegionEpoch region_epoch = 2; + metapb.Buckets buckets = 3; } message ReportBucketsResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message ReportMinResolvedTsRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 store_id = 2; + uint64 store_id = 2; - uint64 min_resolved_ts = 3; + uint64 min_resolved_ts = 3; } message ReportMinResolvedTsResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message SetExternalTimestampRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 timestamp = 2; + uint64 timestamp = 2; } message SetExternalTimestampResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message GetExternalTimestampRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetExternalTimestampResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 timestamp = 2; + uint64 timestamp = 2; } message GetMinTSRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetMinTSResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - Timestamp timestamp = 2; + Timestamp timestamp = 2; } diff --git a/proto/raft_cmdpb.proto b/proto/raft_cmdpb.proto index 155210b6..90121fb5 100644 --- a/proto/raft_cmdpb.proto +++ b/proto/raft_cmdpb.proto @@ -1,47 +1,53 @@ syntax = "proto3"; package raft_cmdpb; -import "metapb.proto"; -import "errorpb.proto"; import "eraftpb.proto"; -import "kvrpcpb.proto"; +import "errorpb.proto"; +import "gogoproto/gogo.proto"; import "import_sstpb.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; import "raft_serverpb.proto"; import "rustproto.proto"; -option (rustproto.lite_runtime_all) = true; - option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; message GetRequest { - string cf = 1; - bytes key = 2; + string cf = 1; + bytes key = 2; } message GetResponse { - bytes value = 1; + bytes value = 1; } message PutRequest { - string cf = 1; - bytes key = 2; - bytes value = 3; + string cf = 1; + bytes key = 2; + bytes value = 3; } message PutResponse {} message DeleteRequest { - string cf = 1; - bytes key = 2; + string cf = 1; + bytes key = 2; } message DeleteResponse {} message DeleteRangeRequest { - string cf = 1; - bytes start_key = 2; - bytes end_key = 3; - bool notify_only = 4; + string cf = 1; + bytes start_key = 2; + bytes end_key = 3; + bool notify_only = 4; } message DeleteRangeResponse {} @@ -49,169 +55,169 @@ message DeleteRangeResponse {} message SnapRequest {} message SnapResponse { - metapb.Region region = 1; + metapb.Region region = 1; } message PrewriteRequest { - bytes key = 1; - bytes value = 2; - bytes lock = 3; + bytes key = 1; + bytes value = 2; + bytes lock = 3; } message PrewriteResponse {} message IngestSSTRequest { - import_sstpb.SSTMeta sst = 1; + import_sstpb.SSTMeta sst = 1; } message IngestSSTResponse {} message ReadIndexRequest { - // In replica read, leader uses start_ts and key_ranges to check memory locks. - uint64 start_ts = 1; - repeated kvrpcpb.KeyRange key_ranges = 2; + // In replica read, leader uses start_ts and key_ranges to check memory locks. + uint64 start_ts = 1; + repeated kvrpcpb.KeyRange key_ranges = 2; } -message ReadIndexResponse{ - uint64 read_index = 1; - // The memory lock blocking this read at the leader - kvrpcpb.LockInfo locked = 2; +message ReadIndexResponse { + uint64 read_index = 1; + // The memory lock blocking this read at the leader + kvrpcpb.LockInfo locked = 2; } enum CmdType { - Invalid = 0; - Get = 1; - Put = 3; - Delete = 4; - Snap = 5; - Prewrite = 6; - DeleteRange = 7; - IngestSST = 8; - ReadIndex = 9; + Invalid = 0; + Get = 1; + Put = 3; + Delete = 4; + Snap = 5; + Prewrite = 6; + DeleteRange = 7; + IngestSST = 8; + ReadIndex = 9; } message Request { - CmdType cmd_type = 1; - GetRequest get = 2; - PutRequest put = 4; - DeleteRequest delete = 5; - SnapRequest snap = 6; - PrewriteRequest prewrite = 7; - DeleteRangeRequest delete_range = 8; - IngestSSTRequest ingest_sst = 9; - ReadIndexRequest read_index = 10; + CmdType cmd_type = 1; + GetRequest get = 2; + PutRequest put = 4; + DeleteRequest delete = 5; + SnapRequest snap = 6; + PrewriteRequest prewrite = 7; + DeleteRangeRequest delete_range = 8; + IngestSSTRequest ingest_sst = 9; + ReadIndexRequest read_index = 10; } message Response { - CmdType cmd_type = 1; - GetResponse get = 2; - PutResponse put = 4; - DeleteResponse delete = 5; - SnapResponse snap = 6; - PrewriteResponse prewrite = 7; - DeleteRangeResponse delte_range = 8; - IngestSSTResponse ingest_sst = 9; - ReadIndexResponse read_index = 10; + CmdType cmd_type = 1; + GetResponse get = 2; + PutResponse put = 4; + DeleteResponse delete = 5; + SnapResponse snap = 6; + PrewriteResponse prewrite = 7; + DeleteRangeResponse delte_range = 8; + IngestSSTResponse ingest_sst = 9; + ReadIndexResponse read_index = 10; } message ChangePeerRequest { - // This can be only called in internal RaftStore now. - eraftpb.ConfChangeType change_type = 1; - metapb.Peer peer = 2; + // This can be only called in internal RaftStore now. + eraftpb.ConfChangeType change_type = 1; + metapb.Peer peer = 2; } message ChangePeerResponse { - metapb.Region region = 1; + metapb.Region region = 1; } message ChangePeerV2Request { - repeated ChangePeerRequest changes = 1; + repeated ChangePeerRequest changes = 1; } message ChangePeerV2Response { - metapb.Region region = 1; + metapb.Region region = 1; } message SplitRequest { - // This can be only called in internal RaftStore now. - // The split_key must be in the been splitting region. - bytes split_key = 1; - // We split the region into two, first uses the origin - // parent region id, and the second uses the new_region_id. - // We must guarantee that the new_region_id is global unique. - uint64 new_region_id = 2; - // The peer ids for the new split region. - repeated uint64 new_peer_ids = 3; - // If true, right region derive the origin region_id, - // left region use new_region_id. - // Will be ignored in batch split, use `BatchSplitRequest::right_derive` instead. - bool right_derive = 4 [deprecated=true]; - // It should be false iff the region split by user key such as split table or create partion table etc, - // the new region's will not share the source region size, so it's region size is zero. - // It should be true iff the region's load reaches the threshold such as size, keys, load check etc, - // the new region's size will share the origin region, so it's region size is half of the source region. - bool share_source_region_size = 5; + // This can be only called in internal RaftStore now. + // The split_key must be in the been splitting region. + bytes split_key = 1; + // We split the region into two, first uses the origin + // parent region id, and the second uses the new_region_id. + // We must guarantee that the new_region_id is global unique. + uint64 new_region_id = 2; + // The peer ids for the new split region. + repeated uint64 new_peer_ids = 3; + // If true, right region derive the origin region_id, + // left region use new_region_id. + // Will be ignored in batch split, use `BatchSplitRequest::right_derive` instead. + bool right_derive = 4 [deprecated = true]; + // It should be false iff the region split by user key such as split table or create partion table etc, + // the new region's will not share the source region size, so it's region size is zero. + // It should be true iff the region's load reaches the threshold such as size, keys, load check etc, + // the new region's size will share the origin region, so it's region size is half of the source region. + bool share_source_region_size = 5; } message SplitResponse { - metapb.Region left = 1; - metapb.Region right = 2; + metapb.Region left = 1; + metapb.Region right = 2; } message BatchSplitRequest { - repeated SplitRequest requests = 1; - // If true, the last region derive the origin region_id, - // other regions use new ids. - bool right_derive = 2; - // It should be false iff the region split by user key such as split table or create partion table etc, - // the new region's will not share the source region size, so it's region size is zero. - // It should be true iff the region's load reaches the threshold such as size, keys, load check etc, - // the new region's size will share the origin region, so it's region size is half of the source region. - bool share_source_region_size = 3; + repeated SplitRequest requests = 1; + // If true, the last region derive the origin region_id, + // other regions use new ids. + bool right_derive = 2; + // It should be false iff the region split by user key such as split table or create partion table etc, + // the new region's will not share the source region size, so it's region size is zero. + // It should be true iff the region's load reaches the threshold such as size, keys, load check etc, + // the new region's size will share the origin region, so it's region size is half of the source region. + bool share_source_region_size = 3; } message BatchSplitResponse { - repeated metapb.Region regions = 1; + repeated metapb.Region regions = 1; } message CompactLogRequest { - uint64 compact_index = 1; - uint64 compact_term = 2; - uint64 voter_replicated_index = 3; + uint64 compact_index = 1; + uint64 compact_term = 2; + uint64 voter_replicated_index = 3; } message CompactLogResponse {} message TransferLeaderRequest { - metapb.Peer peer = 1; - repeated metapb.Peer peers = 2; + metapb.Peer peer = 1; + repeated metapb.Peer peers = 2; } message TransferLeaderResponse {} message ComputeHashRequest { - bytes context = 1; + bytes context = 1; } message VerifyHashRequest { - uint64 index = 1; - bytes hash = 2; + uint64 index = 1; + bytes hash = 2; - bytes context = 3; + bytes context = 3; } message VerifyHashResponse {} message PrepareMergeRequest { - uint64 min_index = 1; - metapb.Region target = 2; + uint64 min_index = 1; + metapb.Region target = 2; } message PrepareMergeResponse {} message PrepareFlashbackRequest { - // The start_ts that the current flashback progress is using. - uint64 start_ts = 1; + // The start_ts that the current flashback progress is using. + uint64 start_ts = 1; } message PrepareFlashbackResponse {} @@ -221,106 +227,107 @@ message FinishFlashbackRequest {} message FinishFlashbackResponse {} message CommitMergeRequest { - metapb.Region source = 1; - uint64 commit = 2; - repeated eraftpb.Entry entries = 3; - // Used in v2. When it's present, `source` and `commit` will not be set. - raft_serverpb.RegionLocalState source_state = 4; + metapb.Region source = 1; + uint64 commit = 2; + repeated eraftpb.Entry entries = 3; + // Used in v2. When it's present, `source` and `commit` will not be set. + raft_serverpb.RegionLocalState source_state = 4; + bytes source_meta = 100; - reserved 100 to 200; + reserved 101 to 200; } message CommitMergeResponse {} message RollbackMergeRequest { - uint64 commit = 1; + uint64 commit = 1; } message RollbackMergeResponse {} message SwitchWitnessRequest { - uint64 peer_id = 1; - bool is_witness = 2; + uint64 peer_id = 1; + bool is_witness = 2; } message BatchSwitchWitnessRequest { - repeated SwitchWitnessRequest switch_witnesses = 1; + repeated SwitchWitnessRequest switch_witnesses = 1; } message BatchSwitchWitnessResponse {} message UpdateGcPeerRequest { - repeated uint64 peer_id = 1; + repeated uint64 peer_id = 1; } enum AdminCmdType { - InvalidAdmin = 0; - ChangePeer = 1; - // Use `BatchSplit` instead. - Split = 2 [deprecated=true]; - CompactLog = 3; - TransferLeader = 4; - ComputeHash = 5; - VerifyHash = 6; - PrepareMerge = 7; - CommitMerge = 8; - RollbackMerge = 9; - BatchSplit = 10; - ChangePeerV2 = 11; - PrepareFlashback = 12; - FinishFlashback = 13; - BatchSwitchWitness = 14; - // Command that updates RegionLocalState.gc_peers - UpdateGcPeer = 15; + InvalidAdmin = 0; + ChangePeer = 1; + // Use `BatchSplit` instead. + Split = 2 [deprecated = true]; + CompactLog = 3; + TransferLeader = 4; + ComputeHash = 5; + VerifyHash = 6; + PrepareMerge = 7; + CommitMerge = 8; + RollbackMerge = 9; + BatchSplit = 10; + ChangePeerV2 = 11; + PrepareFlashback = 12; + FinishFlashback = 13; + BatchSwitchWitness = 14; + // Command that updates RegionLocalState.gc_peers + UpdateGcPeer = 15; } message AdminRequest { - AdminCmdType cmd_type = 1; - ChangePeerRequest change_peer = 2; - SplitRequest split = 3 [deprecated=true]; - CompactLogRequest compact_log = 4; - TransferLeaderRequest transfer_leader = 5; - VerifyHashRequest verify_hash = 6; - PrepareMergeRequest prepare_merge = 7; - CommitMergeRequest commit_merge = 8; - RollbackMergeRequest rollback_merge = 9; - BatchSplitRequest splits = 10; - ChangePeerV2Request change_peer_v2 = 11; - ComputeHashRequest compute_hash = 12; - PrepareFlashbackRequest prepare_flashback = 13; - FinishFlashbackRequest finish_flashback = 14; - BatchSwitchWitnessRequest switch_witnesses = 15; - UpdateGcPeerRequest update_gc_peers = 16; + AdminCmdType cmd_type = 1; + ChangePeerRequest change_peer = 2; + SplitRequest split = 3 [deprecated = true]; + CompactLogRequest compact_log = 4; + TransferLeaderRequest transfer_leader = 5; + VerifyHashRequest verify_hash = 6; + PrepareMergeRequest prepare_merge = 7; + CommitMergeRequest commit_merge = 8; + RollbackMergeRequest rollback_merge = 9; + BatchSplitRequest splits = 10; + ChangePeerV2Request change_peer_v2 = 11; + ComputeHashRequest compute_hash = 12; + PrepareFlashbackRequest prepare_flashback = 13; + FinishFlashbackRequest finish_flashback = 14; + BatchSwitchWitnessRequest switch_witnesses = 15; + UpdateGcPeerRequest update_gc_peers = 16; } message AdminResponse { - AdminCmdType cmd_type = 1; - ChangePeerResponse change_peer = 2; - SplitResponse split = 3 [deprecated=true]; - CompactLogResponse compact_log = 4; - TransferLeaderResponse transfer_leader = 5; - VerifyHashResponse verify_hash = 6; - PrepareMergeResponse prepare_merge = 7; - CommitMergeResponse commit_merge = 8; - RollbackMergeResponse rollback_merge = 9; - BatchSplitResponse splits = 10; - ChangePeerV2Response change_peer_v2 = 11; - PrepareFlashbackResponse prepare_flashback = 12; - FinishFlashbackResponse finish_flashback = 13; - BatchSwitchWitnessResponse switch_witnesses = 14; - // UpdateGcPeer doesn't need to be responded. Avoid consuming a tag number. + AdminCmdType cmd_type = 1; + ChangePeerResponse change_peer = 2; + SplitResponse split = 3 [deprecated = true]; + CompactLogResponse compact_log = 4; + TransferLeaderResponse transfer_leader = 5; + VerifyHashResponse verify_hash = 6; + PrepareMergeResponse prepare_merge = 7; + CommitMergeResponse commit_merge = 8; + RollbackMergeResponse rollback_merge = 9; + BatchSplitResponse splits = 10; + ChangePeerV2Response change_peer_v2 = 11; + PrepareFlashbackResponse prepare_flashback = 12; + FinishFlashbackResponse finish_flashback = 13; + BatchSwitchWitnessResponse switch_witnesses = 14; + // UpdateGcPeer doesn't need to be responded. Avoid consuming a tag number. } // For get the leader of the region. message RegionLeaderRequest {} message RegionLeaderResponse { - metapb.Peer leader = 1; + metapb.Peer leader = 1; } // For getting more information of the region. // We add some admin operations (ChangePeer, Split...) into the pb job list, -// then pd server will peek the first one, handle it and then pop it from the job lib. +// then pd server will peek the first one, handle it and then pop it from the job lib. // But sometimes, the pd server may crash before popping. When another pd server // starts and finds the job is running but not finished, it will first check whether // the raft server already has handled this job. @@ -330,75 +337,78 @@ message RegionLeaderResponse { message RegionDetailRequest {} message RegionDetailResponse { - metapb.Region region = 1; - metapb.Peer leader = 2; + metapb.Region region = 1; + metapb.Peer leader = 2; } - enum StatusCmdType { - InvalidStatus = 0; - RegionLeader = 1; - RegionDetail = 2; + InvalidStatus = 0; + RegionLeader = 1; + RegionDetail = 2; } message StatusRequest { - StatusCmdType cmd_type = 1; - RegionLeaderRequest region_leader = 2; - RegionDetailRequest region_detail = 3; + StatusCmdType cmd_type = 1; + RegionLeaderRequest region_leader = 2; + RegionDetailRequest region_detail = 3; } message StatusResponse { - StatusCmdType cmd_type = 1; - RegionLeaderResponse region_leader = 2; - RegionDetailResponse region_detail = 3; + StatusCmdType cmd_type = 1; + RegionLeaderResponse region_leader = 2; + RegionDetailResponse region_detail = 3; } message RaftRequestHeader { - uint64 region_id = 1; - metapb.Peer peer = 2; - // true for read linearization - bool read_quorum = 3; - // 16 bytes, to distinguish request. - bytes uuid = 4; - - metapb.RegionEpoch region_epoch = 5; - uint64 term = 6; - - bool sync_log = 7; - bool replica_read = 8; - - // Read requests can be responsed directly after the Raft applys to `applied_index`. - uint64 applied_index = 9; - - // Custom flags for this raft request. - uint64 flags = 10; - bytes flag_data = 11; - - kvrpcpb.CommandPri priority = 12; - string resource_group_name = 13; + uint64 region_id = 1; + metapb.Peer peer = 2; + // true for read linearization + bool read_quorum = 3; + // 16 bytes, to distinguish request. + bytes uuid = 4; + + metapb.RegionEpoch region_epoch = 5; + uint64 term = 6; + + bool sync_log = 7; + bool replica_read = 8; + + // Read requests can be responsed directly after the Raft applys to `applied_index`. + uint64 applied_index = 9; + + // Custom flags for this raft request. + uint64 flags = 10; + bytes flag_data = 11; + + kvrpcpb.CommandPri priority = 12; + string resource_group_name = 13; } message RaftResponseHeader { - errorpb.Error error = 1; - bytes uuid = 2; - uint64 current_term = 3; + errorpb.Error error = 1; + bytes uuid = 2; + uint64 current_term = 3; +} + +message CustomRequest { + bytes data = 1; } message RaftCmdRequest { - RaftRequestHeader header = 1; - // We can't enclose normal requests and administrator request - // at same time. - repeated Request requests = 2; - AdminRequest admin_request = 3; - StatusRequest status_request = 4; + RaftRequestHeader header = 1; + // We can't enclose normal requests and administrator request + // at same time. + repeated Request requests = 2; + AdminRequest admin_request = 3; + StatusRequest status_request = 4; + CustomRequest custom_request = 5; - reserved 5; - reserved 100 to 200; + reserved 100 to 200; } message RaftCmdResponse { - RaftResponseHeader header = 1; - repeated Response responses = 2; - AdminResponse admin_response = 3; - StatusResponse status_response = 4; + RaftResponseHeader header = 1; + repeated Response responses = 2; + AdminResponse admin_response = 3; + StatusResponse status_response = 4; } diff --git a/proto/raft_serverpb.proto b/proto/raft_serverpb.proto index 49d513ce..c212e5d9 100644 --- a/proto/raft_serverpb.proto +++ b/proto/raft_serverpb.proto @@ -1,80 +1,89 @@ syntax = "proto3"; package raft_serverpb; -import "eraftpb.proto"; -import "metapb.proto"; -import "kvrpcpb.proto"; import "disk_usage.proto"; import "encryptionpb.proto"; +import "eraftpb.proto"; +import "gogoproto/gogo.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; import "rustproto.proto"; -option (rustproto.lite_runtime_all) = true; - option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; message RaftMessage { - uint64 region_id = 1; - metapb.Peer from_peer = 2; - metapb.Peer to_peer = 3; - eraftpb.Message message = 4; - metapb.RegionEpoch region_epoch = 5; - // true means to_peer is a tombstone peer and it should remove itself. - bool is_tombstone = 6; - // Region key range [start_key, end_key). - bytes start_key = 7; - bytes end_key = 8; - // If it has value, to_peer should be removed if merge is never going to complete. - metapb.Region merge_target = 9; - ExtraMessage extra_msg = 10; - bytes extra_ctx = 11; - - disk_usage.DiskUsage disk_usage = 12; + uint64 region_id = 1; + metapb.Peer from_peer = 2; + metapb.Peer to_peer = 3; + eraftpb.Message message = 4; + metapb.RegionEpoch region_epoch = 5; + // true means to_peer is a tombstone peer and it should remove itself. + bool is_tombstone = 6; + // Region key range [start_key, end_key). + bytes start_key = 7; + bytes end_key = 8; + // If it has value, to_peer should be removed if merge is never going to complete. + metapb.Region merge_target = 9; + ExtraMessage extra_msg = 10; + bytes extra_ctx = 11; + + disk_usage.DiskUsage disk_usage = 12; } message RaftTruncatedState { - uint64 index = 1; - uint64 term = 2; + uint64 index = 1; + uint64 term = 2; } message SnapshotCFFile { - string cf = 1; - uint64 size = 2; - uint32 checksum = 3; + string cf = 1; + uint64 size = 2; + uint32 checksum = 3; } message SnapshotMeta { - repeated SnapshotCFFile cf_files = 1; - // true means this snapshot is triggered for load balance - bool for_balance = 2; - // true means this is an empty snapshot for witness - bool for_witness = 3; - // the timestamp second to generate snapshot - uint64 start = 4; - // the duration of generating snapshot - uint64 generate_duration_sec = 5; - // the path of the tablet snapshot, it should only be used for v1 to receive - // snapshot from v2 - string tablet_snap_path = 6; - // A hint of the latest commit index on leader when sending snapshot. - // It should only be used for v2 to send snapshot to v1. - // See https://github.com/pingcap/tiflash/issues/7568 - uint64 commit_index_hint = 7; + repeated SnapshotCFFile cf_files = 1; + // true means this snapshot is triggered for load balance + bool for_balance = 2; + // true means this is an empty snapshot for witness + bool for_witness = 3; + // the timestamp second to generate snapshot + uint64 start = 4; + // the duration of generating snapshot + uint64 generate_duration_sec = 5; + // the path of the tablet snapshot, it should only be used for v1 to receive + // snapshot from v2 + string tablet_snap_path = 6; + // A hint of the latest commit index on leader when sending snapshot. + // It should only be used for v2 to send snapshot to v1. + // See https://github.com/pingcap/tiflash/issues/7568 + uint64 commit_index_hint = 7; + + // Reserved for a downstream fork + reserved 1000 to 1199; } message SnapshotChunk { - RaftMessage message = 1; - bytes data = 2; + RaftMessage message = 1; + bytes data = 2; } message Done {} message TabletSnapshotFileMeta { - uint64 file_size = 1; - string file_name = 2; - // Some block data. Unencrypted. - bytes head_chunk = 3; - // trailing data including checksum. Unencrypted. - bytes trailing_chunk = 4; + uint64 file_size = 1; + string file_name = 2; + // Some block data. Unencrypted. + bytes head_chunk = 3; + // trailing data including checksum. Unencrypted. + bytes trailing_chunk = 4; } // Snapshot preview for server to decide whether skip some files. @@ -82,215 +91,230 @@ message TabletSnapshotFileMeta { // keep sending specified files. Only SST files can be skipped, all // other files should always be sent. message TabletSnapshotPreview { - repeated TabletSnapshotFileMeta metas = 1; - // There may be too many metas, use a flag to indicate all metas - // are sent. - bool end = 2; + repeated TabletSnapshotFileMeta metas = 1; + // There may be too many metas, use a flag to indicate all metas + // are sent. + bool end = 2; } message TabletSnapshotFileChunk { - uint64 file_size = 1; - string file_name = 2; - // Encrypted. - bytes data = 3; - // Initial vector if encryption is enabled. - bytes iv = 4; - encryptionpb.DataKey key = 5; + uint64 file_size = 1; + string file_name = 2; + // Encrypted. + bytes data = 3; + // Initial vector if encryption is enabled. + bytes iv = 4; + encryptionpb.DataKey key = 5; } message TabletSnapshotHead { - RaftMessage message = 1; - bool use_cache = 2; + RaftMessage message = 1; + bool use_cache = 2; } message TabletSnapshotEnd { - // Checksum of all data sent in `TabletSnapshotFileChunk.data` and - // `TabletSnapshotFileChunk.file_name`. - uint64 checksum = 1; + // Checksum of all data sent in `TabletSnapshotFileChunk.data` and + // `TabletSnapshotFileChunk.file_name`. + uint64 checksum = 1; } message TabletSnapshotRequest { - oneof payload { - TabletSnapshotHead head = 1; - TabletSnapshotPreview preview = 2; - TabletSnapshotFileChunk chunk = 3; - TabletSnapshotEnd end = 4; - } + oneof payload { + TabletSnapshotHead head = 1; + TabletSnapshotPreview preview = 2; + TabletSnapshotFileChunk chunk = 3; + TabletSnapshotEnd end = 4; + } } message AcceptedSnapshotFiles { - repeated string file_name = 1; + repeated string file_name = 1; } message TabletSnapshotResponse { - AcceptedSnapshotFiles files = 1; + AcceptedSnapshotFiles files = 1; } message KeyValue { - bytes key = 1; - bytes value = 2; + bytes key = 1; + bytes value = 2; } message RaftSnapshotData { - metapb.Region region = 1; - uint64 file_size = 2; - repeated KeyValue data = 3; - uint64 version = 4; - SnapshotMeta meta = 5; - repeated metapb.Peer removed_records = 6; - repeated MergedRecord merged_records = 7; + metapb.Region region = 1; + uint64 file_size = 2; + repeated KeyValue data = 3; + uint64 version = 4; + SnapshotMeta meta = 5; + repeated metapb.Peer removed_records = 6; + repeated MergedRecord merged_records = 7; } message StoreIdent { - uint64 cluster_id = 1; - uint64 store_id = 2; - kvrpcpb.APIVersion api_version = 3; + uint64 cluster_id = 1; + uint64 store_id = 2; + kvrpcpb.APIVersion api_version = 3; } message StoreRecoverState { - // Used for TiKV start recovery when WAL of KVDB was disabled. - // TiKV may read all relations between seqno and raft log index, and replay - // all raft logs which corresponding seqno smaller than the seqno here. - // After TiKV replays all raft logs and flushed KV data, the seqno here must - // be updated. - uint64 seqno = 1; + // Used for TiKV start recovery when WAL of KVDB was disabled. + // TiKV may read all relations between seqno and raft log index, and replay + // all raft logs which corresponding seqno smaller than the seqno here. + // After TiKV replays all raft logs and flushed KV data, the seqno here must + // be updated. + uint64 seqno = 1; } message RaftLocalState { - eraftpb.HardState hard_state = 1; - uint64 last_index = 2; + eraftpb.HardState hard_state = 1; + uint64 last_index = 2; } message RaftApplyState { - uint64 applied_index = 1; - uint64 last_commit_index = 3; - uint64 commit_index = 4; - uint64 commit_term = 5; - RaftTruncatedState truncated_state = 2; + uint64 applied_index = 1; + uint64 last_commit_index = 3; + uint64 commit_index = 4; + uint64 commit_term = 5; + RaftTruncatedState truncated_state = 2; } enum PeerState { - Normal = 0; - Applying = 1; - Tombstone = 2; - Merging = 3; - // Currently used for witness to non-witness conversion: When a witness - // has just become a non-witness, we need to set and persist this state, - // so that when the service restarts before applying snapshot, we can - // actively request snapshot when initializing this peer. - Unavailable = 4; + Normal = 0; + Applying = 1; + Tombstone = 2; + Merging = 3; + // Currently used for witness to non-witness conversion: When a witness + // has just become a non-witness, we need to set and persist this state, + // so that when the service restarts before applying snapshot, we can + // actively request snapshot when initializing this peer. + Unavailable = 4; } message MergeState { - uint64 min_index = 1; - metapb.Region target = 2; - uint64 commit = 3; + uint64 min_index = 1; + metapb.Region target = 2; + uint64 commit = 3; } message MergedRecord { - uint64 source_region_id = 1; - metapb.RegionEpoch source_epoch = 2; - // Peers of source region when merge is committed. - repeated metapb.Peer source_peers = 3; - // Removed peers (by confchange) of source region when merge is committed. - repeated metapb.Peer source_removed_records = 9; - uint64 target_region_id = 4; - metapb.RegionEpoch target_epoch = 5; - repeated metapb.Peer target_peers = 6; - // Commit merge index. - uint64 index = 7; - // Prepare merge index. - uint64 source_index = 8; + uint64 source_region_id = 1; + metapb.RegionEpoch source_epoch = 2; + // Peers of source region when merge is committed. + repeated metapb.Peer source_peers = 3; + // Removed peers (by confchange) of source region when merge is committed. + repeated metapb.Peer source_removed_records = 9; + uint64 target_region_id = 4; + metapb.RegionEpoch target_epoch = 5; + repeated metapb.Peer target_peers = 6; + // Commit merge index. + uint64 index = 7; + // Prepare merge index. + uint64 source_index = 8; } message RegionLocalState { - PeerState state = 1; - metapb.Region region = 2; - MergeState merge_state = 3; - // The apply index corresponding to the storage when it's initialized. - uint64 tablet_index = 4; - // Raft doesn't guarantee peer will be removed in the end. In v1, peer finds - // out its destiny by logs or broadcast; in v2, leader is responsible to - // ensure removed peers are destroyed. - // Note: only peers who has been part of this region can be in this list. - repeated metapb.Peer removed_records = 5; - // Merged peer can't be deleted like gc peers. Instead, leader needs to - // query target peer to decide whether source peer can be destroyed. - repeated MergedRecord merged_records = 6; + PeerState state = 1; + metapb.Region region = 2; + MergeState merge_state = 3; + // The apply index corresponding to the storage when it's initialized. + uint64 tablet_index = 4; + // Raft doesn't guarantee peer will be removed in the end. In v1, peer finds + // out its destiny by logs or broadcast; in v2, leader is responsible to + // ensure removed peers are destroyed. + // Note: only peers who has been part of this region can be in this list. + repeated metapb.Peer removed_records = 5; + // Merged peer can't be deleted like gc peers. Instead, leader needs to + // query target peer to decide whether source peer can be destroyed. + repeated MergedRecord merged_records = 6; } message RegionSequenceNumberRelation { - uint64 region_id = 1; - uint64 sequence_number = 2; - RaftApplyState apply_state = 3; - RegionLocalState region_state = 4; + uint64 region_id = 1; + uint64 sequence_number = 2; + RaftApplyState apply_state = 3; + RegionLocalState region_state = 4; } message AvailabilityContext { - uint64 from_region_id = 1; - metapb.RegionEpoch from_region_epoch = 2; - bool unavailable = 3; - bool trimmed = 4; + uint64 from_region_id = 1; + metapb.RegionEpoch from_region_epoch = 2; + bool unavailable = 3; + bool trimmed = 4; } enum ExtraMessageType { - MsgRegionWakeUp = 0; - MsgWantRollbackMerge = 1; - MsgCheckStalePeer = 2; - MsgCheckStalePeerResponse = 3; - // If leader is going to sleep, it will send requests to all its followers - // to make sure they all agree to sleep. - MsgHibernateRequest = 4; - MsgHibernateResponse = 5; - MsgRejectRaftLogCausedByMemoryUsage = 6; - MsgAvailabilityRequest = 7; - MsgAvailabilityResponse = 8; - MsgVoterReplicatedIndexRequest = 9; - MsgVoterReplicatedIndexResponse = 10; - // Message means that `from` is tombstone. Leader can then update removed_records. - MsgGcPeerRequest = 11; - MsgGcPeerResponse = 12; - MsgFlushMemtable = 13; - MsgRefreshBuckets = 14; + MsgRegionWakeUp = 0; + MsgWantRollbackMerge = 1; + MsgCheckStalePeer = 2; + MsgCheckStalePeerResponse = 3; + // If leader is going to sleep, it will send requests to all its followers + // to make sure they all agree to sleep. + MsgHibernateRequest = 4; + MsgHibernateResponse = 5; + MsgRejectRaftLogCausedByMemoryUsage = 6; + MsgAvailabilityRequest = 7; + MsgAvailabilityResponse = 8; + MsgVoterReplicatedIndexRequest = 9; + MsgVoterReplicatedIndexResponse = 10; + // Message means that `from` is tombstone. Leader can then update removed_records. + MsgGcPeerRequest = 11; + MsgGcPeerResponse = 12; + MsgFlushMemtable = 13; + MsgRefreshBuckets = 14; + // Messages for the snapshot gen precheck process. + MsgSnapGenPrecheckRequest = 15; + MsgSnapGenPrecheckResponse = 16; + // Used in transfer leader process for leader to inform follower to load the + // region into in-memory engine if the relevant region is cached. + MsgPreLoadRegionRequest = 17; + // Used in transfer leader process for follower to inform leader the completes + // of the region cache + MsgPreLoadRegionResponse = 18; } message FlushMemtable { - uint64 region_id = 1; + uint64 region_id = 1; } message RefreshBuckets { - uint64 version = 1; - repeated bytes keys = 2; - repeated uint64 sizes = 3; + uint64 version = 1; + repeated bytes keys = 2; + repeated uint64 sizes = 3; } message CheckGcPeer { - // The region ID who triggers the check and wait for report. It should be - // the ID of RaftMessage.from. - uint64 from_region_id = 1; - // The region ID to be checked if should be destroyed. - uint64 check_region_id = 2; - // The epoch of the region to be checked. - metapb.RegionEpoch check_region_epoch = 3; - // The peer to be checked. - metapb.Peer check_peer = 4; + // The region ID who triggers the check and wait for report. It should be + // the ID of RaftMessage.from. + uint64 from_region_id = 1; + // The region ID to be checked if should be destroyed. + uint64 check_region_id = 2; + // The epoch of the region to be checked. + metapb.RegionEpoch check_region_epoch = 3; + // The peer to be checked. + metapb.Peer check_peer = 4; } message ExtraMessage { - ExtraMessageType type = 1; - // It's merge related index. In `WantRollbackMerge`, it's prepare merge index. In - // `MsgGcPeerRequest`, it's the commit merge index. In `MsgVoterReplicatedIndexRequest` - // it's the voter_replicated_index. - uint64 index = 2; - // In `MsgCheckStalePeerResponse`, it's the peers that receiver can continue to query. - repeated metapb.Peer check_peers = 3; - bool wait_data = 4; - // Flag for forcely wake up hibernate regions if true. - bool forcely_awaken = 5; - CheckGcPeer check_gc_peer = 6; - FlushMemtable flush_memtable = 7; - // Used by `MsgAvailabilityRequest` and `MsgAvailabilityResponse` in v2. - AvailabilityContext availability_context = 8; - // notice the peer to refresh buckets version - RefreshBuckets refresh_buckets = 9; + ExtraMessageType type = 1; + // It's merge related index. In `WantRollbackMerge`, it's prepare merge index. In + // `MsgGcPeerRequest`, it's the commit merge index. In `MsgVoterReplicatedIndexRequest` + // it's the voter_replicated_index. + uint64 index = 2; + // In `MsgCheckStalePeerResponse`, it's the peers that receiver can continue to query. + repeated metapb.Peer check_peers = 3; + bool wait_data = 4; + // Flag for forcely wake up hibernate regions if true. + bool forcely_awaken = 5; + CheckGcPeer check_gc_peer = 6; + FlushMemtable flush_memtable = 7; + // Used by `MsgAvailabilityRequest` and `MsgAvailabilityResponse` in v2. + AvailabilityContext availability_context = 8; + // notice the peer to refresh buckets version + RefreshBuckets refresh_buckets = 9; + // snap_gen_precheck_passed is used to indicate the precheck result when + // a follower responds to a leader's snapshot gen precheck request. + bool snap_gen_precheck_passed = 10; + + // Reserved for a downstream fork + reserved 1000 to 1199; } diff --git a/proto/recoverdatapb.proto b/proto/recoverdatapb.proto index de7a1b54..ada1f2e6 100644 --- a/proto/recoverdatapb.proto +++ b/proto/recoverdatapb.proto @@ -4,65 +4,68 @@ package recover_data; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; // request to read region meata from a store message ReadRegionMetaRequest { - uint64 store_id = 1; + uint64 store_id = 1; } message Error { - string msg = 1; + string msg = 1; } message RegionMeta { - uint64 region_id = 1; - uint64 peer_id = 2; - uint64 last_log_term = 3; - uint64 last_index = 4; - uint64 commit_index = 5; - uint64 version = 6; - bool tombstone = 7; //reserved, it may be used in late phase for peer check - bytes start_key = 8; - bytes end_key = 9; + uint64 region_id = 1; + uint64 peer_id = 2; + uint64 last_log_term = 3; + uint64 last_index = 4; + uint64 commit_index = 5; + uint64 version = 6; + bool tombstone = 7; //reserved, it may be used in late phase for peer check + bytes start_key = 8; + bytes end_key = 9; } // command to store for recover region message RecoverRegionRequest { - uint64 region_id = 1; - bool as_leader = 2; // force region_id as leader - bool tombstone = 3; // set Peer to tombstoned in late phase + uint64 region_id = 1; + bool as_leader = 2; // force region_id as leader + bool tombstone = 3; // set Peer to tombstoned in late phase } message RecoverRegionResponse { - Error error = 1; - uint64 store_id = 2; + Error error = 1; + uint64 store_id = 2; } // wait apply to last index - message WaitApplyRequest { - uint64 store_id = 1; - } +message WaitApplyRequest { + uint64 store_id = 1; +} - message WaitApplyResponse { - Error error = 1; - } +message WaitApplyResponse { + Error error = 1; +} // resolve data by resolved_ts message ResolveKvDataRequest { - uint64 resolved_ts = 1; + uint64 resolved_ts = 1; } message ResolveKvDataResponse { - Error error = 1; - uint64 store_id = 2; - uint64 resolved_key_count = 3; // reserved for summary of restore - // cursor of delete key.commit_ts, reserved for progress of restore - // progress is (current_commit_ts - resolved_ts) / (backup_ts - resolved_ts) x 100% - uint64 current_commit_ts = 4; + Error error = 1; + uint64 store_id = 2; + uint64 resolved_key_count = 3; // reserved for summary of restore + // cursor of delete key.commit_ts, reserved for progress of restore + // progress is (current_commit_ts - resolved_ts) / (backup_ts - resolved_ts) x 100% + uint64 current_commit_ts = 4; } // a recovery workflow likes @@ -71,12 +74,12 @@ message ResolveKvDataResponse { // 3. BR wait all regions in tikv to apply to last index (no write during the recovery) // 4. BR resolved kv data service RecoverData { - // read region meta to ready region meta - rpc ReadRegionMeta(ReadRegionMetaRequest) returns (stream RegionMeta) {} - // execute the recovery command - rpc RecoverRegion(stream RecoverRegionRequest) returns (RecoverRegionResponse) {} - // wait all region apply to last index - rpc WaitApply(WaitApplyRequest) returns (WaitApplyResponse) {} - // execute delete data from kv db - rpc ResolveKvData(ResolveKvDataRequest) returns (stream ResolveKvDataResponse) {} -} \ No newline at end of file + // read region meta to ready region meta + rpc ReadRegionMeta(ReadRegionMetaRequest) returns (stream RegionMeta) {} + // execute the recovery command + rpc RecoverRegion(stream RecoverRegionRequest) returns (RecoverRegionResponse) {} + // wait all region apply to last index + rpc WaitApply(WaitApplyRequest) returns (WaitApplyResponse) {} + // execute delete data from kv db + rpc ResolveKvData(ResolveKvDataRequest) returns (stream ResolveKvDataResponse) {} +} diff --git a/proto/replication_modepb.proto b/proto/replication_modepb.proto index c60005b6..af31307b 100644 --- a/proto/replication_modepb.proto +++ b/proto/replication_modepb.proto @@ -1,62 +1,76 @@ syntax = "proto3"; package replication_modepb; +import "gogoproto/gogo.proto"; +import "rustproto.proto"; + +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; + enum ReplicationMode { - // The standard mode. Replicate logs to majority peer. - MAJORITY = 0; - // DR mode. Replicate logs among 2 DCs. - DR_AUTO_SYNC = 1; + // The standard mode. Replicate logs to majority peer. + MAJORITY = 0; + // DR mode. Replicate logs among 2 DCs. + DR_AUTO_SYNC = 1; } // The replication status sync from PD to TiKV. message ReplicationStatus { - ReplicationMode mode = 1; - DRAutoSync dr_auto_sync = 2; + ReplicationMode mode = 1; + DRAutoSync dr_auto_sync = 2; + + // Reserved for a downstream fork + reserved 1000 to 1199; } enum DRAutoSyncState { - // Raft logs need to sync between different DCs - SYNC = 0; - // Wait for switching to ASYNC. Stop sync raft logs between DCs. - ASYNC_WAIT = 1; - // Raft logs need to sync to majority peers - ASYNC = 2; - // Switching from ASYNC to SYNC mode - SYNC_RECOVER = 3; + // Raft logs need to sync between different DCs + SYNC = 0; + // Wait for switching to ASYNC. Stop sync raft logs between DCs. + ASYNC_WAIT = 1; + // Raft logs need to sync to majority peers + ASYNC = 2; + // Switching from ASYNC to SYNC mode + SYNC_RECOVER = 3; } // The status of dr-autosync mode. message DRAutoSync { - // The key of the label that used for distinguish different DC. - string label_key = 1; - DRAutoSyncState state = 2; - // Unique ID of the state, it increases after each state transfer. - uint64 state_id = 3; - // Duration to wait before switching to SYNC by force (in seconds) - int32 wait_sync_timeout_hint = 4; - // Stores should only sync messages with available stores when state is ASYNC or ASYNC_WAIT. - repeated uint64 available_stores = 5; - // Stores should forbid region split. - bool pause_region_split = 6; + // The key of the label that used for distinguish different DC. + string label_key = 1; + DRAutoSyncState state = 2; + // Unique ID of the state, it increases after each state transfer. + uint64 state_id = 3; + // Duration to wait before switching to SYNC by force (in seconds) + int32 wait_sync_timeout_hint = 4; + // Stores should only sync messages with available stores when state is ASYNC or ASYNC_WAIT. + repeated uint64 available_stores = 5; + // Stores should forbid region split. + bool pause_region_split = 6; } enum RegionReplicationState { - // The region's state is unknown - UNKNOWN = 0; - // Logs sync to majority peers - SIMPLE_MAJORITY = 1; - // Logs sync to different DCs - INTEGRITY_OVER_LABEL = 2; + // The region's state is unknown + UNKNOWN = 0; + // Logs sync to majority peers + SIMPLE_MAJORITY = 1; + // Logs sync to different DCs + INTEGRITY_OVER_LABEL = 2; } // The replication status sync from TiKV to PD. message RegionReplicationStatus { - RegionReplicationState state = 1; - // Unique ID of the state, it increases after each state transfer. - uint64 state_id = 2; + RegionReplicationState state = 1; + // Unique ID of the state, it increases after each state transfer. + uint64 state_id = 2; } message StoreDRAutoSyncStatus { - DRAutoSyncState state = 1; - uint64 state_id = 2; + DRAutoSyncState state = 1; + uint64 state_id = 2; } diff --git a/proto/resource_manager.proto b/proto/resource_manager.proto index aa7de748..302efc9d 100644 --- a/proto/resource_manager.proto +++ b/proto/resource_manager.proto @@ -4,8 +4,11 @@ package resource_manager; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; @@ -23,27 +26,54 @@ service ResourceManager { rpc AcquireTokenBuckets(stream TokenBucketsRequest) returns (stream TokenBucketsResponse) {} } -message ListResourceGroupsRequest{} +// KeyspaceIDValue is a wrapper for the value of keyspace ID. +// Because the 0 value is a valid keyspace ID, we need to use a wrapper to distinguish it from the null keyspace ID. +message KeyspaceIDValue { + uint32 value = 1; +} + +message ListResourceGroupsRequest { + bool with_ru_stats = 1; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, the listed resource groups will be filtered + // by the given keyspace ID. + KeyspaceIDValue keyspace_id = 2; +} -message ListResourceGroupsResponse{ +message ListResourceGroupsResponse { Error error = 1; repeated ResourceGroup groups = 2; } message GetResourceGroupRequest { string resource_group_name = 1; + bool with_ru_stats = 2; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, it will try to get the resource group within + // the given keyspace ID. + KeyspaceIDValue keyspace_id = 3; } -message GetResourceGroupResponse{ +message GetResourceGroupResponse { Error error = 1; ResourceGroup group = 2; } message DeleteResourceGroupRequest { string resource_group_name = 1; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, it will try to delete the resource group within + // the given keyspace ID. + KeyspaceIDValue keyspace_id = 2; } -message DeleteResourceGroupResponse{ +message DeleteResourceGroupResponse { Error error = 1; string body = 2; } @@ -64,25 +94,31 @@ message TokenBucketsRequest { } message TokenBucketRequest { - message RequestRU { - repeated RequestUnitItem request_r_u = 1; - } - message RequestRawResource { - repeated RawResourceItem request_raw_resource = 1; - } - - string resource_group_name = 1; - oneof request { - // RU mode, group settings with WRU/RRU etc resource abstract unit. - RequestRU ru_items = 2; - // Raw mode, group settings with CPU/IO etc resource unit. - RequestRawResource raw_resource_items = 3; - } - // Aggregate statistics in group level. - Consumption consumption_since_last_request = 4; - // label background request. - bool is_background = 5; - bool is_tiflash = 6; + message RequestRU { + repeated RequestUnitItem request_r_u = 1; + } + message RequestRawResource { + repeated RawResourceItem request_raw_resource = 1; + } + + string resource_group_name = 1; + oneof request { + // RU mode, group settings with WRU/RRU etc resource abstract unit. + RequestRU ru_items = 2; + // Raw mode, group settings with CPU/IO etc resource unit. + RequestRawResource raw_resource_items = 3; + } + // Aggregate statistics in group level. + Consumption consumption_since_last_request = 4; + // label background request. + bool is_background = 5; + bool is_tiflash = 6; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, it will try to request the token bucket from + // the resource group within the given keyspace ID. + KeyspaceIDValue keyspace_id = 7; } message TokenBucketsResponse { @@ -96,6 +132,12 @@ message TokenBucketResponse { repeated GrantedRUTokenBucket granted_r_u_tokens = 2; // Raw mode repeated GrantedRawResourceTokenBucket granted_resource_tokens = 3; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version, + // which can be safely ignored to keep compatibility. + // - If the keyspace ID is set to a valid value, it means this response is from the resource + // group within this keyspace ID. + KeyspaceIDValue keyspace_id = 4; } message GrantedRUTokenBucket { @@ -116,7 +158,7 @@ enum RequestUnitType { enum RawResourceType { CPU = 0; - IOReadFlow = 1; + IOReadFlow = 1; IOWriteFlow = 2; } @@ -129,6 +171,14 @@ message Consumption { double sql_layer_cpu_time_ms = 6; double kv_read_rpc_count = 7; double kv_write_rpc_count = 8; + uint64 read_cross_az_traffic_bytes = 9; + uint64 write_cross_az_traffic_bytes = 10; + // RUv2 is an experimental v2 RU calculation. + // For now it only records the consumption without actual token deduction. + double tikv_r_u_v2 = 11; + double tidb_r_u_v2 = 12; + // tiflash_r_u_v2 equals tiflash RRU + tiflash WRU. + double tiflash_r_u_v2 = 13; } message RequestUnitItem { @@ -142,12 +192,12 @@ message RawResourceItem { } enum GroupMode { - Unknown = 0; - RUMode = 1; - RawMode = 2; + Unknown = 0; + RUMode = 1; + RawMode = 2; } -// ResourceGroup the settings definitions. +// ResourceGroup the settings definitions. message ResourceGroup { string name = 1; GroupMode mode = 2; @@ -159,7 +209,16 @@ message ResourceGroup { uint32 priority = 5; // Runaway queries settings RunawaySettings runaway_settings = 6; + // Background task control settings. BackgroundSettings background_settings = 7; + // RU consumption statistics. + Consumption RUStats = 8; + // The keyspace ID that the resource group belongs to. + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, it will directly be used. + KeyspaceIDValue keyspace_id = 9; } message GroupRequestUnitSettings { @@ -176,7 +235,7 @@ message TokenBucket { TokenLimitSettings settings = 1; // Once used to reconfigure, the tokens is delta tokens. double tokens = 2; -} +} message TokenLimitSettings { uint64 fill_rate = 1; @@ -185,14 +244,15 @@ message TokenLimitSettings { } message Error { - string message = 1; + string message = 1; } enum RunawayAction { NoneAction = 0; - DryRun = 1; // do nothing - CoolDown = 2; // deprioritize the task - Kill = 3; // kill the task + DryRun = 1; // do nothing + CoolDown = 2; // deprioritize the task + Kill = 3; // kill the task + SwitchGroup = 4; // switch the task to another group } enum RunawayWatchType { @@ -204,30 +264,38 @@ enum RunawayWatchType { message RunawayRule { uint64 exec_elapsed_time_ms = 1; + int64 processed_keys = 2; + int64 request_unit = 3; } message RunawayWatch { // how long would the watch last int64 lasting_duration_ms = 1; - RunawayWatchType type = 2; + RunawayWatchType type = 2; } message RunawaySettings { - RunawayRule rule = 1; + RunawayRule rule = 1; RunawayAction action = 2; RunawayWatch watch = 3; + // When the runaway action is `SwitchGroup`, + // this field will be used to indicate which group to switch. + string switch_group_name = 4; } message BackgroundSettings { - repeated string job_types = 1; + // background task types. + repeated string job_types = 1; + // the percentage limit of total resource(cpu/io) that background tasks can use. + uint64 utilization_limit = 2; } message Participant { - // name is the unique name of the resource manager participant. - string name = 1; - // id is the unique id of the resource manager participant. - uint64 id = 2; - // listen_urls is the serivce endpoint list in the url format. - // listen_urls[0] is primary service endpoint. - repeated string listen_urls = 3; + // name is the unique name of the resource manager participant. + string name = 1; + // id is the unique id of the resource manager participant. + uint64 id = 2; + // listen_urls is the serivce endpoint list in the url format. + // listen_urls[0] is primary service endpoint. + repeated string listen_urls = 3; } diff --git a/proto/resource_usage_agent.proto b/proto/resource_usage_agent.proto index 4250ec25..576ab926 100644 --- a/proto/resource_usage_agent.proto +++ b/proto/resource_usage_agent.proto @@ -5,25 +5,27 @@ package resource_usage_agent; import "gogoproto/gogo.proto"; import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // ResourceUsageAgent is the service for storing resource usage records. service ResourceUsageAgent { - // Report the resource usage records. By default, the records with the same - // resource group tag will be batched by minute. - rpc Report(stream ResourceUsageRecord) returns (EmptyResponse) {} + // Report the resource usage records. By default, the records with the same + // resource group tag will be batched by minute. + rpc Report(stream ResourceUsageRecord) returns (EmptyResponse) {} } // TiKV implements ResourceMeteringPubSub service for clients to subscribe to resource metering records. service ResourceMeteringPubSub { - // Clients subscribe to resource metering records through this RPC, and TiKV periodically (e.g. per minute) - // publishes resource metering records to clients via gRPC stream. - rpc Subscribe(ResourceMeteringRequest) returns (stream ResourceUsageRecord) {} + // Clients subscribe to resource metering records through this RPC, and TiKV periodically (e.g. per minute) + // publishes resource metering records to clients via gRPC stream. + rpc Subscribe(ResourceMeteringRequest) returns (stream ResourceUsageRecord) {} } message ResourceMeteringRequest {} @@ -31,20 +33,31 @@ message ResourceMeteringRequest {} message EmptyResponse {} message ResourceUsageRecord { - oneof record_oneof { - GroupTagRecord record = 1; - } + oneof record_oneof { + GroupTagRecord record = 1; + RegionRecord region_record = 2; + } } // GroupTagRecord is a set of resource usage data grouped by resource_group_tag. message GroupTagRecord { - bytes resource_group_tag = 1; - repeated GroupTagRecordItem items = 2; + bytes resource_group_tag = 1; + repeated GroupTagRecordItem items = 2; } message GroupTagRecordItem { - uint64 timestamp_sec = 1; - uint32 cpu_time_ms = 2; - uint32 read_keys = 3; - uint32 write_keys = 4; + uint64 timestamp_sec = 1; + uint32 cpu_time_ms = 2; + uint32 read_keys = 3; + uint32 write_keys = 4; + uint64 network_in_bytes = 5; + uint64 network_out_bytes = 6; + uint64 logical_read_bytes = 7; + uint64 logical_write_bytes = 8; +} + +// RegionRecord is a set of resource usage data grouped by region. +message RegionRecord { + uint64 region_id = 1; + repeated GroupTagRecordItem items = 2; } diff --git a/proto/schedulingpb.proto b/proto/schedulingpb.proto index 83b7de46..9167a3ae 100644 --- a/proto/schedulingpb.proto +++ b/proto/schedulingpb.proto @@ -1,197 +1,217 @@ syntax = "proto3"; package schedulingpb; -import "pdpb.proto"; import "gogoproto/gogo.proto"; -import "rustproto.proto"; import "metapb.proto"; +import "pdpb.proto"; +import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - service Scheduling { - rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} + rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} + + rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {} - rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {} + rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} - rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} + rpc ScatterRegions(ScatterRegionsRequest) returns (ScatterRegionsResponse) {} - rpc ScatterRegions(ScatterRegionsRequest) returns (ScatterRegionsResponse) {} + rpc GetOperator(GetOperatorRequest) returns (GetOperatorResponse) {} - rpc GetOperator(GetOperatorRequest) returns (GetOperatorResponse) {} - - rpc AskBatchSplit(AskBatchSplitRequest) returns (AskBatchSplitResponse) {} + rpc AskBatchSplit(AskBatchSplitRequest) returns (AskBatchSplitResponse) {} + + rpc RegionBuckets(stream RegionBucketsRequest) returns (stream RegionBucketsResponse) {} } message RequestHeader { - // cluster_id is the ID of the cluster which be sent to. - uint64 cluster_id = 1; - // sender_id is the ID of the sender server. - uint64 sender_id = 2; + // cluster_id is the ID of the cluster which be sent to. + uint64 cluster_id = 1; + // sender_id is the ID of the sender server. + uint64 sender_id = 2; } message ResponseHeader { - // cluster_id is the ID of the cluster which sent the response. - uint64 cluster_id = 1; - Error error = 2; + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; } enum ErrorType { - OK = 0; - UNKNOWN = 1; - NOT_BOOTSTRAPPED = 2; - ALREADY_BOOTSTRAPPED = 3; - INVALID_VALUE = 4; - CLUSTER_MISMATCHED = 5; + OK = 0; + UNKNOWN = 1; + NOT_BOOTSTRAPPED = 2; + ALREADY_BOOTSTRAPPED = 3; + INVALID_VALUE = 4; + CLUSTER_MISMATCHED = 5; } message Error { - ErrorType type = 1; - string message = 2; + ErrorType type = 1; + string message = 2; } message Participant { - // name is the unique name of the scheduling participant. - string name = 1; - // id is the unique id of the scheduling participant. - uint64 id = 2; - // listen_urls is the serivce endpoint list in the url format. - // listen_urls[0] is primary service endpoint. - repeated string listen_urls = 3; + // name is the unique name of the scheduling participant. + string name = 1; + // id is the unique id of the scheduling participant. + uint64 id = 2; + // listen_urls is the serivce endpoint list in the url format. + // listen_urls[0] is primary service endpoint. + repeated string listen_urls = 3; } message StoreHeartbeatRequest { - RequestHeader header = 1; - pdpb.StoreStats stats = 2; + RequestHeader header = 1; + pdpb.StoreStats stats = 2; } message StoreHeartbeatResponse { - ResponseHeader header = 1; - string cluster_version = 2; + ResponseHeader header = 1; + string cluster_version = 2; } message RegionHeartbeatRequest { - RequestHeader header = 1; - - metapb.Region region = 2; - // Leader Peer sending the heartbeat. - metapb.Peer leader = 3; - // Term is the term of raft group. - uint64 term = 4; - // Leader considers that these peers are down. - repeated pdpb.PeerStats down_peers = 5; - // Pending peers are the peers that the leader can't consider as - // working followers. - repeated metapb.Peer pending_peers = 6; - // Bytes read/written during this period. - uint64 bytes_written = 7; - uint64 bytes_read = 8; - // Keys read/written during this period. - uint64 keys_written = 9; - uint64 keys_read = 10; - // Approximate region size. - uint64 approximate_size = 11; - // Approximate number of keys. - uint64 approximate_keys = 12; - // QueryStats reported write query stats, and there are read query stats in store heartbeat - pdpb.QueryStats query_stats = 13; - // Actually reported time interval - pdpb.TimeInterval interval = 14; + RequestHeader header = 1; + + metapb.Region region = 2; + // Leader Peer sending the heartbeat. + metapb.Peer leader = 3; + // Term is the term of raft group. + uint64 term = 4; + // Leader considers that these peers are down. + repeated pdpb.PeerStats down_peers = 5; + // Pending peers are the peers that the leader can't consider as + // working followers. + repeated metapb.Peer pending_peers = 6; + // Bytes read/written during this period. + uint64 bytes_written = 7; + uint64 bytes_read = 8; + // Keys read/written during this period. + uint64 keys_written = 9; + uint64 keys_read = 10; + // Approximate region size. + uint64 approximate_size = 11; + // Approximate number of keys. + uint64 approximate_keys = 12; + // QueryStats reported write query stats, and there are read query stats in store heartbeat + pdpb.QueryStats query_stats = 13; + // Actually reported time interval + pdpb.TimeInterval interval = 14; + // BucketMeta is the bucket version and keys of this region if TiKV enabled the bucket feature + metapb.BucketMeta bucket_meta = 15; } message RegionHeartbeatResponse { - ResponseHeader header = 1; - - // ID of the region - uint64 region_id = 2; - metapb.RegionEpoch region_epoch = 3; - // Leader of the region at the moment of the corresponding request was made. - metapb.Peer target_peer = 4; - // Notice, Pd only allows handling reported epoch >= current pd's. - // Leader peer reports region status with RegionHeartbeatRequest - // to pd regularly, pd will determine whether this region - // should do ChangePeer or not. - // E,g, max peer number is 3, region A, first only peer 1 in A. - // 1. Pd region state -> Peers (1), ConfVer (1). - // 2. Leader peer 1 reports region state to pd, pd finds the - // peer number is < 3, so first changes its current region - // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2. - // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2), - // pd updates its state -> Peers (1, 2), ConfVer (2). - // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange - // finished, pd stills responses ChangePeer Adding 2, of course, we must - // guarantee the second ChangePeer can't be applied in TiKV. - pdpb.ChangePeer change_peer = 5; - // Pd can return transfer_leader to let TiKV does leader transfer itself. - pdpb.TransferLeader transfer_leader = 6; - pdpb.Merge merge = 7; - // PD sends split_region to let TiKV split a region into two regions. - pdpb.SplitRegion split_region = 8; - // Multiple change peer operations atomically. - // Note: PD can use both ChangePeer and ChangePeerV2 at the same time - // (not in the same RegionHeartbeatResponse). - // Now, PD use ChangePeerV2 in following scenarios: - // 1. replacing peers - // 2. demoting voter directly - pdpb.ChangePeerV2 change_peer_v2 = 9; - pdpb.BatchSwitchWitness switch_witnesses = 10; + ResponseHeader header = 1; + + // ID of the region + uint64 region_id = 2; + metapb.RegionEpoch region_epoch = 3; + // Leader of the region at the moment of the corresponding request was made. + metapb.Peer target_peer = 4; + // Notice, Pd only allows handling reported epoch >= current pd's. + // Leader peer reports region status with RegionHeartbeatRequest + // to pd regularly, pd will determine whether this region + // should do ChangePeer or not. + // E,g, max peer number is 3, region A, first only peer 1 in A. + // 1. Pd region state -> Peers (1), ConfVer (1). + // 2. Leader peer 1 reports region state to pd, pd finds the + // peer number is < 3, so first changes its current region + // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2. + // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2), + // pd updates its state -> Peers (1, 2), ConfVer (2). + // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange + // finished, pd stills responses ChangePeer Adding 2, of course, we must + // guarantee the second ChangePeer can't be applied in TiKV. + pdpb.ChangePeer change_peer = 5; + // Pd can return transfer_leader to let TiKV does leader transfer itself. + pdpb.TransferLeader transfer_leader = 6; + pdpb.Merge merge = 7; + // PD sends split_region to let TiKV split a region into two regions. + pdpb.SplitRegion split_region = 8; + // Multiple change peer operations atomically. + // Note: PD can use both ChangePeer and ChangePeerV2 at the same time + // (not in the same RegionHeartbeatResponse). + // Now, PD use ChangePeerV2 in following scenarios: + // 1. replacing peers + // 2. demoting voter directly + pdpb.ChangePeerV2 change_peer_v2 = 9; + pdpb.BatchSwitchWitness switch_witnesses = 10; + pdpb.ChangeSplit change_split = 11; } message ScatterRegionsRequest { - RequestHeader header = 1; - // If group is defined, the regions with the same group would be scattered as a whole group. - // If not defined, the regions would be scattered in a cluster level. - string group = 2; + RequestHeader header = 1; + // If group is defined, the regions with the same group would be scattered as a whole group. + // If not defined, the regions would be scattered in a cluster level. + string group = 2; - // If regions_id is defined, the region_id would be ignored. - repeated uint64 regions_id = 3; - uint64 retry_limit = 4; - bool skip_store_limit = 5; + // If regions_id is defined, the region_id would be ignored. + repeated uint64 regions_id = 3; + uint64 retry_limit = 4; + bool skip_store_limit = 5; } message ScatterRegionsResponse { - ResponseHeader header = 1; - uint64 finished_percentage = 2; + ResponseHeader header = 1; + uint64 finished_percentage = 2; + repeated uint64 failed_regions_id = 3; } message SplitRegionsRequest { - RequestHeader header = 1; - repeated bytes split_keys = 2; - uint64 retry_limit = 3; + RequestHeader header = 1; + repeated bytes split_keys = 2; + uint64 retry_limit = 3; } message SplitRegionsResponse { - ResponseHeader header = 1; - uint64 finished_percentage = 2; - repeated uint64 regions_id = 3; + ResponseHeader header = 1; + uint64 finished_percentage = 2; + repeated uint64 regions_id = 3; } message GetOperatorRequest { - RequestHeader header = 1; - uint64 region_id = 2; + RequestHeader header = 1; + uint64 region_id = 2; } message GetOperatorResponse { - ResponseHeader header = 1; - uint64 region_id = 2; - bytes desc = 3; - pdpb.OperatorStatus status = 4; - bytes kind = 5; + ResponseHeader header = 1; + uint64 region_id = 2; + bytes desc = 3; + pdpb.OperatorStatus status = 4; + bytes kind = 5; } message AskBatchSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Region region = 2; - uint32 split_count = 3; + metapb.Region region = 2; + uint32 split_count = 3; + pdpb.SplitReason reason = 4; } message AskBatchSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; + + repeated pdpb.SplitID ids = 2; +} + +message RegionBucketsRequest { + RequestHeader header = 1; + + metapb.RegionEpoch region_epoch = 2; + metapb.Buckets buckets = 3; +} - repeated pdpb.SplitID ids = 2; +message RegionBucketsResponse { + ResponseHeader header = 1; } diff --git a/proto/tikvpb.proto b/proto/tikvpb.proto index f48e9d14..4d1986ba 100644 --- a/proto/tikvpb.proto +++ b/proto/tikvpb.proto @@ -2,258 +2,304 @@ syntax = "proto3"; package tikvpb; import "coprocessor.proto"; +import "disaggregated.proto"; +import "gogoproto/gogo.proto"; import "kvrpcpb.proto"; import "mpp.proto"; import "raft_serverpb.proto"; -import "disaggregated.proto"; - -import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // Key/value store API for TiKV. service Tikv { - // Commands using a transactional interface. - rpc KvGet(kvrpcpb.GetRequest) returns (kvrpcpb.GetResponse) {} - rpc KvScan(kvrpcpb.ScanRequest) returns (kvrpcpb.ScanResponse) {} - rpc KvPrewrite(kvrpcpb.PrewriteRequest) returns (kvrpcpb.PrewriteResponse) {} - rpc KvPessimisticLock(kvrpcpb.PessimisticLockRequest) returns (kvrpcpb.PessimisticLockResponse) {} - rpc KVPessimisticRollback(kvrpcpb.PessimisticRollbackRequest) returns (kvrpcpb.PessimisticRollbackResponse) {} - rpc KvTxnHeartBeat(kvrpcpb.TxnHeartBeatRequest) returns (kvrpcpb.TxnHeartBeatResponse) {} - rpc KvCheckTxnStatus(kvrpcpb.CheckTxnStatusRequest) returns (kvrpcpb.CheckTxnStatusResponse) {} - rpc KvCheckSecondaryLocks(kvrpcpb.CheckSecondaryLocksRequest) returns (kvrpcpb.CheckSecondaryLocksResponse) {} - rpc KvCommit(kvrpcpb.CommitRequest) returns (kvrpcpb.CommitResponse) {} - rpc KvImport(kvrpcpb.ImportRequest) returns (kvrpcpb.ImportResponse) {} - rpc KvCleanup(kvrpcpb.CleanupRequest) returns (kvrpcpb.CleanupResponse) {} - rpc KvBatchGet(kvrpcpb.BatchGetRequest) returns (kvrpcpb.BatchGetResponse) {} - rpc KvBatchRollback(kvrpcpb.BatchRollbackRequest) returns (kvrpcpb.BatchRollbackResponse) {} - rpc KvScanLock(kvrpcpb.ScanLockRequest) returns (kvrpcpb.ScanLockResponse) {} - rpc KvResolveLock(kvrpcpb.ResolveLockRequest) returns (kvrpcpb.ResolveLockResponse) {} - rpc KvGC(kvrpcpb.GCRequest) returns (kvrpcpb.GCResponse) {} - rpc KvDeleteRange(kvrpcpb.DeleteRangeRequest) returns (kvrpcpb.DeleteRangeResponse) {} - rpc KvPrepareFlashbackToVersion(kvrpcpb.PrepareFlashbackToVersionRequest) returns (kvrpcpb.PrepareFlashbackToVersionResponse) {} - rpc KvFlashbackToVersion(kvrpcpb.FlashbackToVersionRequest) returns (kvrpcpb.FlashbackToVersionResponse) {} - - // Raw commands; no transaction support. - rpc RawGet(kvrpcpb.RawGetRequest) returns (kvrpcpb.RawGetResponse) {} - rpc RawBatchGet(kvrpcpb.RawBatchGetRequest) returns (kvrpcpb.RawBatchGetResponse) {} - rpc RawPut(kvrpcpb.RawPutRequest) returns (kvrpcpb.RawPutResponse) {} - rpc RawBatchPut(kvrpcpb.RawBatchPutRequest) returns (kvrpcpb.RawBatchPutResponse) {} - rpc RawDelete(kvrpcpb.RawDeleteRequest) returns (kvrpcpb.RawDeleteResponse) {} - rpc RawBatchDelete(kvrpcpb.RawBatchDeleteRequest) returns (kvrpcpb.RawBatchDeleteResponse) {} - rpc RawScan(kvrpcpb.RawScanRequest) returns (kvrpcpb.RawScanResponse) {} - rpc RawDeleteRange(kvrpcpb.RawDeleteRangeRequest) returns (kvrpcpb.RawDeleteRangeResponse) {} - rpc RawBatchScan(kvrpcpb.RawBatchScanRequest) returns (kvrpcpb.RawBatchScanResponse) {} - // Get TTL of the key. Returns 0 if TTL is not set for the key. - rpc RawGetKeyTTL(kvrpcpb.RawGetKeyTTLRequest) returns (kvrpcpb.RawGetKeyTTLResponse) {} - // Compare if the value in database equals to `RawCASRequest.previous_value` before putting the new value. If not, this request will have no effect and the value in the database will be returned. - rpc RawCompareAndSwap(kvrpcpb.RawCASRequest) returns (kvrpcpb.RawCASResponse) {} - rpc RawChecksum(kvrpcpb.RawChecksumRequest) returns (kvrpcpb.RawChecksumResponse) {} - - // Store commands (sent to a each TiKV node in a cluster, rather than a certain region). - rpc UnsafeDestroyRange(kvrpcpb.UnsafeDestroyRangeRequest) returns (kvrpcpb.UnsafeDestroyRangeResponse) {} - rpc RegisterLockObserver(kvrpcpb.RegisterLockObserverRequest) returns (kvrpcpb.RegisterLockObserverResponse) {} - rpc CheckLockObserver(kvrpcpb.CheckLockObserverRequest) returns (kvrpcpb.CheckLockObserverResponse) {} - rpc RemoveLockObserver(kvrpcpb.RemoveLockObserverRequest) returns (kvrpcpb.RemoveLockObserverResponse) {} - rpc PhysicalScanLock(kvrpcpb.PhysicalScanLockRequest) returns (kvrpcpb.PhysicalScanLockResponse) {} - - // Commands for executing SQL in the TiKV coprocessor (i.e., 'pushed down' to TiKV rather than - // executed in TiDB). - rpc Coprocessor(coprocessor.Request) returns (coprocessor.Response) {} - rpc CoprocessorStream(coprocessor.Request) returns (stream coprocessor.Response) {} - rpc BatchCoprocessor(coprocessor.BatchRequest) returns (stream coprocessor.BatchResponse) {} - - // Command for executing custom user requests in TiKV coprocessor_v2. - rpc RawCoprocessor(kvrpcpb.RawCoprocessorRequest) returns (kvrpcpb.RawCoprocessorResponse) {} - - // Raft commands (sent between TiKV nodes). - rpc Raft(stream raft_serverpb.RaftMessage) returns (raft_serverpb.Done) {} - rpc BatchRaft(stream BatchRaftMessage) returns (raft_serverpb.Done) {} - rpc Snapshot(stream raft_serverpb.SnapshotChunk) returns (raft_serverpb.Done) {} - rpc TabletSnapshot(stream raft_serverpb.TabletSnapshotRequest) returns (stream raft_serverpb.TabletSnapshotResponse) {} - - // Sent from PD or TiDB to a TiKV node. - rpc SplitRegion (kvrpcpb.SplitRegionRequest) returns (kvrpcpb.SplitRegionResponse) {} - // Sent from TiFlash or TiKV to a TiKV node. - rpc ReadIndex(kvrpcpb.ReadIndexRequest) returns (kvrpcpb.ReadIndexResponse) {} - - // Commands for debugging transactions. - rpc MvccGetByKey(kvrpcpb.MvccGetByKeyRequest) returns (kvrpcpb.MvccGetByKeyResponse) {} - rpc MvccGetByStartTs(kvrpcpb.MvccGetByStartTsRequest) returns (kvrpcpb.MvccGetByStartTsResponse) {} - - // Batched commands. - rpc BatchCommands(stream BatchCommandsRequest) returns (stream BatchCommandsResponse) {} - - // These are for mpp execution. - rpc DispatchMPPTask(mpp.DispatchTaskRequest) returns (mpp.DispatchTaskResponse) {} - rpc CancelMPPTask(mpp.CancelTaskRequest) returns (mpp.CancelTaskResponse) {} - rpc EstablishMPPConnection(mpp.EstablishMPPConnectionRequest) returns (stream mpp.MPPDataPacket) {} - rpc IsAlive(mpp.IsAliveRequest) returns (mpp.IsAliveResponse) {} - rpc ReportMPPTaskStatus(mpp.ReportTaskStatusRequest) returns (mpp.ReportTaskStatusResponse) {} - - /// CheckLeader sends all information (includes region term and epoch) to other stores. - /// Once a store receives a request, it checks term and epoch for each region, and sends the regions whose - /// term and epoch match with local information in the store. - /// After the client collected all responses from all stores, it checks if got a quorum of responses from - /// other stores for every region, and decides to advance resolved ts from these regions. - rpc CheckLeader(kvrpcpb.CheckLeaderRequest) returns (kvrpcpb.CheckLeaderResponse); - - /// Get the minimal `safe_ts` from regions at the store - rpc GetStoreSafeTS(kvrpcpb.StoreSafeTSRequest) returns (kvrpcpb.StoreSafeTSResponse); - - /// Get the information about lock waiting from TiKV. - rpc GetLockWaitInfo(kvrpcpb.GetLockWaitInfoRequest) returns (kvrpcpb.GetLockWaitInfoResponse); - - /// Compact a specified key range. This request is not restricted to raft leaders and will not be replicated. - /// It only compacts data on this node. - /// TODO: Currently this RPC is designed to be only compatible with TiFlash. - /// Shall be move out in https://github.com/pingcap/kvproto/issues/912 - rpc Compact(kvrpcpb.CompactRequest) returns (kvrpcpb.CompactResponse); - /// Get the information about history lock waiting from TiKV. - rpc GetLockWaitHistory(kvrpcpb.GetLockWaitHistoryRequest) returns (kvrpcpb.GetLockWaitHistoryResponse); - - /// Get system table from TiFlash - rpc GetTiFlashSystemTable(kvrpcpb.TiFlashSystemTableRequest) returns (kvrpcpb.TiFlashSystemTableResponse) {} - - // These are for TiFlash disaggregated architecture - /// Try to lock a S3 object, atomically - rpc tryAddLock(disaggregated.TryAddLockRequest) returns (disaggregated.TryAddLockResponse) {}; - /// Try to delete a S3 object, atomically - rpc tryMarkDelete(disaggregated.TryMarkDeleteRequest) returns (disaggregated.TryMarkDeleteResponse) {}; - /// Build the disaggregated task on TiFlash write node - rpc EstablishDisaggTask(disaggregated.EstablishDisaggTaskRequest) returns (disaggregated.EstablishDisaggTaskResponse) {} - /// Cancel the disaggregated task on TiFlash write node - rpc CancelDisaggTask(disaggregated.CancelDisaggTaskRequest) returns (disaggregated.CancelDisaggTaskResponse) {} - /// Exchange page data between TiFlash write node and compute node - rpc FetchDisaggPages(disaggregated.FetchDisaggPagesRequest) returns (stream disaggregated.PagesPacket) {} - /// Compute node get configuration from Write node - rpc GetDisaggConfig(disaggregated.GetDisaggConfigRequest) returns (disaggregated.GetDisaggConfigResponse) {} + // Commands using a transactional interface. + rpc KvGet(kvrpcpb.GetRequest) returns (kvrpcpb.GetResponse) {} + rpc KvScan(kvrpcpb.ScanRequest) returns (kvrpcpb.ScanResponse) {} + rpc KvPrewrite(kvrpcpb.PrewriteRequest) returns (kvrpcpb.PrewriteResponse) {} + rpc KvPessimisticLock(kvrpcpb.PessimisticLockRequest) returns (kvrpcpb.PessimisticLockResponse) {} + rpc KVPessimisticRollback(kvrpcpb.PessimisticRollbackRequest) returns (kvrpcpb.PessimisticRollbackResponse) {} + rpc KvTxnHeartBeat(kvrpcpb.TxnHeartBeatRequest) returns (kvrpcpb.TxnHeartBeatResponse) {} + rpc KvCheckTxnStatus(kvrpcpb.CheckTxnStatusRequest) returns (kvrpcpb.CheckTxnStatusResponse) {} + rpc KvCheckSecondaryLocks(kvrpcpb.CheckSecondaryLocksRequest) returns (kvrpcpb.CheckSecondaryLocksResponse) {} + rpc KvCommit(kvrpcpb.CommitRequest) returns (kvrpcpb.CommitResponse) {} + rpc KvImport(kvrpcpb.ImportRequest) returns (kvrpcpb.ImportResponse) {} + rpc KvCleanup(kvrpcpb.CleanupRequest) returns (kvrpcpb.CleanupResponse) {} + rpc KvBatchGet(kvrpcpb.BatchGetRequest) returns (kvrpcpb.BatchGetResponse) {} + rpc KvBatchRollback(kvrpcpb.BatchRollbackRequest) returns (kvrpcpb.BatchRollbackResponse) {} + rpc KvScanLock(kvrpcpb.ScanLockRequest) returns (kvrpcpb.ScanLockResponse) {} + rpc KvResolveLock(kvrpcpb.ResolveLockRequest) returns (kvrpcpb.ResolveLockResponse) {} + rpc KvGC(kvrpcpb.GCRequest) returns (kvrpcpb.GCResponse) {} + rpc KvDeleteRange(kvrpcpb.DeleteRangeRequest) returns (kvrpcpb.DeleteRangeResponse) {} + rpc KvPrepareFlashbackToVersion(kvrpcpb.PrepareFlashbackToVersionRequest) returns (kvrpcpb.PrepareFlashbackToVersionResponse) {} + rpc KvFlashbackToVersion(kvrpcpb.FlashbackToVersionRequest) returns (kvrpcpb.FlashbackToVersionResponse) {} + rpc KvFlush(kvrpcpb.FlushRequest) returns (kvrpcpb.FlushResponse) {} + rpc KvBufferBatchGet(kvrpcpb.BufferBatchGetRequest) returns (kvrpcpb.BufferBatchGetResponse) {} + + // Raw commands; no transaction support. + rpc RawGet(kvrpcpb.RawGetRequest) returns (kvrpcpb.RawGetResponse) {} + rpc RawBatchGet(kvrpcpb.RawBatchGetRequest) returns (kvrpcpb.RawBatchGetResponse) {} + rpc RawPut(kvrpcpb.RawPutRequest) returns (kvrpcpb.RawPutResponse) {} + rpc RawBatchPut(kvrpcpb.RawBatchPutRequest) returns (kvrpcpb.RawBatchPutResponse) {} + rpc RawDelete(kvrpcpb.RawDeleteRequest) returns (kvrpcpb.RawDeleteResponse) {} + rpc RawBatchDelete(kvrpcpb.RawBatchDeleteRequest) returns (kvrpcpb.RawBatchDeleteResponse) {} + rpc RawScan(kvrpcpb.RawScanRequest) returns (kvrpcpb.RawScanResponse) {} + rpc RawDeleteRange(kvrpcpb.RawDeleteRangeRequest) returns (kvrpcpb.RawDeleteRangeResponse) {} + rpc RawBatchScan(kvrpcpb.RawBatchScanRequest) returns (kvrpcpb.RawBatchScanResponse) {} + // Get TTL of the key. Returns 0 if TTL is not set for the key. + rpc RawGetKeyTTL(kvrpcpb.RawGetKeyTTLRequest) returns (kvrpcpb.RawGetKeyTTLResponse) {} + // Compare if the value in database equals to `RawCASRequest.previous_value` before putting the new value. If not, this request will have no effect and the value in the database will be returned. + rpc RawCompareAndSwap(kvrpcpb.RawCASRequest) returns (kvrpcpb.RawCASResponse) {} + rpc RawChecksum(kvrpcpb.RawChecksumRequest) returns (kvrpcpb.RawChecksumResponse) {} + + // Store commands (sent to a each TiKV node in a cluster, rather than a certain region). + rpc UnsafeDestroyRange(kvrpcpb.UnsafeDestroyRangeRequest) returns (kvrpcpb.UnsafeDestroyRangeResponse) {} + rpc RegisterLockObserver(kvrpcpb.RegisterLockObserverRequest) returns (kvrpcpb.RegisterLockObserverResponse) {} + rpc CheckLockObserver(kvrpcpb.CheckLockObserverRequest) returns (kvrpcpb.CheckLockObserverResponse) {} + rpc RemoveLockObserver(kvrpcpb.RemoveLockObserverRequest) returns (kvrpcpb.RemoveLockObserverResponse) {} + rpc PhysicalScanLock(kvrpcpb.PhysicalScanLockRequest) returns (kvrpcpb.PhysicalScanLockResponse) {} + + // Commands for executing SQL in the TiKV coprocessor (i.e., 'pushed down' to TiKV rather than + // executed in TiDB). + rpc Coprocessor(coprocessor.Request) returns (coprocessor.Response) {} + rpc CoprocessorStream(coprocessor.Request) returns (stream coprocessor.Response) {} + rpc BatchCoprocessor(coprocessor.BatchRequest) returns (stream coprocessor.BatchResponse) {} + + // Command send by remote coprocessor to TiKV for executing coprocessor request. + rpc DelegateCoprocessor(coprocessor.DelegateRequest) returns (coprocessor.DelegateResponse) {} + + // Command for executing custom user requests in TiKV coprocessor_v2. + rpc RawCoprocessor(kvrpcpb.RawCoprocessorRequest) returns (kvrpcpb.RawCoprocessorResponse) {} + + // Raft commands (sent between TiKV nodes). + rpc Raft(stream raft_serverpb.RaftMessage) returns (raft_serverpb.Done) {} + rpc BatchRaft(stream BatchRaftMessage) returns (raft_serverpb.Done) {} + rpc Snapshot(stream raft_serverpb.SnapshotChunk) returns (raft_serverpb.Done) {} + rpc TabletSnapshot(stream raft_serverpb.TabletSnapshotRequest) returns (stream raft_serverpb.TabletSnapshotResponse) {} + + // Sent from PD or TiDB to a TiKV node. + rpc SplitRegion(kvrpcpb.SplitRegionRequest) returns (kvrpcpb.SplitRegionResponse) {} + // Sent from TiFlash or TiKV to a TiKV node. + rpc ReadIndex(kvrpcpb.ReadIndexRequest) returns (kvrpcpb.ReadIndexResponse) {} + + // Commands for debugging transactions. + rpc MvccGetByKey(kvrpcpb.MvccGetByKeyRequest) returns (kvrpcpb.MvccGetByKeyResponse) {} + rpc MvccGetByStartTs(kvrpcpb.MvccGetByStartTsRequest) returns (kvrpcpb.MvccGetByStartTsResponse) {} + + // Batched commands. + rpc BatchCommands(stream BatchCommandsRequest) returns (stream BatchCommandsResponse) {} + + // These are for mpp execution. + rpc DispatchMPPTask(mpp.DispatchTaskRequest) returns (mpp.DispatchTaskResponse) {} + rpc CancelMPPTask(mpp.CancelTaskRequest) returns (mpp.CancelTaskResponse) {} + rpc EstablishMPPConnection(mpp.EstablishMPPConnectionRequest) returns (stream mpp.MPPDataPacket) {} + rpc IsAlive(mpp.IsAliveRequest) returns (mpp.IsAliveResponse) {} + rpc ReportMPPTaskStatus(mpp.ReportTaskStatusRequest) returns (mpp.ReportTaskStatusResponse) {} + + /// CheckLeader sends all information (includes region term and epoch) to other stores. + /// Once a store receives a request, it checks term and epoch for each region, and sends the regions whose + /// term and epoch match with local information in the store. + /// After the client collected all responses from all stores, it checks if got a quorum of responses from + /// other stores for every region, and decides to advance resolved ts from these regions. + rpc CheckLeader(kvrpcpb.CheckLeaderRequest) returns (kvrpcpb.CheckLeaderResponse); + + /// Get the minimal `safe_ts` from regions at the store + rpc GetStoreSafeTS(kvrpcpb.StoreSafeTSRequest) returns (kvrpcpb.StoreSafeTSResponse); + + /// Get the information about lock waiting from TiKV. + rpc GetLockWaitInfo(kvrpcpb.GetLockWaitInfoRequest) returns (kvrpcpb.GetLockWaitInfoResponse); + + /// Compact a specified key range. This request is not restricted to raft leaders and will not be replicated. + /// It only compacts data on this node. + /// TODO: Currently this RPC is designed to be only compatible with TiFlash. + /// Shall be move out in https://github.com/pingcap/kvproto/issues/912 + rpc Compact(kvrpcpb.CompactRequest) returns (kvrpcpb.CompactResponse); + /// Get the information about history lock waiting from TiKV. + rpc GetLockWaitHistory(kvrpcpb.GetLockWaitHistoryRequest) returns (kvrpcpb.GetLockWaitHistoryResponse); + + /// Get system table from TiFlash + rpc GetTiFlashSystemTable(kvrpcpb.TiFlashSystemTableRequest) returns (kvrpcpb.TiFlashSystemTableResponse) {} + /// Get estimate tici count from TiFlash + rpc GetEstimateTiCICount(coprocessor.TiCIEstimateCountRequest) returns (coprocessor.TiCIEstimateCountResponse) {} + + // These are for TiFlash disaggregated architecture + /// Try to lock a S3 object, atomically + rpc tryAddLock(disaggregated.TryAddLockRequest) returns (disaggregated.TryAddLockResponse) {} + /// Try to delete a S3 object, atomically + rpc tryMarkDelete(disaggregated.TryMarkDeleteRequest) returns (disaggregated.TryMarkDeleteResponse) {} + /// Build the disaggregated task on TiFlash write node + rpc EstablishDisaggTask(disaggregated.EstablishDisaggTaskRequest) returns (disaggregated.EstablishDisaggTaskResponse) {} + /// Cancel the disaggregated task on TiFlash write node + rpc CancelDisaggTask(disaggregated.CancelDisaggTaskRequest) returns (disaggregated.CancelDisaggTaskResponse) {} + /// Exchange page data between TiFlash write node and compute node + rpc FetchDisaggPages(disaggregated.FetchDisaggPagesRequest) returns (stream disaggregated.PagesPacket) {} + /// Compute node get configuration from Write node + rpc GetDisaggConfig(disaggregated.GetDisaggConfigRequest) returns (disaggregated.GetDisaggConfigResponse) {} + + /// Get health feedback info from the TiKV node. + rpc GetHealthFeedback(kvrpcpb.GetHealthFeedbackRequest) returns (kvrpcpb.GetHealthFeedbackResponse) {} + /// Broadcast the transaction status to all TiKV nodes + rpc BroadcastTxnStatus(kvrpcpb.BroadcastTxnStatusRequest) returns (kvrpcpb.BroadcastTxnStatusResponse) {} +} + +// VersionedKv provides versioned coprocessor APIs for TiCI lookup. +// +// Invariants: +// - For `VersionedCoprocessor`, callers should fill `coprocessor.Request.versioned_ranges` +// (each `VersionedKeyRange.range` must be a point range) and keep `coprocessor.Request.ranges` empty. +service VersionedKv { + rpc VersionedCoprocessor(coprocessor.Request) returns (coprocessor.Response) {} } message BatchCommandsRequest { - repeated Request requests = 1; - repeated uint64 request_ids = 2; - - message Request { - oneof cmd { - kvrpcpb.GetRequest Get = 1; - kvrpcpb.ScanRequest Scan = 2; - kvrpcpb.PrewriteRequest Prewrite = 3; - kvrpcpb.CommitRequest Commit = 4; - kvrpcpb.ImportRequest Import = 5; - kvrpcpb.CleanupRequest Cleanup = 6; - kvrpcpb.BatchGetRequest BatchGet = 7; - kvrpcpb.BatchRollbackRequest BatchRollback = 8; - kvrpcpb.ScanLockRequest ScanLock = 9; - kvrpcpb.ResolveLockRequest ResolveLock = 10; - kvrpcpb.GCRequest GC = 11; - kvrpcpb.DeleteRangeRequest DeleteRange = 12; - - kvrpcpb.RawGetRequest RawGet = 13; - kvrpcpb.RawBatchGetRequest RawBatchGet = 14; - kvrpcpb.RawPutRequest RawPut = 15; - kvrpcpb.RawBatchPutRequest RawBatchPut = 16; - kvrpcpb.RawDeleteRequest RawDelete = 17; - kvrpcpb.RawBatchDeleteRequest RawBatchDelete = 18; - kvrpcpb.RawScanRequest RawScan = 19; - kvrpcpb.RawDeleteRangeRequest RawDeleteRange = 20; - kvrpcpb.RawBatchScanRequest RawBatchScan = 21; - - coprocessor.Request Coprocessor = 22; - kvrpcpb.PessimisticLockRequest PessimisticLock = 23; - kvrpcpb.PessimisticRollbackRequest PessimisticRollback = 24; - - kvrpcpb.CheckTxnStatusRequest CheckTxnStatus = 25; - kvrpcpb.TxnHeartBeatRequest TxnHeartBeat = 26; - - kvrpcpb.CheckSecondaryLocksRequest CheckSecondaryLocks = 33; - - kvrpcpb.RawCoprocessorRequest RawCoprocessor = 34; - - kvrpcpb.FlashbackToVersionRequest FlashbackToVersion = 35; - kvrpcpb.PrepareFlashbackToVersionRequest PrepareFlashbackToVersion = 36; - - // For some test cases. - BatchCommandsEmptyRequest Empty = 255; - } - - // Reserved for deprecated VerKV - reserved 27 to 32; + repeated Request requests = 1; + repeated uint64 request_ids = 2; + // Unix epoch timestamp in nanoseconds recorded by the client when this + // batch request is sent to TiKV. + uint64 client_send_time_ns = 3; + + message Request { + oneof cmd { + kvrpcpb.GetRequest Get = 1; + kvrpcpb.ScanRequest Scan = 2; + kvrpcpb.PrewriteRequest Prewrite = 3; + kvrpcpb.CommitRequest Commit = 4; + kvrpcpb.ImportRequest Import = 5; + kvrpcpb.CleanupRequest Cleanup = 6; + kvrpcpb.BatchGetRequest BatchGet = 7; + kvrpcpb.BatchRollbackRequest BatchRollback = 8; + kvrpcpb.ScanLockRequest ScanLock = 9; + kvrpcpb.ResolveLockRequest ResolveLock = 10; + kvrpcpb.GCRequest GC = 11; + kvrpcpb.DeleteRangeRequest DeleteRange = 12; + + kvrpcpb.RawGetRequest RawGet = 13; + kvrpcpb.RawBatchGetRequest RawBatchGet = 14; + kvrpcpb.RawPutRequest RawPut = 15; + kvrpcpb.RawBatchPutRequest RawBatchPut = 16; + kvrpcpb.RawDeleteRequest RawDelete = 17; + kvrpcpb.RawBatchDeleteRequest RawBatchDelete = 18; + kvrpcpb.RawScanRequest RawScan = 19; + kvrpcpb.RawDeleteRangeRequest RawDeleteRange = 20; + kvrpcpb.RawBatchScanRequest RawBatchScan = 21; + + coprocessor.Request Coprocessor = 22; + kvrpcpb.PessimisticLockRequest PessimisticLock = 23; + kvrpcpb.PessimisticRollbackRequest PessimisticRollback = 24; + + kvrpcpb.CheckTxnStatusRequest CheckTxnStatus = 25; + kvrpcpb.TxnHeartBeatRequest TxnHeartBeat = 26; + + kvrpcpb.CheckSecondaryLocksRequest CheckSecondaryLocks = 33; + + kvrpcpb.RawCoprocessorRequest RawCoprocessor = 34; + + kvrpcpb.FlashbackToVersionRequest FlashbackToVersion = 35; + kvrpcpb.PrepareFlashbackToVersionRequest PrepareFlashbackToVersion = 36; + kvrpcpb.FlushRequest Flush = 37; + kvrpcpb.BufferBatchGetRequest BufferBatchGet = 38; + + kvrpcpb.GetHealthFeedbackRequest GetHealthFeedback = 39; + + kvrpcpb.BroadcastTxnStatusRequest BroadcastTxnStatus = 40; + // For some test cases. + BatchCommandsEmptyRequest Empty = 255; } + + // Reserved for deprecated VerKV + reserved 27 to 32; + + // Reserved for a downstream fork + reserved 1000 to 1199; + } } message BatchCommandsResponse { - repeated Response responses = 1; - repeated uint64 request_ids = 2; - // 280 means TiKV gRPC cpu usage is 280%. - uint64 transport_layer_load = 3; - - message Response { - oneof cmd { - kvrpcpb.GetResponse Get = 1; - kvrpcpb.ScanResponse Scan = 2; - kvrpcpb.PrewriteResponse Prewrite = 3; - kvrpcpb.CommitResponse Commit = 4; - kvrpcpb.ImportResponse Import = 5; - kvrpcpb.CleanupResponse Cleanup = 6; - kvrpcpb.BatchGetResponse BatchGet = 7; - kvrpcpb.BatchRollbackResponse BatchRollback = 8; - kvrpcpb.ScanLockResponse ScanLock = 9; - kvrpcpb.ResolveLockResponse ResolveLock = 10; - kvrpcpb.GCResponse GC = 11; - kvrpcpb.DeleteRangeResponse DeleteRange = 12; - - kvrpcpb.RawGetResponse RawGet = 13; - kvrpcpb.RawBatchGetResponse RawBatchGet = 14; - kvrpcpb.RawPutResponse RawPut = 15; - kvrpcpb.RawBatchPutResponse RawBatchPut = 16; - kvrpcpb.RawDeleteResponse RawDelete = 17; - kvrpcpb.RawBatchDeleteResponse RawBatchDelete = 18; - kvrpcpb.RawScanResponse RawScan = 19; - kvrpcpb.RawDeleteRangeResponse RawDeleteRange = 20; - kvrpcpb.RawBatchScanResponse RawBatchScan = 21; - - coprocessor.Response Coprocessor = 22; - kvrpcpb.PessimisticLockResponse PessimisticLock = 23; - kvrpcpb.PessimisticRollbackResponse PessimisticRollback = 24; - - kvrpcpb.CheckTxnStatusResponse CheckTxnStatus = 25; - kvrpcpb.TxnHeartBeatResponse TxnHeartBeat = 26; - - kvrpcpb.CheckSecondaryLocksResponse CheckSecondaryLocks = 33; - - kvrpcpb.RawCoprocessorResponse RawCoprocessor = 34; - - kvrpcpb.FlashbackToVersionResponse FlashbackToVersion = 35; - kvrpcpb.PrepareFlashbackToVersionResponse PrepareFlashbackToVersion = 36; - - // For some test cases. - BatchCommandsEmptyResponse Empty = 255; - } - - // Reserved for deprecated VerKV - reserved 27 to 32; + repeated Response responses = 1; + repeated uint64 request_ids = 2; + // 280 means TiKV gRPC cpu usage is 280%. + uint64 transport_layer_load = 3; + kvrpcpb.HealthFeedback health_feedback = 4; + // Unix epoch timestamp in nanoseconds recorded by TiKV when this batch + // response is ready to be sent back to the client. + uint64 tikv_send_time_ns = 5; + + message Response { + oneof cmd { + kvrpcpb.GetResponse Get = 1; + kvrpcpb.ScanResponse Scan = 2; + kvrpcpb.PrewriteResponse Prewrite = 3; + kvrpcpb.CommitResponse Commit = 4; + kvrpcpb.ImportResponse Import = 5; + kvrpcpb.CleanupResponse Cleanup = 6; + kvrpcpb.BatchGetResponse BatchGet = 7; + kvrpcpb.BatchRollbackResponse BatchRollback = 8; + kvrpcpb.ScanLockResponse ScanLock = 9; + kvrpcpb.ResolveLockResponse ResolveLock = 10; + kvrpcpb.GCResponse GC = 11; + kvrpcpb.DeleteRangeResponse DeleteRange = 12; + + kvrpcpb.RawGetResponse RawGet = 13; + kvrpcpb.RawBatchGetResponse RawBatchGet = 14; + kvrpcpb.RawPutResponse RawPut = 15; + kvrpcpb.RawBatchPutResponse RawBatchPut = 16; + kvrpcpb.RawDeleteResponse RawDelete = 17; + kvrpcpb.RawBatchDeleteResponse RawBatchDelete = 18; + kvrpcpb.RawScanResponse RawScan = 19; + kvrpcpb.RawDeleteRangeResponse RawDeleteRange = 20; + kvrpcpb.RawBatchScanResponse RawBatchScan = 21; + + coprocessor.Response Coprocessor = 22; + kvrpcpb.PessimisticLockResponse PessimisticLock = 23; + kvrpcpb.PessimisticRollbackResponse PessimisticRollback = 24; + + kvrpcpb.CheckTxnStatusResponse CheckTxnStatus = 25; + kvrpcpb.TxnHeartBeatResponse TxnHeartBeat = 26; + + kvrpcpb.CheckSecondaryLocksResponse CheckSecondaryLocks = 33; + + kvrpcpb.RawCoprocessorResponse RawCoprocessor = 34; + + kvrpcpb.FlashbackToVersionResponse FlashbackToVersion = 35; + kvrpcpb.PrepareFlashbackToVersionResponse PrepareFlashbackToVersion = 36; + + kvrpcpb.FlushResponse Flush = 37; + kvrpcpb.BufferBatchGetResponse BufferBatchGet = 38; + + kvrpcpb.GetHealthFeedbackResponse GetHealthFeedback = 39; + + kvrpcpb.BroadcastTxnStatusResponse BroadcastTxnStatus = 40; + + // For some test cases. + BatchCommandsEmptyResponse Empty = 255; } + + // Reserved for deprecated VerKV + reserved 27 to 32; + } } message BatchRaftMessage { - repeated raft_serverpb.RaftMessage msgs = 1; + repeated raft_serverpb.RaftMessage msgs = 1; + // Used for measure the send duration. + uint64 last_observed_time = 13; } message BatchCommandsEmptyRequest { - // ID of the test request. - uint64 test_id = 1; - // TiKV needs to delay at least such a time to response the client. - uint64 delay_time = 2; + // ID of the test request. + uint64 test_id = 1; + // TiKV needs to delay at least such a time to response the client. + uint64 delay_time = 2; } message BatchCommandsEmptyResponse { - // ID of the test request. - uint64 test_id = 1; + // ID of the test request. + uint64 test_id = 1; } diff --git a/proto/tracepb.proto b/proto/tracepb.proto index c229454c..9065cb4d 100644 --- a/proto/tracepb.proto +++ b/proto/tracepb.proto @@ -1,60 +1,70 @@ syntax = "proto3"; package tracepb; +import "gogoproto/gogo.proto"; +import "rustproto.proto"; + option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; service TraceRecordPubSub { - // Subscribe the Trace records generated on this service. The service will periodically (e.g. per minute) - // publishes Trace records to clients via gRPC stream. - rpc Subscribe(TraceRecordRequest) returns (stream TraceRecord) {} + // Subscribe the Trace records generated on this service. The service will periodically (e.g. per minute) + // publishes Trace records to clients via gRPC stream. + rpc Subscribe(TraceRecordRequest) returns (stream TraceRecord) {} } message TraceRecordRequest {} message TraceRecord { - oneof record_oneof { - Report report = 1; - NotifyCollect notify_collect = 2; - } + oneof record_oneof { + Report report = 1; + NotifyCollect notify_collect = 2; + } } message RemoteParentSpan { - // A unique id to identify the request. It's usually a UUID. - uint64 trace_id = 1; - // The span of remote caller that is awaiting the request. - uint64 span_id = 2; + // A unique id to identify the request. It's usually a UUID. + uint64 trace_id = 1; + // The span of remote caller that is awaiting the request. + uint64 span_id = 2; } // The context of the request to be traced. message TraceContext { - repeated RemoteParentSpan remote_parent_spans = 1; - // Report the trace records only if the duration of handling the request exceeds the threshold. - uint32 duration_threshold_ms = 2; + repeated RemoteParentSpan remote_parent_spans = 1; + // Report the trace records only if the duration of handling the request exceeds the threshold. + uint32 duration_threshold_ms = 2; } // Report the spans collected when handling a request on a service. message Report { - repeated RemoteParentSpan remote_parent_spans = 1; - repeated Span spans = 2; + repeated RemoteParentSpan remote_parent_spans = 1; + repeated Span spans = 2; } // Notify the subscriber to persis the spans of the trace. message NotifyCollect { - uint64 trace_id = 1; + uint64 trace_id = 1; } message Span { - // The unique span id within the spans with the same `trace_id`. - // The most significant 32 bits should be random number generated by each service instance. - uint64 span_id = 1; - uint64 parent_id = 2; - uint64 begin_unix_ns = 3; - uint64 duration_ns = 4; - string event = 5; - repeated Property properties = 6; + // The unique span id within the spans with the same `trace_id`. + // The most significant 32 bits should be random number generated by each service instance. + uint64 span_id = 1; + uint64 parent_id = 2; + uint64 begin_unix_ns = 3; + uint64 duration_ns = 4; + string event = 5; + repeated Property properties = 6; } message Property { - string key = 1; - string value = 2; + string key = 1; + string value = 2; } diff --git a/proto/tsopb.proto b/proto/tsopb.proto index 061bcadc..7a8f6a48 100644 --- a/proto/tsopb.proto +++ b/proto/tsopb.proto @@ -1,126 +1,134 @@ syntax = "proto3"; package tsopb; -import "pdpb.proto"; - import "gogoproto/gogo.proto"; +import "pdpb.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - service TSO { - rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} - // Find the keyspace group that the keyspace belongs to by keyspace id. - rpc FindGroupByKeyspaceID (FindGroupByKeyspaceIDRequest) returns (FindGroupByKeyspaceIDResponse) {} - // Get the minimum timestamp across all keyspace groups served by the TSO server who receives - // and handle the request. If the TSO server/pod is not serving any keyspace group, return - // an empty timestamp, and the client needs to skip the empty timestamps when collecting - // the min timestamp from all TSO servers/pods. - rpc GetMinTS (GetMinTSRequest) returns (GetMinTSResponse) {} + rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} + // Find the keyspace group that the keyspace belongs to by keyspace id. + rpc FindGroupByKeyspaceID(FindGroupByKeyspaceIDRequest) returns (FindGroupByKeyspaceIDResponse) {} + // Get the minimum timestamp across all keyspace groups served by the TSO server who receives + // and handle the request. If the TSO server/pod is not serving any keyspace group, return + // an empty timestamp, and the client needs to skip the empty timestamps when collecting + // the min timestamp from all TSO servers/pods. + rpc GetMinTS(GetMinTSRequest) returns (GetMinTSResponse) {} } message RequestHeader { - // cluster_id is the ID of the cluster which be sent to. - uint64 cluster_id = 1; - // sender_id is the ID of the sender server. - uint64 sender_id = 2; - - // keyspace_id is the unique id of the tenant/keyspace. - uint32 keyspace_id = 3; - // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. - uint32 keyspace_group_id = 4; + // cluster_id is the ID of the cluster which be sent to. + uint64 cluster_id = 1; + // sender_id is the ID of the sender server. + uint64 sender_id = 2; + + // keyspace_id is the unique id of the tenant/keyspace. + uint32 keyspace_id = 3; + // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. + uint32 keyspace_group_id = 4; + // callee_id is the ID of the server which the client expects to receive the request. + // such as tso-0, tso-1, pd-0, pd-1 etc. + // This field is used to check if the request is sent to the expected server. + // If it is not matched, the server will return an error. + string callee_id = 5; } message ResponseHeader { - // cluster_id is the ID of the cluster which sent the response. - uint64 cluster_id = 1; - Error error = 2; - - // keyspace_id is the unique id of the tenant/keyspace as the response receiver. - uint32 keyspace_id = 3; - // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. - uint32 keyspace_group_id = 4; + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; + + // keyspace_id is the unique id of the tenant/keyspace as the response receiver. + uint32 keyspace_id = 3; + // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. + uint32 keyspace_group_id = 4; } enum ErrorType { - OK = 0; - UNKNOWN = 1; - NOT_BOOTSTRAPPED = 2; - ALREADY_BOOTSTRAPPED = 3; - INVALID_VALUE = 4; - CLUSTER_MISMATCHED = 5; + OK = 0; + UNKNOWN = 1; + NOT_BOOTSTRAPPED = 2; + ALREADY_BOOTSTRAPPED = 3; + INVALID_VALUE = 4; + CLUSTER_MISMATCHED = 5; } message Error { - ErrorType type = 1; - string message = 2; + ErrorType type = 1; + string message = 2; } message TsoRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 count = 2; - string dc_location = 3; + uint32 count = 2; + string dc_location = 3; } message TsoResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint32 count = 2; - pdpb.Timestamp timestamp = 3; + uint32 count = 2; + pdpb.Timestamp timestamp = 3; } message Participant { - // name is the unique name of the TSO participant. - string name = 1; - // id is the unique id of the TSO participant. - uint64 id = 2; - // listen_urls is the serivce endpoint list in the url format. - // listen_urls[0] is primary service endpoint. - repeated string listen_urls = 3; + // name is the unique name of the TSO participant. + string name = 1; + // id is the unique id of the TSO participant. + uint64 id = 2; + // listen_urls is the serivce endpoint list in the url format. + // listen_urls[0] is primary service endpoint. + repeated string listen_urls = 3; } message KeyspaceGroupMember { - string address = 1; - bool is_primary = 2; + string address = 1; + bool is_primary = 2; } message SplitState { - uint32 split_source = 1; + uint32 split_source = 1; } message KeyspaceGroup { - uint32 id = 1; - string user_kind = 2; - SplitState split_state = 3; - repeated KeyspaceGroupMember members = 4; + uint32 id = 1; + string user_kind = 2; + SplitState split_state = 3; + repeated KeyspaceGroupMember members = 4; } - + message FindGroupByKeyspaceIDRequest { - RequestHeader header = 1; - uint32 keyspace_id = 2; + RequestHeader header = 1; + uint32 keyspace_id = 2; + uint64 mod_revision = 3; } message FindGroupByKeyspaceIDResponse { - ResponseHeader header = 1; - KeyspaceGroup keyspace_group = 2; + ResponseHeader header = 1; + KeyspaceGroup keyspace_group = 2; + uint64 mod_revision = 3; } message GetMinTSRequest { - RequestHeader header = 1; - string dc_location = 2; + RequestHeader header = 1; + string dc_location = 2; } message GetMinTSResponse { - ResponseHeader header = 1; - pdpb.Timestamp timestamp = 2; - // the count of keyspace group primaries that the TSO server/pod is serving - uint32 keyspace_groups_serving = 3; - // the total count of keyspace groups - uint32 keyspace_groups_total = 4; + ResponseHeader header = 1; + pdpb.Timestamp timestamp = 2; + // the count of keyspace group primaries that the TSO server/pod is serving + uint32 keyspace_groups_serving = 3; + // the total count of keyspace groups + uint32 keyspace_groups_total = 4; } diff --git a/src/generated/backup.rs b/src/generated/backup.rs index a283075f..1d4d331d 100644 --- a/src/generated/backup.rs +++ b/src/generated/backup.rs @@ -1,4 +1,36 @@ // This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrepareSnapshotBackupRequest { + /// The type of the request. + #[prost(enumeration = "PrepareSnapshotBackupRequestType", tag = "1")] + pub ty: i32, + /// The regions related to the request. + /// If the request is "UpdateLease" or "Finish", it should be empty. + #[prost(message, repeated, tag = "2")] + pub regions: ::prost::alloc::vec::Vec, + /// The lease duration for "UpdateLease" request. + #[prost(uint64, tag = "3")] + pub lease_in_seconds: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrepareSnapshotBackupResponse { + /// The type of the event. + #[prost(enumeration = "PrepareSnapshotBackupEventType", tag = "1")] + pub ty: i32, + /// The region bind to the type. + /// For "WaitApplyDone". + #[prost(message, optional, tag = "2")] + pub region: ::core::option::Option, + /// The error of the execution. + #[prost(message, optional, tag = "3")] + pub error: ::core::option::Option, + /// Whether the last lease is valid. + /// For "UpdateLease" and "Finish". + #[prost(bool, tag = "4")] + pub last_lease_is_valid: bool, +} /// The message save the metadata of a backup. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -39,6 +71,10 @@ pub struct BackupMeta { /// An index to files contains Schemas. #[prost(message, optional, tag = "14")] pub schema_index: ::core::option::Option, + /// Backup metadata compatibility version for restore. + /// 0 means the backup was created before this field existed. + #[prost(uint32, tag = "26")] + pub backup_schema_version: u32, /// If in raw kv mode, `start_versions`, `end_versions` and `schemas` will be /// ignored, and the backup data's range is represented by raw_ranges. #[prost(bool, tag = "8")] @@ -96,6 +132,20 @@ pub struct BackupRange { #[prost(message, repeated, tag = "3")] pub files: ::prost::alloc::vec::Vec, } +/// The message saves the physical table checksum +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableMeta { + /// physical id is either table id or partition id + #[prost(int64, tag = "1")] + pub physical_id: i64, + #[prost(uint64, tag = "2")] + pub crc64xor: u64, + #[prost(uint64, tag = "3")] + pub total_kvs: u64, + #[prost(uint64, tag = "4")] + pub total_bytes: u64, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct File { @@ -111,12 +161,16 @@ pub struct File { pub start_version: u64, #[prost(uint64, tag = "6")] pub end_version: u64, + /// recorded the total statistics across multiple tables #[prost(uint64, tag = "7")] pub crc64xor: u64, #[prost(uint64, tag = "8")] pub total_kvs: u64, #[prost(uint64, tag = "9")] pub total_bytes: u64, + /// recorded when the SST kvs are from different tables + #[prost(message, repeated, tag = "13")] + pub table_metas: ::prost::alloc::vec::Vec, #[prost(string, tag = "10")] pub cf: ::prost::alloc::string::String, #[prost(uint64, tag = "11")] @@ -157,6 +211,43 @@ pub struct PlacementPolicy { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct StatsBlock { + /// The physical id which the stats json data belongs to. + #[prost(int64, tag = "1")] + pub physical_id: i64, + /// The marshal data of the part of json_table. Currently it is the partition table. + #[prost(bytes = "vec", tag = "2")] + pub json_table: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StatsFile { + /// A set of files that contains json_tables. + #[prost(message, repeated, tag = "1")] + pub blocks: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StatsFileIndex { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(bytes = "vec", tag = "2")] + pub sha256: ::prost::alloc::vec::Vec, + /// the size of the file content encrypted + #[prost(uint64, tag = "3")] + pub size_enc: u64, + /// the size of the original file content + #[prost(uint64, tag = "4")] + pub size_ori: u64, + /// cipher_iv is used for AES cipher + #[prost(bytes = "vec", tag = "5")] + pub cipher_iv: ::prost::alloc::vec::Vec, + /// If the size of the stats is very small, save the data into the metafile + #[prost(bytes = "vec", tag = "6")] + pub inline_data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Schema { #[prost(bytes = "vec", tag = "1")] pub db: ::prost::alloc::vec::Vec, @@ -174,6 +265,20 @@ pub struct Schema { /// #[prost(bytes = "vec", tag = "7")] pub stats: ::prost::alloc::vec::Vec, + /// An index of files that contains stats data + #[prost(message, repeated, tag = "8")] + pub stats_index: ::prost::alloc::vec::Vec, + /// is_merge_option_allowed indicates whether merge option is allowed for this schema. + /// For backward compatibility, this field defaults to false when not present. + #[prost(bool, tag = "9")] + pub is_merge_option_allowed: bool, + /// partition_merge_option_allowed indicates whether merge option is allowed for each partition. + /// is_merge_option_allowed is table level, and the partition_merge_option_allowed is partition level. + #[prost(map = "string, bool", tag = "10")] + pub partition_merge_option_allowed: ::std::collections::HashMap< + ::prost::alloc::string::String, + bool, + >, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -192,6 +297,9 @@ pub struct PitrTableMap { pub id_map: ::core::option::Option, #[prost(message, repeated, tag = "3")] pub partitions: ::prost::alloc::vec::Vec, + /// whether this table has been filtered out + #[prost(bool, tag = "4")] + pub filtered_out: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -202,6 +310,13 @@ pub struct PitrDbMap { pub id_map: ::core::option::Option, #[prost(message, repeated, tag = "3")] pub tables: ::prost::alloc::vec::Vec, + /// whether this db has been filtered out + #[prost(bool, tag = "4")] + pub filtered_out: bool, + /// whether this db has been reused + /// skip restoring the metakv of database in log restore if it is true + #[prost(bool, tag = "5")] + pub reused: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -244,6 +359,26 @@ pub mod error { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct StreamBackupTaskSecurityConfig { + #[prost(oneof = "stream_backup_task_security_config::Encryption", tags = "1, 2")] + pub encryption: ::core::option::Option< + stream_backup_task_security_config::Encryption, + >, +} +/// Nested message and enum types in `StreamBackupTaskSecurityConfig`. +pub mod stream_backup_task_security_config { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Encryption { + /// not recommended in production. configure KMS based or locally managed master key instead in TiKV StreamBackupConfig + #[prost(message, tag = "1")] + PlaintextDataKey(super::CipherInfo), + #[prost(message, tag = "2")] + MasterKeyConfig(super::MasterKeyConfig), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct CipherInfo { #[prost(enumeration = "super::encryptionpb::EncryptionMethod", tag = "1")] pub cipher_type: i32, @@ -252,6 +387,20 @@ pub struct CipherInfo { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct MasterKeyConfig { + #[prost(enumeration = "super::encryptionpb::EncryptionMethod", tag = "1")] + pub encryption_type: i32, + #[prost(message, repeated, tag = "2")] + pub master_keys: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SortedSubRanges { + #[prost(message, repeated, tag = "1")] + pub sub_ranges: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct BackupRequest { #[prost(uint64, tag = "1")] pub cluster_id: u64, @@ -299,6 +448,10 @@ pub struct BackupRequest { /// only the remaining sub ranges of the range need to be backed up this time. #[prost(message, repeated, tag = "16")] pub sub_ranges: ::prost::alloc::vec::Vec, + /// only ranges in the same groups can be merged into one SST file. + /// the ranges in each groups are sorted. + #[prost(message, repeated, tag = "21")] + pub sorted_sub_ranges_groups: ::prost::alloc::vec::Vec, /// replica_read indicates whether to use replica read for backup. /// If it is false, the backup will only read data from leader. /// If it is true, the backup will read data from both leader and follower. @@ -325,7 +478,7 @@ pub struct StreamBackupTaskInfo { pub start_ts: u64, #[prost(uint64, tag = "3")] pub end_ts: u64, - /// Misc meta datas. + /// Misc meta data. /// The name of the task, also the ID of the task. #[prost(string, tag = "4")] pub name: ::prost::alloc::string::String, @@ -336,6 +489,9 @@ pub struct StreamBackupTaskInfo { /// compression type #[prost(enumeration = "CompressionType", tag = "6")] pub compression_type: i32, + /// security config for backup files + #[prost(message, optional, tag = "7")] + pub security_config: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -684,7 +840,7 @@ pub struct DataFileGroup { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct DataFileInfo { - /// SHA256 of the file. + /// Checksum of the plaintext file, i.e., pre-compression, pre-encryption. #[prost(bytes = "vec", tag = "1")] pub sha256: ::prost::alloc::vec::Vec, /// Path of the file. @@ -738,6 +894,34 @@ pub struct DataFileInfo { /// The compression type for the file. #[prost(enumeration = "CompressionType", tag = "18")] pub compression_type: i32, + /// The CRC64XOR of all contents of the file. + /// In older versions, this might be empty. + #[prost(uint64, tag = "19")] + pub crc64xor: u64, + /// The original region range the log file belongs to. + /// In older versions, this might be empty. + /// This may not be accurative: file may contain keys less than this. + #[prost(bytes = "vec", tag = "20")] + pub region_start_key: ::prost::alloc::vec::Vec, + /// The original region range the log file belongs to. + /// In older versions, this might be empty. + /// This may not be accurative: file may contain keys greater than this. + #[prost(bytes = "vec", tag = "21")] + pub region_end_key: ::prost::alloc::vec::Vec, + /// The region epoch that the log file belongs to. + /// In older versions, this might be empty. + /// If a region get split or merged during observing, the file may contain multi epoches. + /// This may not be complete: file may contain records from other versions. + /// + /// If there is exactly one epoch, + /// `region_start_key` and `region_end_key` must match this epoch. + #[prost(message, repeated, tag = "22")] + pub region_epoch: ::prost::alloc::vec::Vec, + /// Encryption information of this data file, not set if plaintext. + #[prost(message, optional, tag = "23")] + pub file_encryption_info: ::core::option::Option< + super::encryptionpb::FileEncryptionInfo, + >, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -755,6 +939,321 @@ pub struct StreamBackupError { #[prost(uint64, tag = "4")] pub store_id: u64, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Span { + #[prost(uint64, tag = "1")] + pub offset: u64, + #[prost(uint64, tag = "2")] + pub length: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SpansOfFile { + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub spans: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFileSubcompactionMeta { + /// The whole size of this subcompaction. + /// It should be the physical size of all input logs. + #[prost(uint64, tag = "1")] + pub size: u64, + /// The region that we perform the subcompaction. + #[prost(uint64, tag = "2")] + pub region_id: u64, + #[prost(string, tag = "3")] + pub cf: ::prost::alloc::string::String, + #[prost(uint64, tag = "4")] + pub input_max_ts: u64, + #[prost(uint64, tag = "5")] + pub input_min_ts: u64, + /// The whole compaction's configure. + #[prost(uint64, tag = "6")] + pub compact_from_ts: u64, + #[prost(uint64, tag = "7")] + pub compact_until_ts: u64, + /// The key range of the input logs. + /// They are encoded user key. (i.e. mem-comparable, no 'z' prefix) + #[prost(bytes = "vec", tag = "8")] + pub min_key: ::prost::alloc::vec::Vec, + /// The largest key from the input. + #[prost(bytes = "vec", tag = "9")] + pub max_key: ::prost::alloc::vec::Vec, + /// The type of operations in the file. (Write or Delete) + #[prost(enumeration = "FileType", tag = "10")] + pub ty: i32, + #[prost(int64, tag = "11")] + pub table_id: i64, + /// The input log files of this. + #[prost(message, repeated, tag = "12")] + pub sources: ::prost::alloc::vec::Vec, +} +/// A subcompaction from a compaction. +/// Subcompactions are actions that collect then sort key values from one region. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFileSubcompaction { + /// The metadata of the subcompaction. + #[prost(message, optional, tag = "1")] + pub meta: ::core::option::Option, + /// The generated SSTs. + #[prost(message, repeated, tag = "2")] + pub sst_outputs: ::prost::alloc::vec::Vec, + /// The hint for the history of the region that contains the keys being compacted. + /// This may be helpful for splitting, but it does not guarantee completeness and accuracy. + #[prost(message, repeated, tag = "3")] + pub region_meta_hints: ::prost::alloc::vec::Vec, +} +/// You may get region id from `LogFileSubcompactionMeta`. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionMetaHint { + /// The left boundary of the region of this epoch. + /// It is an byte-comparable encoded key without TS. + #[prost(bytes = "vec", tag = "1")] + pub start_key: ::prost::alloc::vec::Vec, + /// The right boundary of the region of the epoch. + #[prost(bytes = "vec", tag = "2")] + pub end_key: ::prost::alloc::vec::Vec, + /// The region epoch. + #[prost(message, optional, tag = "3")] + pub region_epoch: ::core::option::Option, +} +/// Batched version of `LogFileSubcompaction`. +/// So we can store many subcompactions to one file, to reduce the number of file. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFileSubcompactions { + #[prost(message, repeated, tag = "1")] + pub subcompactions: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFileCompaction { + /// The selector of compaction source. + /// Files contains ANY record greater than this (and less than `until_ts`) will be selected. + /// The source and artificats may contain records out of the `from_ts` and `until_ts` range. + /// When restoring, please use `input_min_ts` to filter out not needed compactions instead. + #[prost(uint64, tag = "1")] + pub compaction_from_ts: u64, + /// The selector of compaction source. + /// Files contains ANY record less than this (and not less than `from_ts`) will be selected. + /// The source and artificats may contain records out of the `from_ts` and `until_ts` range. + /// When restoring, please use `input_max_ts` to filter out not needed compactions instead. + #[prost(uint64, tag = "2")] + pub compaction_until_ts: u64, + /// The user provided name for the compaction. + #[prost(string, tag = "3")] + pub name: ::prost::alloc::string::String, + /// The statistics or other useful information for human. + #[prost(string, tag = "4")] + pub comments: ::prost::alloc::string::String, + /// relative path to the generated subcompaction metas. ("artificates") + #[prost(string, tag = "5")] + pub artifacts: ::prost::alloc::string::String, + /// the XOR of all artificate's hash. + #[prost(uint64, tag = "6")] + pub artifacts_hash: u64, + /// relative path to the subcompaction generated SST files. + /// this is a hint for garbage-collecting, so we don't need to read the metas one by one. + /// Those SSTs cannot be directly used, what you need is probably in `artificates`. + #[prost(string, tag = "7")] + pub generated_files: ::prost::alloc::string::String, + /// The smallest TS of record contained among all selected files of this compaction. + #[prost(uint64, tag = "8")] + pub input_min_ts: u64, + /// The largest TS of record contained among all selected files of this compaction. + #[prost(uint64, tag = "9")] + pub input_max_ts: u64, + /// Total size of generated compact-log-backup cmeta files. + #[prost(uint64, tag = "10")] + pub generated_cmeta_files_total_size: u64, + /// Total size of generated compact-log-backup SST files. + #[prost(uint64, tag = "11")] + pub generated_sst_files_total_size: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MetaEdit { + /// Path to the meta file. + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, + /// Delete the physical files (MetaFileGroup) in the meta file. + #[prost(string, repeated, tag = "2")] + pub delete_physical_files: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Delete the logical files (MetaFileInfo) in the meta file. + /// Note: Even the operation have been performed in the meta, + /// this modification should be kept as long as the corresponding physical + /// file not deleted. Or we may cannot know when to delete the physical file. + /// Then the file will be leak until truncated. + #[prost(message, repeated, tag = "3")] + pub delete_logical_files: ::prost::alloc::vec::Vec, + /// Whether to delete the metadata itself. + #[prost(bool, tag = "4")] + pub destruct_self: bool, + /// Whether all data kv files have been compacted but meta kv file still remains. + /// this field used to determine whether we can ignore the physical data kv log files in a compaction. + #[prost(bool, tag = "5")] + pub all_data_files_compacted: bool, +} +/// RewrittenTableID records a sort of modification over the SSTs during restoring. +/// +/// When "backing up" this "restored" SST, we want to backup the original SST before any rewrite. +/// But in some cases, SSTs are not restored "as is", TiKV may rewrite or truncate its +/// content by the client's request. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RewrittenTableId { + /// The table ID in the backup data. + #[prost(int64, tag = "1")] + pub ancestor_upstream: i64, + /// The rewritten table ID during restoring. + #[prost(int64, tag = "2")] + pub upstream: i64, +} +/// IngestedSSTs is created by a client that wants to put data to the cluster by `Ingest` APIs, +/// these write cannot be directly recorded by log backup. The client should put a migration +/// that contains the SSTs it uploaded. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestedSsTs { + /// The table IDs rewritten during restoring. + #[prost(message, repeated, tag = "1")] + pub rewritten_tables: ::prost::alloc::vec::Vec, + /// The SST files restored. + #[prost(message, repeated, tag = "2")] + pub files: ::prost::alloc::vec::Vec, + /// Treating the whole batch of SSTs as an huge atomic + /// write as this timestamp. + /// That is, when user wants to PiTR to sometime after this, + /// everything restored will present after the PiTR. + /// Otherwise nothing will present. + #[prost(uint64, tag = "3")] + pub as_if_ts: u64, + /// The hint of the common prefix of the files. + /// Used for speed up garbage collecting. + #[prost(string, tag = "4")] + pub files_prefix_hint: ::prost::alloc::string::String, + /// Whether this batch have finished. + /// If not, `as_if_ts` should be ignored and this should only + /// be restored when there is a finished backup with the same + /// `backup_uuid`. + #[prost(bool, tag = "5")] + pub finished: bool, + /// When checkpoint enabled, one restoration may be separated to many + /// of `IngestedSST`s. + /// `IngestedSST`s sharing the same UUID should be treated as one. + /// That is: + /// + /// * The `as_if_ts` should be the largest one among all finished ingested SST batches. + /// * When restoring, all batches sharing the same uuid should be restored or not. + #[prost(bytes = "vec", tag = "6")] + pub backup_uuid: ::prost::alloc::vec::Vec, +} +/// An extended version of `SpansOfFile`, added more metadata for the +/// execution of delayed deletion. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteSpansOfFile { + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub spans: ::prost::alloc::vec::Vec, + /// This is a hint for garbage collecting. + /// So we can easily check whether a set of spans consists the whole file. + #[prost(uint64, tag = "3")] + pub whole_file_length: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Migration { + #[prost(string, tag = "5")] + pub creator: ::prost::alloc::string::String, + #[prost(enumeration = "MigrationVersion", tag = "6")] + pub version: i32, + #[prost(message, repeated, tag = "1")] + pub edit_meta: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "2")] + pub truncated_to: u64, + #[prost(message, repeated, tag = "3")] + pub compactions: ::prost::alloc::vec::Vec, + /// Delete all files matches this prefix. + #[prost(string, repeated, tag = "4")] + pub destruct_prefix: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// The ingested SSTs. + /// It is mutable during restoring, hence a reference. + #[prost(string, repeated, tag = "7")] + pub ingested_sst_paths: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum PrepareSnapshotBackupRequestType { + /// Update the lease of suspending some commands (Which may impact the + /// disk snapshot backup) to be proposed. + /// As long as the lease is kept, those commands may not be proposed. + /// If the client has gone, the lease will be destroyed after its TTL. + UpdateLease = 0, + /// Wait a region apply to the last index. + WaitApply = 1, + /// Hint before closing the stream. Server will go back to normal after this. + /// This should also return a final "UpdateLeaseResult" to the client, which + /// indices whether the last lease is valid. + Finish = 2, +} +impl PrepareSnapshotBackupRequestType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + PrepareSnapshotBackupRequestType::UpdateLease => "UpdateLease", + PrepareSnapshotBackupRequestType::WaitApply => "WaitApply", + PrepareSnapshotBackupRequestType::Finish => "Finish", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UpdateLease" => Some(Self::UpdateLease), + "WaitApply" => Some(Self::WaitApply), + "Finish" => Some(Self::Finish), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum PrepareSnapshotBackupEventType { + /// A region has finished wait apply. + WaitApplyDone = 0, + /// A lease has been updated. + UpdateLeaseResult = 1, +} +impl PrepareSnapshotBackupEventType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + PrepareSnapshotBackupEventType::WaitApplyDone => "WaitApplyDone", + PrepareSnapshotBackupEventType::UpdateLeaseResult => "UpdateLeaseResult", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "WaitApplyDone" => Some(Self::WaitApplyDone), + "UpdateLeaseResult" => Some(Self::UpdateLeaseResult), + _ => None, + } + } +} /// sst files or log files compression algorithm /// for log files, unknown means not use compression algorithm #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] @@ -875,6 +1374,38 @@ impl FileType { } } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum MigrationVersion { + /// The initial version. Somehow "UNKNOWN". + M0 = 0, + /// Added `creator` and `version`. + M1 = 1, + /// Added `ingested_sst_paths`. + M2 = 2, +} +impl MigrationVersion { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + MigrationVersion::M0 => "M0", + MigrationVersion::M1 => "M1", + MigrationVersion::M2 => "M2", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "M0" => Some(Self::M0), + "M1" => Some(Self::M1), + "M2" => Some(Self::M2), + _ => None, + } + } +} /// Generated client implementations. pub mod backup_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -1010,6 +1541,37 @@ pub mod backup_client { .insert(GrpcMethod::new("backup.Backup", "CheckPendingAdminOp")); self.inner.server_streaming(req, path, codec).await } + /// PrepareSnapshotBackup is an advanced version of preparing snapshot backup. + /// Check the defination of `PrepareSnapshotBackupRequest` for more. + pub async fn prepare_snapshot_backup( + &mut self, + request: impl tonic::IntoStreamingRequest< + Message = super::PrepareSnapshotBackupRequest, + >, + ) -> std::result::Result< + tonic::Response< + tonic::codec::Streaming, + >, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/backup.Backup/PrepareSnapshotBackup", + ); + let mut req = request.into_streaming_request(); + req.extensions_mut() + .insert(GrpcMethod::new("backup.Backup", "PrepareSnapshotBackup")); + self.inner.streaming(req, path, codec).await + } /// prepare is used for file-copy backup. before we start the backup for a TiKV. /// we need invoke this function to generate the SST files map. or we get nothing to backup. pub async fn prepare( diff --git a/src/generated/cdcpb.rs b/src/generated/cdcpb.rs index e80572e2..5d2a993f 100644 --- a/src/generated/cdcpb.rs +++ b/src/generated/cdcpb.rs @@ -31,6 +31,15 @@ pub struct ClusterIdMismatch { #[prost(uint64, tag = "2")] pub request: u64, } +/// Congested is an error variable that +/// tells people that the TiKV-CDC is congested. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Congested { + /// The region ID that triggers the congestion. + #[prost(uint64, tag = "1")] + pub region_id: u64, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Error { @@ -48,6 +57,8 @@ pub struct Error { pub cluster_id_mismatch: ::core::option::Option, #[prost(message, optional, tag = "7")] pub server_is_busy: ::core::option::Option, + #[prost(message, optional, tag = "8")] + pub congested: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -92,6 +103,9 @@ pub mod event { pub commit_ts: u64, #[prost(enumeration = "LogType", tag = "3")] pub r#type: i32, + /// generation is for pipelined DML protocol. See kvrpcpb.FlushRequest.generation. + #[prost(uint64, tag = "10")] + pub generation: u64, #[prost(enumeration = "row::OpType", tag = "4")] pub op_type: i32, #[prost(bytes = "vec", tag = "5")] diff --git a/src/generated/coprocessor.rs b/src/generated/coprocessor.rs index 907f9697..16676352 100644 --- a/src/generated/coprocessor.rs +++ b/src/generated/coprocessor.rs @@ -8,6 +8,16 @@ pub struct KeyRange { #[prost(bytes = "vec", tag = "2")] pub end: ::prost::alloc::vec::Vec, } +/// KeyRange with an attached read_ts (version). +/// It is used by TiCI versioned lookup. Callers must ensure `range` is a point range. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct VersionedKeyRange { + #[prost(message, optional, tag = "1")] + pub range: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub read_ts: u64, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Request { @@ -44,6 +54,33 @@ pub struct Request { /// This is the session alias between a client and tidb #[prost(string, tag = "13")] pub connection_alias: ::prost::alloc::string::String, + /// Shard infos for FTS index, used by TiFlash reading TiCI. + #[prost(message, repeated, tag = "14")] + pub table_shard_infos: ::prost::alloc::vec::Vec, + /// Versioned point ranges for TiCI lookup. + /// When `versioned_ranges` is non-empty, all `versioned_ranges\[i\].range` must be point range. + #[prost(message, repeated, tag = "15")] + pub versioned_ranges: ::prost::alloc::vec::Vec, + /// max_keys_read is 0 when disabled, otherwise limits the number of storage + /// engine keys scanned per coprocessor task. It is a per-task hard ceiling + /// applied uniformly to any coprocessor request, used to bound worst-case + /// scan amplification regardless of how the request is paginated. + #[prost(uint64, tag = "16")] + pub max_keys_read: u64, + /// paging_size_bytes is 0 when disabled, otherwise it should be a positive number. + /// When set, within a paged coprocessor request, TiKV stops scanning the current + /// page once accumulated scanned bytes reach this limit and returns the page + /// boundary so the next page can resume from there. + /// + /// Unlike max_keys_read, which is a per-task hard ceiling on keys scanned, + /// paging_size_bytes is an independent rate-control dimension intended for + /// Resource Control: it bounds the byte volume of a single page so that RU + /// pre-charging in PD's resource controller can be performed at byte + /// granularity, complementing the existing row-count-based paging + /// (paging_size). The two fields target different scenarios and may be set + /// independently. + #[prost(uint64, tag = "17")] + pub paging_size_bytes: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -92,6 +129,54 @@ pub struct RegionInfo { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct ShardInfo { + #[prost(uint64, tag = "1")] + pub shard_id: u64, + #[prost(uint64, tag = "2")] + pub shard_epoch: u64, + #[prost(message, repeated, tag = "3")] + pub ranges: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableShardInfos { + /// The executor ID is used to identify the tici executor. + #[prost(string, tag = "1")] + pub executor_id: ::prost::alloc::string::String, + /// The shard_infos contains the shard information for each tici executor. + #[prost(message, repeated, tag = "2")] + pub shard_infos: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TiCiEstimateCountRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub start_ts: u64, + #[prost(int64, tag = "3")] + pub table_id: i64, + #[prost(int64, tag = "4")] + pub index_id: i64, + #[prost(bytes = "vec", tag = "5")] + pub fts_query_info: ::prost::alloc::vec::Vec, + #[prost(string, tag = "6")] + pub time_zone_name: ::prost::alloc::string::String, + #[prost(int64, tag = "7")] + pub time_zone_offset: i64, + #[prost(message, repeated, tag = "8")] + pub shard_infos: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TiCiEstimateCountResponse { + #[prost(uint64, tag = "1")] + pub est_count: u64, + #[prost(string, tag = "2")] + pub other_error: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TableRegions { #[prost(int64, tag = "1")] pub physical_table_id: i64, @@ -125,6 +210,9 @@ pub struct BatchRequest { /// This is the session alias between a client and tidb #[prost(string, tag = "10")] pub connection_alias: ::prost::alloc::string::String, + /// Shard infos for FTS index, used by TiFlash reading TiCI. + #[prost(message, repeated, tag = "11")] + pub table_shard_infos: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -137,6 +225,8 @@ pub struct BatchResponse { pub exec_details: ::core::option::Option, #[prost(message, repeated, tag = "4")] pub retry_regions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub retry_shards: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -151,6 +241,13 @@ pub struct StoreBatchTask { pub ranges: ::prost::alloc::vec::Vec, #[prost(uint64, tag = "5")] pub task_id: u64, + /// Versioned point ranges for TiCI lookup. + /// When `versioned_ranges` is non-empty, all `versioned_ranges\[i\].range` must be point range. + #[prost(message, repeated, tag = "6")] + pub versioned_ranges: ::prost::alloc::vec::Vec, + /// Bucket metadata version used to validate this child task. + #[prost(uint64, tag = "7")] + pub buckets_version: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -168,3 +265,38 @@ pub struct StoreBatchTaskResponse { #[prost(message, optional, tag = "6")] pub exec_details_v2: ::core::option::Option, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DelegateRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub start_ts: u64, + #[prost(message, repeated, tag = "3")] + pub ranges: ::prost::alloc::vec::Vec, + /// Used for avoid redundant mem-table copying. + /// If the sequence is the same, tikv-server will not return the mem-table. + #[prost(uint64, tag = "4")] + pub mem_table_sequence: u64, + /// Used for avoid redundant snapshot copying. + /// If the sequence is the same, tikv-server will not return the snapshot. + #[prost(uint64, tag = "5")] + pub snapshot_sequence: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DelegateResponse { + #[prost(bytes = "vec", tag = "1")] + pub mem_table_data: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "2")] + pub snapshot: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "3")] + pub region_error: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub locked: ::core::option::Option, + #[prost(string, tag = "5")] + pub other_error: ::prost::alloc::string::String, + /// Used for avoid redundant mem-table copying. + #[prost(uint64, tag = "6")] + pub mem_table_sequence: u64, +} diff --git a/src/generated/deadlock.rs b/src/generated/deadlock.rs index ef9c61e3..da4c957a 100644 --- a/src/generated/deadlock.rs +++ b/src/generated/deadlock.rs @@ -32,11 +32,31 @@ pub struct WaitForEntry { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReplaceLockByKeyItem { + #[prost(uint64, tag = "1")] + pub key_hash: u64, + #[prost(bytes = "vec", tag = "2")] + pub key: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "3")] + pub old_lock_ts: u64, + #[prost(uint64, tag = "4")] + pub new_lock_ts: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReplaceLocksByKeysRequest { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct DeadlockRequest { #[prost(enumeration = "DeadlockRequestType", tag = "1")] pub tp: i32, #[prost(message, optional, tag = "2")] pub entry: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub replace_locks_by_keys: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -44,13 +64,16 @@ pub struct DeadlockResponse { /// The same entry sent by DeadlockRequest, identifies the sender. #[prost(message, optional, tag = "1")] pub entry: ::core::option::Option, - /// The key hash of the lock that is hold by the waiting transaction. + /// The key hash of the lock that is hold by the waiting transaction. The hash of the `deadlock_key` field. #[prost(uint64, tag = "2")] pub deadlock_key_hash: u64, /// The other entries of the dead lock circle. The current entry is in `entry` field and not /// included in this field. #[prost(message, repeated, tag = "3")] pub wait_chain: ::prost::alloc::vec::Vec, + /// The key of the lock that is hold by the waiting transaction. + #[prost(bytes = "vec", tag = "4")] + pub deadlock_key: ::prost::alloc::vec::Vec, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/src/generated/encryptionpb.rs b/src/generated/encryptionpb.rs index f7d7a924..6a944b9d 100644 --- a/src/generated/encryptionpb.rs +++ b/src/generated/encryptionpb.rs @@ -108,6 +108,53 @@ pub struct MasterKeyKms { /// KMS endpoint. Normally not needed. #[prost(string, tag = "4")] pub endpoint: ::prost::alloc::string::String, + /// optional, used to set up azure master key backend + #[prost(message, optional, tag = "5")] + pub azure_kms: ::core::option::Option, + /// optional, used to set up gcp master key backend + #[prost(message, optional, tag = "6")] + pub gcp_kms: ::core::option::Option, + /// optional, used to set up aws master key backend + #[prost(message, optional, tag = "7")] + pub aws_kms: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AzureKms { + #[prost(string, tag = "1")] + pub tenant_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub client_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub client_secret: ::prost::alloc::string::String, + /// Key vault to encrypt/decrypt data key. + #[prost(string, tag = "4")] + pub key_vault_url: ::prost::alloc::string::String, + /// optional hsm used to generate data key + #[prost(string, tag = "5")] + pub hsm_name: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub hsm_url: ::prost::alloc::string::String, + #[prost(string, tag = "7")] + pub client_certificate: ::prost::alloc::string::String, + #[prost(string, tag = "8")] + pub client_certificate_path: ::prost::alloc::string::String, + #[prost(string, tag = "9")] + pub client_certificate_password: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GcpKms { + #[prost(string, tag = "1")] + pub credential: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AwsKms { + #[prost(string, tag = "1")] + pub access_key: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub secret_access_key: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -134,6 +181,44 @@ pub struct EncryptedContent { #[prost(bytes = "vec", tag = "5")] pub ciphertext_key: ::prost::alloc::vec::Vec, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FileEncryptionInfo { + /// file encryption method + #[prost(enumeration = "EncryptionMethod", tag = "3")] + pub encryption_method: i32, + /// iv to encrypt the file by data key + #[prost(bytes = "vec", tag = "4")] + pub file_iv: ::prost::alloc::vec::Vec, + /// file checksum after encryption, optional if using GCM + #[prost(bytes = "vec", tag = "5")] + pub checksum: ::prost::alloc::vec::Vec, + #[prost(oneof = "file_encryption_info::Mode", tags = "1, 2")] + pub mode: ::core::option::Option, +} +/// Nested message and enum types in `FileEncryptionInfo`. +pub mod file_encryption_info { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Mode { + #[prost(message, tag = "1")] + PlainTextDataKey(super::PlainTextDataKey), + #[prost(message, tag = "2")] + MasterKeyBased(super::MasterKeyBased), + } +} +/// not recommended in production. +/// user needs to pass back the same data key for restore. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PlainTextDataKey {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MasterKeyBased { + /// encrypted data key with metadata + #[prost(message, repeated, tag = "1")] + pub data_key_encrypted_content: ::prost::alloc::vec::Vec, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum EncryptionMethod { diff --git a/src/generated/errorpb.rs b/src/generated/errorpb.rs index 4ed789b0..ddba0b60 100644 --- a/src/generated/errorpb.rs +++ b/src/generated/errorpb.rs @@ -207,6 +207,14 @@ pub struct MismatchPeerId { #[prost(uint64, tag = "2")] pub store_peer_id: u64, } +/// UndeterminedResult is the error variant that tells the result is not determined yet. +/// For example, the raft protocol timed out and the apply result is unknown. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UndeterminedResult { + #[prost(string, tag = "1")] + pub message: ::prost::alloc::string::String, +} /// Error wraps all region errors, indicates an error encountered by a request. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -263,4 +271,7 @@ pub struct Error { /// BucketVersionNotMatch is the error variant that tells the request buckets version is not match. #[prost(message, optional, tag = "21")] pub bucket_version_not_match: ::core::option::Option, + /// UndeterminedResult is the error variant that tells the result is not determined yet. + #[prost(message, optional, tag = "22")] + pub undetermined_result: ::core::option::Option, } diff --git a/src/generated/import_sstpb.rs b/src/generated/import_sstpb.rs index 4f8a1274..f1169ae0 100644 --- a/src/generated/import_sstpb.rs +++ b/src/generated/import_sstpb.rs @@ -90,8 +90,15 @@ pub struct RewriteRule { pub old_key_prefix: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "2")] pub new_key_prefix: ::prost::alloc::vec::Vec, + /// (Optional) Rewrite all keys in the range to use this timestamp. #[prost(uint64, tag = "3")] pub new_timestamp: u64, + /// (Optional) Skip keys with timestamps greater than this during download, useful for compacted SST backups. + #[prost(uint64, tag = "4")] + pub ignore_after_timestamp: u64, + /// (Optional) Skip write CF keys with timestamps less than this during download. Default CF keys are preserved. + #[prost(uint64, tag = "5")] + pub ignore_before_timestamp: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -185,6 +192,8 @@ pub struct DownloadRequest { /// same order, otherwise the RPC request will fail. #[prost(message, optional, tag = "13")] pub rewrite_rule: ::core::option::Option, + #[prost(message, repeated, tag = "20")] + pub sorted_rewrite_rules: ::prost::alloc::vec::Vec, #[prost(message, optional, tag = "14")] pub storage_backend: ::core::option::Option, /// The identity for the stroage backend. @@ -246,6 +255,12 @@ pub struct SetDownloadSpeedLimitRequest { /// The download speed limit (bytes/second). Set to 0 for unlimited speed. #[prost(uint64, tag = "1")] pub speed_limit: u64, + /// The download speed limit task id. + #[prost(string, tag = "2")] + pub task_id: ::prost::alloc::string::String, + /// The download speed limit ttl. Set to 0 means it will never expire. + #[prost(uint64, tag = "3")] + pub ttl_seconds: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -464,6 +479,11 @@ pub struct KvMeta { /// the compression type for the file. #[prost(enumeration = "super::backup::CompressionType", tag = "13")] pub compression_type: i32, + /// encryption information of the kv file, not set if encryption is not enabled. + #[prost(message, optional, tag = "14")] + pub file_encryption_info: ::core::option::Option< + super::encryptionpb::FileEncryptionInfo, + >, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -498,9 +518,12 @@ pub struct ApplyRequest { /// context represents region info and it used to build raft commands. #[prost(message, optional, tag = "4")] pub context: ::core::option::Option, - /// cipher_info is used to decrypt kv file when download file. + /// plaintext data key to decrypt kv file if configured during log backup. #[prost(message, optional, tag = "11")] pub cipher_info: ::core::option::Option, + /// master keys config used to decrypt data keys in restore if configured during log backup. + #[prost(message, repeated, tag = "14")] + pub master_keys: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -525,6 +548,31 @@ pub struct ClearResponse { #[prost(message, optional, tag = "1")] pub error: ::core::option::Option, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AddPartitionRangeRequest { + #[prost(message, optional, tag = "1")] + pub range: ::core::option::Option, + /// the number of seconds this range is valid. + /// After this duration, if this range is still not removed by + /// `RemoveForcePartitionRange`, tikv will automically remove it. So the client + /// should set a big enough value to avoid auto cleanup. But in general, 1h + /// should be a big enough value. If its value is 0, tikv will auto adjust it to 3600(1h). + #[prost(uint64, tag = "2")] + pub ttl_seconds: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AddPartitionRangeResponse {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RemovePartitionRangeRequest { + #[prost(message, optional, tag = "1")] + pub range: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RemovePartitionRangeResponse {} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum SwitchMode { @@ -858,6 +906,62 @@ pub mod import_sst_client { .insert(GrpcMethod::new("import_sstpb.ImportSST", "Download")); self.inner.unary(req, path, codec).await } + /// Download SST files in batch from external storage, perform key rewrite, + /// and merge them into one SST after downloading. + pub async fn batch_download( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/import_sstpb.ImportSST/BatchDownload", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("import_sstpb.ImportSST", "BatchDownload")); + self.inner.unary(req, path, codec).await + } + /// Download SST files in batch from external storage, perform key rewrite, + /// merge them into one SST, and keep only the latest MVCC version after downloading. + pub async fn batch_download_latest_mvcc( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/import_sstpb.ImportSST/BatchDownloadLatestMVCC", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("import_sstpb.ImportSST", "BatchDownloadLatestMVCC"), + ); + self.inner.unary(req, path, codec).await + } /// Open a write stream to generate sst files pub async fn write( &mut self, @@ -1027,5 +1131,68 @@ pub mod import_sst_client { .insert(GrpcMethod::new("import_sstpb.ImportSST", "SuspendImportRPC")); self.inner.unary(req, path, codec).await } + /// AddForcePartitionRange marks a range in tikv that any compact overlaps with this range + /// should generates SST files partitioned at region boundaries as well as this range boundary. + /// TiKV will also try to do manual compact(if needed) after setting this range to eusure + /// any incoming SST under this range can be ingested into the bottom level if there is no real kv overlap. + pub async fn add_force_partition_range( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/import_sstpb.ImportSST/AddForcePartitionRange", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("import_sstpb.ImportSST", "AddForcePartitionRange"), + ); + self.inner.unary(req, path, codec).await + } + /// Remove the force partition range after the task is finished. If this function is not called, + /// tikv will cleanup the range after TTL to ensure it can be cleaned eventually. + pub async fn remove_force_partition_range( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/import_sstpb.ImportSST/RemoveForcePartitionRange", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "import_sstpb.ImportSST", + "RemoveForcePartitionRange", + ), + ); + self.inner.unary(req, path, codec).await + } } } diff --git a/src/generated/keyspacepb.rs b/src/generated/keyspacepb.rs index 3cf798d9..53e3bb00 100644 --- a/src/generated/keyspacepb.rs +++ b/src/generated/keyspacepb.rs @@ -28,6 +28,14 @@ pub struct LoadKeyspaceRequest { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct LoadKeyspaceByIdRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(uint32, tag = "2")] + pub id: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct LoadKeyspaceResponse { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -227,6 +235,31 @@ pub mod keyspace_client { .insert(GrpcMethod::new("keyspacepb.Keyspace", "LoadKeyspace")); self.inner.unary(req, path, codec).await } + pub async fn load_keyspace_by_id( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/keyspacepb.Keyspace/LoadKeyspaceByID", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("keyspacepb.Keyspace", "LoadKeyspaceByID")); + self.inner.unary(req, path, codec).await + } /// WatchKeyspaces first return all current keyspaces' metadata as its first response. /// Then, it returns responses containing keyspaces that had their metadata changed. pub async fn watch_keyspaces( diff --git a/src/generated/kvrpcpb.rs b/src/generated/kvrpcpb.rs index 4bf66314..89f2d472 100644 --- a/src/generated/kvrpcpb.rs +++ b/src/generated/kvrpcpb.rs @@ -10,6 +10,9 @@ pub struct GetRequest { pub key: ::prost::alloc::vec::Vec, #[prost(uint64, tag = "3")] pub version: u64, + /// If true, the response will include the commit ts of the key. + #[prost(bool, tag = "4")] + pub need_commit_ts: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -30,6 +33,10 @@ pub struct GetResponse { /// Time and scan details when processing the request. #[prost(message, optional, tag = "6")] pub exec_details_v2: ::core::option::Option, + /// The commit timestamp of the key. + /// If it is zero, it means the commit timestamp is unknown. + #[prost(uint64, tag = "7")] + pub commit_ts: u64, } /// Scan fetches values for a range of keys; it is part of the transaction with /// starting timestamp = `version`. @@ -166,10 +173,13 @@ pub mod prewrite_request { #[repr(i32)] pub enum PessimisticAction { /// The key needn't be locked and no extra write conflict checks are needed. + /// Deprecated in next-gen (cloud-storage-engine). SkipPessimisticCheck = 0, - /// The key should have been locked at the time of prewrite. + /// The key should have been locked at the time of prewrite. If the lock is missing, + /// the lock will be amended. This is the normal case for pessimistic transactions. DoPessimisticCheck = 1, - /// The key doesn't need a pessimistic lock. But we need to do data constraint checks. + /// The key does not acquire a pessimistic lock for performance optimization. + /// Constraint checking (write conflicts + data constraints) is deferred to prewrite. DoConstraintCheck = 2, } impl PessimisticAction { @@ -366,6 +376,11 @@ pub struct TxnHeartBeatRequest { /// The new TTL the sender would like. #[prost(uint64, tag = "4")] pub advise_lock_ttl: u64, + /// Optionally update PK's min_commit_ts. + /// Only for non-async-commmit and non-1PC transactions. + /// If it is 0, ignore this field. + #[prost(uint64, tag = "5")] + pub min_commit_ts: u64, /// Reserved for file based transaction. #[prost(bool, tag = "100")] pub is_txn_file: bool, @@ -506,6 +521,17 @@ pub struct CommitRequest { /// Timestamp for the end of the transaction. Must be greater than `start_version`. #[prost(uint64, tag = "4")] pub commit_version: u64, + /// commit_role indicates the current commit request is a primary commit or a secondary commit. + /// It's value maybe `Unknown` when using a client with an old version. + #[prost(enumeration = "CommitRole", tag = "6")] + pub commit_role: i32, + /// primary_key indicates the primary key of the transaction. + /// Its value may be empty when using an old version client. + #[prost(bytes = "vec", tag = "7")] + pub primary_key: ::prost::alloc::vec::Vec, + /// Whether committing an `use_async_commit` (i.e. should be treated as committed) prewrite. + #[prost(bool, tag = "8")] + pub use_async_commit: bool, /// Reserved for file based transaction. #[prost(bool, tag = "100")] pub is_txn_file: bool, @@ -579,6 +605,9 @@ pub struct BatchGetRequest { pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, #[prost(uint64, tag = "3")] pub version: u64, + /// If true, the response will include the commit ts of the key. + #[prost(bool, tag = "4")] + pub need_commit_ts: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -676,6 +705,9 @@ pub struct ResolveLockRequest { /// Only resolve specified keys. #[prost(bytes = "vec", repeated, tag = "5")] pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// If true, the resolve lock action is performed asynchronously and the response may be returned before it's finished. + #[prost(bool, tag = "6")] + pub is_async: bool, /// Reserved for file based transaction. #[prost(bool, tag = "100")] pub is_txn_file: bool, @@ -1112,6 +1144,9 @@ pub struct SplitRegionResponse { /// include all result regions. #[prost(message, repeated, tag = "4")] pub regions: ::prost::alloc::vec::Vec, + /// Reserved for file based transaction. + #[prost(message, repeated, tag = "100")] + pub errors: ::prost::alloc::vec::Vec, } /// Sent from TiFlash to a TiKV node. #[allow(clippy::derive_partial_eq_without_eq)] @@ -1255,6 +1290,17 @@ pub struct Context { /// Some information used for resource control. #[prost(message, optional, tag = "28")] pub resource_control_context: ::core::option::Option, + /// The declared origin of the request. TiDB server requests set this to TIDB. + /// This field is client-provided metadata; consumers should rely on it only when + /// the caller is authenticated/authorized by the transport or deployment boundary. + /// UNKNOWN means the origin is unset or not recognized by this protocol version. + /// Other components should add dedicated enum variants when they need origin-specific behavior. + #[prost(enumeration = "RequestOrigin", tag = "29")] + pub request_origin: i32, + /// The keyspace that the request is sent to. + /// NOTE: This field is only meaningful while the api_version is V2. + #[prost(string, tag = "31")] + pub keyspace_name: ::prost::alloc::string::String, /// The keyspace that the request is sent to. /// NOTE: This field is only meaningful while the api_version is V2. #[prost(uint32, tag = "32")] @@ -1267,6 +1313,21 @@ pub struct Context { /// This is for tests only and thus can be safely changed/removed without affecting compatibility. #[prost(message, optional, tag = "34")] pub source_stmt: ::core::option::Option, + /// The cluster id of the request + #[prost(uint64, tag = "35")] + pub cluster_id: u64, + /// The trace id of the request, will be used for tracing the request's execution's inner steps. + #[prost(bytes = "vec", tag = "36")] + pub trace_id: ::prost::alloc::vec::Vec, + /// Control flags for trace logging behavior. + /// Bit 0: immediate_log - Force immediate logging without buffering + /// Bit 1: category_req_resp - Enable request/response tracing + /// Bit 2: category_write_details - Enable detailed write tracing + /// Bit 3: category_read_details - Enable detailed read tracing + /// Bits 4-63: Reserved for future use + /// This field is set by client-go based on an extractor function provided by TiDB. + #[prost(uint64, tag = "37")] + pub trace_control_flags: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1326,6 +1387,14 @@ pub struct LockInfo { /// It can be used to help the client decide whether to try resolving the lock. #[prost(uint64, tag = "11")] pub duration_to_last_update_ms: u64, + /// When lock_type is SharedLock, this describes transactions holding the shared lock. + /// Important: when lock_type is SharedLock, all shared locks must use shared_lock_infos; + /// DO NOT read from the wrapper LockInfo. + /// TODO(slock): tidb should send requests with a feature flag to indicate whether it + /// supports shared locks, so that tikv can fail the requests from old tidb versions + /// when needed. + #[prost(message, repeated, tag = "12")] + pub shared_lock_infos: ::prost::alloc::vec::Vec, /// Reserved for file based transaction. #[prost(bool, tag = "100")] pub is_txn_file: bool, @@ -1366,6 +1435,12 @@ pub struct KeyError { /// CheckTxnStatus is sent to a lock that's not the primary. #[prost(message, optional, tag = "11")] pub primary_mismatch: ::core::option::Option, + /// TxnLockNotFound indicates the txn lock is not found. + #[prost(message, optional, tag = "12")] + pub txn_lock_not_found: ::core::option::Option, + /// Extra information for error debugging + #[prost(message, optional, tag = "100")] + pub debug_info: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1407,8 +1482,10 @@ pub mod write_conflict { SelfRolledBack = 3, /// RcCheckTs failure by meeting a newer version, let TiDB retry. RcCheckTs = 4, - /// write conflict found in lazy uniqueness check in pessimistic transactions. + /// write conflict found when deferring constraint checks in pessimistic transactions. Deprecated in next-gen (cloud-storage-engine). LazyUniquenessCheck = 5, + /// write conflict found on keys that do not acquire pessimistic locks in pessimistic transactions. + NotLockedKeyConflict = 6, } impl Reason { /// String value of the enum field names used in the ProtoBuf definition. @@ -1423,6 +1500,7 @@ pub mod write_conflict { Reason::SelfRolledBack => "SelfRolledBack", Reason::RcCheckTs => "RcCheckTs", Reason::LazyUniquenessCheck => "LazyUniquenessCheck", + Reason::NotLockedKeyConflict => "NotLockedKeyConflict", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -1434,6 +1512,7 @@ pub mod write_conflict { "SelfRolledBack" => Some(Self::SelfRolledBack), "RcCheckTs" => Some(Self::RcCheckTs), "LazyUniquenessCheck" => Some(Self::LazyUniquenessCheck), + "NotLockedKeyConflict" => Some(Self::NotLockedKeyConflict), _ => None, } } @@ -1452,10 +1531,14 @@ pub struct Deadlock { pub lock_ts: u64, #[prost(bytes = "vec", tag = "2")] pub lock_key: ::prost::alloc::vec::Vec, + /// The hash of `deadlock_key` field. #[prost(uint64, tag = "3")] pub deadlock_key_hash: u64, #[prost(message, repeated, tag = "4")] pub wait_chain: ::prost::alloc::vec::Vec, + /// The key that the current transaction has already acquired and blocks another transaction to form the deadlock. + #[prost(bytes = "vec", tag = "5")] + pub deadlock_key: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1506,6 +1589,26 @@ pub struct PrimaryMismatch { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxnLockNotFound { + #[prost(bytes = "vec", tag = "1")] + pub key: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MvccDebugInfo { + #[prost(bytes = "vec", tag = "1")] + pub key: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "2")] + pub mvcc: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DebugInfo { + #[prost(message, repeated, tag = "1")] + pub mvcc_info: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TimeDetail { /// Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and /// other kind of waitings in series. (Wait time in the raftstore is not included.) @@ -1548,6 +1651,12 @@ pub struct TimeDetailV2 { /// Total wall clock time spent on this RPC in TiKV . #[prost(uint64, tag = "5")] pub total_rpc_wall_time_ns: u64, + /// Time spent on the gRPC layer. + #[prost(uint64, tag = "6")] + pub kv_grpc_process_time_ns: u64, + /// Time spent on waiting for run again in grpc pool from other executor pool. + #[prost(uint64, tag = "7")] + pub kv_grpc_wait_time_ns: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1622,6 +1731,29 @@ pub struct ScanDetailV2 { /// Time used for read pool scheduling. #[prost(uint64, tag = "13")] pub read_pool_schedule_wait_nanos: u64, + /// The total MVCC key-value size meet during scanning. It includes + /// deleted versions, but does not include RocksDB tombstone keys. + /// + /// When this field is notably larger than `processed_versions_size`, it means + /// there are a lot of deleted MVCC keys. + #[prost(uint64, tag = "14")] + pub total_versions_size: u64, + /// Total number of IA segment cache hits for this request. + #[prost(uint64, tag = "15")] + pub ia_cache_hit_count: u64, + /// Total number of IA remote segment reads for this request. + /// + /// IA segments are only read on cache misses, so this is also the IA segment + /// cache miss count for the request. + #[prost(uint64, tag = "16")] + pub ia_remote_read_segment_count: u64, + /// Total number of logical bytes returned from IA remote segment reads for this + /// request. + #[prost(uint64, tag = "17")] + pub ia_remote_read_segment_bytes: u64, + /// Total time spent serving IA remote segment reads for this request. + #[prost(uint64, tag = "18")] + pub ia_remote_read_segment_nanos: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1649,6 +1781,51 @@ pub struct ExecDetailsV2 { /// Available when ctx.record_time_stat = true or meet slow query. #[prost(message, optional, tag = "4")] pub time_detail_v2: ::core::option::Option, + /// RU (Request Unit) consumption details. + #[prost(message, optional, tag = "5")] + pub ru_v2: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Ruv2 { + #[prost(uint64, tag = "1")] + pub kv_engine_cache_miss: u64, + #[prost(message, optional, tag = "2")] + pub executor_inputs: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub coprocessor_executor_iterations: u64, + #[prost(uint64, tag = "4")] + pub coprocessor_response_bytes: u64, + #[prost(uint64, tag = "5")] + pub raftstore_store_write_trigger_wb_bytes: u64, + #[prost(uint64, tag = "6")] + pub storage_processed_keys_batch_get: u64, + #[prost(uint64, tag = "7")] + pub storage_processed_keys_get: u64, + /// Filled by client-go. + #[prost(uint64, tag = "8")] + pub read_rpc_count: u64, + /// Filled by client-go. + #[prost(uint64, tag = "9")] + pub write_rpc_count: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ExecutorInputs { + #[prost(uint64, tag = "1")] + pub tikv_coprocessor_executor_work_total_batch_index_scan: u64, + #[prost(uint64, tag = "2")] + pub tikv_coprocessor_executor_work_total_batch_table_scan: u64, + #[prost(uint64, tag = "3")] + pub tikv_coprocessor_executor_work_total_batch_selection: u64, + #[prost(uint64, tag = "4")] + pub tikv_coprocessor_executor_work_total_batch_top_n: u64, + #[prost(uint64, tag = "5")] + pub tikv_coprocessor_executor_work_total_batch_limit: u64, + #[prost(uint64, tag = "6")] + pub tikv_coprocessor_executor_work_total_batch_simple_aggr: u64, + #[prost(uint64, tag = "7")] + pub tikv_coprocessor_executor_work_total_batch_fast_hash_aggr: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1714,6 +1891,10 @@ pub struct KvPair { pub key: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "3")] pub value: ::prost::alloc::vec::Vec, + /// The commit timestamp of the key. + /// If it is zero, it means the commit timestamp is unknown. + #[prost(uint64, tag = "4")] + pub commit_ts: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1905,6 +2086,9 @@ pub struct RawCasRequest { pub cf: ::prost::alloc::string::String, #[prost(uint64, tag = "7")] pub ttl: u64, + /// If `delete` is true, the request is to delete the key when the comparison succeeds + #[prost(bool, tag = "8")] + pub delete: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -2097,6 +2281,129 @@ pub struct TiFlashSystemTableResponse { #[prost(bytes = "vec", tag = "1")] pub data: ::prost::alloc::vec::Vec, } +/// Flush is introduced from the pipelined DML protocol. +/// A Flush request writes some keys and values to TiKV, storing in LOCK and DEFAULT CF, just like a Prewrite request. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub mutations: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "3")] + pub primary_key: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "4")] + pub start_ts: u64, + #[prost(uint64, tag = "5")] + pub min_commit_ts: u64, + /// generation of the flush request. It is a monotonically increasing number in each transaction. + #[prost(uint64, tag = "6")] + pub generation: u64, + #[prost(uint64, tag = "7")] + pub lock_ttl: u64, + #[prost(enumeration = "AssertionLevel", tag = "8")] + pub assertion_level: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushResponse { + #[prost(message, optional, tag = "1")] + pub region_error: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub errors: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "3")] + pub exec_details_v2: ::core::option::Option, +} +/// BufferBatchGet is introduced from the pipelined DML protocol. +/// It is similar to a BatchGet request, except that it can only read the data that has been flushed by itself. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BufferBatchGetRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(bytes = "vec", repeated, tag = "2")] + pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + #[prost(uint64, tag = "3")] + pub version: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BufferBatchGetResponse { + #[prost(message, optional, tag = "1")] + pub region_error: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub error: ::core::option::Option, + #[prost(message, repeated, tag = "3")] + pub pairs: ::prost::alloc::vec::Vec, + /// Time and scan details when processing the request. + #[prost(message, optional, tag = "4")] + pub exec_details_v2: ::core::option::Option, +} +/// Actively request TiKV to report health feedback information. TiKV won't omit the health feedback information when sending the +/// `BatchCommandsResponse` that contains this response. +/// The health feedback information won't be replied in the response, but will be attached to `BatchCommandsResponse.health_feedback` field as usual. +/// Only works when batch RPC is enabled. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetHealthFeedbackRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetHealthFeedbackResponse { + /// The error field is added for keeping consistent. This request won't meet any region error as it's store level rather than region level. + #[prost(message, optional, tag = "1")] + pub region_error: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub health_feedback: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HealthFeedback { + #[prost(uint64, tag = "1")] + pub store_id: u64, + /// The sequence number of the feedback message. + /// It's defined as an incrementing integer, starting from the unix timestamp (milliseconds) at + /// the time that the TiKV node is started. + /// This can be useful for filtering out out-of-order feedback messages. + /// Note that considering the possibility of system clock changing, this field doesn't guarantee + /// uniqueness and monotonic if the TiKV node is restarted. + #[prost(uint64, tag = "2")] + pub feedback_seq_no: u64, + /// The slow_score calculated in raftstore module. Due to some limitations of slow score, this would + /// be replaced by `SlowTrend` in the future. + #[prost(int32, tag = "3")] + pub slow_score: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BroadcastTxnStatusRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub txn_status: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxnStatus { + #[prost(uint64, tag = "1")] + pub start_ts: u64, + /// a non-zero min_commit_ts indicates the transaction is ongoing + #[prost(uint64, tag = "2")] + pub min_commit_ts: u64, + /// a non-zero commit_ts indicates the transaction is committed + #[prost(uint64, tag = "3")] + pub commit_ts: u64, + #[prost(bool, tag = "4")] + pub rolled_back: bool, + /// The txn has unlocked all keys, implying that it can be removed from txn_status_cache. + #[prost(bool, tag = "5")] + pub is_completed: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BroadcastTxnStatusResponse {} /// Used to specify the behavior when a pessimistic lock request is woken up after waiting for another /// lock. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] @@ -2163,6 +2470,65 @@ impl PessimisticLockKeyResultType { } } } +/// CommitRole indicates the current commit request is a primary commit or a secondary commit. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum CommitRole { + /// Unknown about the current request is a primary commit or a secondary commit; + Unknown = 0, + /// Current request is a primary commit; + Primary = 1, + /// Current request is a secondary commit; + Secondary = 2, +} +impl CommitRole { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + CommitRole::Unknown => "Unknown", + CommitRole::Primary => "Primary", + CommitRole::Secondary => "Secondary", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Unknown" => Some(Self::Unknown), + "Primary" => Some(Self::Primary), + "Secondary" => Some(Self::Secondary), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum RequestOrigin { + Unknown = 0, + TiDb = 1, +} +impl RequestOrigin { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + RequestOrigin::Unknown => "RequestOriginUnknown", + RequestOrigin::TiDb => "RequestOriginTiDB", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "RequestOriginUnknown" => Some(Self::Unknown), + "RequestOriginTiDB" => Some(Self::TiDb), + _ => None, + } + } +} /// The API version the server and the client is using. /// See more details in #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] @@ -2344,8 +2710,13 @@ pub enum Op { Rollback = 3, /// insert operation has a constraint that key should not exist before. Insert = 4, + /// PessimisticLock is exclusive lock acquired in pessimistic transaction. PessimisticLock = 5, CheckNotExists = 6, + /// SharedLock likes Lock but in shared mode. + SharedLock = 7, + /// SharedPessimisticLock is shared lock acquired in pessimistic transaction. + SharedPessimisticLock = 8, } impl Op { /// String value of the enum field names used in the ProtoBuf definition. @@ -2361,6 +2732,8 @@ impl Op { Op::Insert => "Insert", Op::PessimisticLock => "PessimisticLock", Op::CheckNotExists => "CheckNotExists", + Op::SharedLock => "SharedLock", + Op::SharedPessimisticLock => "SharedPessimisticLock", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -2373,6 +2746,8 @@ impl Op { "Insert" => Some(Self::Insert), "PessimisticLock" => Some(Self::PessimisticLock), "CheckNotExists" => Some(Self::CheckNotExists), + "SharedLock" => Some(Self::SharedLock), + "SharedPessimisticLock" => Some(Self::SharedPessimisticLock), _ => None, } } diff --git a/src/generated/logbackup.rs b/src/generated/logbackup.rs index 51b3cf5f..ecc33476 100644 --- a/src/generated/logbackup.rs +++ b/src/generated/logbackup.rs @@ -54,6 +54,25 @@ pub struct FlushEvent { #[prost(uint64, tag = "3")] pub checkpoint: u64, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushNowRequest {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushNowResponse { + #[prost(message, repeated, tag = "1")] + pub results: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushResult { + #[prost(string, tag = "1")] + pub task_name: ::prost::alloc::string::String, + #[prost(bool, tag = "2")] + pub success: bool, + #[prost(string, tag = "3")] + pub error_message: ::prost::alloc::string::String, +} /// Generated client implementations. pub mod log_backup_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -195,5 +214,30 @@ pub mod log_backup_client { .insert(GrpcMethod::new("logbackup.LogBackup", "SubscribeFlushEvent")); self.inner.server_streaming(req, path, codec).await } + pub async fn flush_now( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/logbackup.LogBackup/FlushNow", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("logbackup.LogBackup", "FlushNow")); + self.inner.unary(req, path, codec).await + } } } diff --git a/src/generated/meta_storagepb.rs b/src/generated/meta_storagepb.rs index 1355ebd6..a0c67c24 100644 --- a/src/generated/meta_storagepb.rs +++ b/src/generated/meta_storagepb.rs @@ -109,6 +109,37 @@ pub struct PutResponse { /// copied from etcd #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(bytes = "vec", tag = "2")] + pub key: ::prost::alloc::vec::Vec, + /// range_end is the key following the last key to delete for the range \[key, range_end). + /// If range_end is not given, the range is defined to contain only the key argument. + /// If range_end is one bit larger than the given key, then the range is all the keys + /// with the prefix (the given key). + /// If range_end is '\0', the range is all keys greater than or equal to the key argument. + #[prost(bytes = "vec", tag = "3")] + pub range_end: ::prost::alloc::vec::Vec, + /// If prev_kv is set, etcd gets the previous key-value pairs before deleting it. + /// The previous key-value pairs will be returned in the delete response. + #[prost(bool, tag = "4")] + pub prev_kv: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + /// deleted is the number of keys deleted by the delete range request. + #[prost(int64, tag = "2")] + pub deleted: i64, + #[prost(message, repeated, tag = "3")] + pub prev_kvs: ::prost::alloc::vec::Vec, +} +/// copied from etcd +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct KeyValue { /// key is the key in bytes. An empty key is not allowed. #[prost(bytes = "vec", tag = "1")] @@ -379,5 +410,29 @@ pub mod meta_storage_client { .insert(GrpcMethod::new("meta_storagepb.MetaStorage", "Put")); self.inner.unary(req, path, codec).await } + /// Delete is the same as etcd Range which might be implemented in a more common way + /// so that we can use other storages to replace etcd in the future. + pub async fn delete( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/meta_storagepb.MetaStorage/Delete", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("meta_storagepb.MetaStorage", "Delete")); + self.inner.unary(req, path, codec).await + } } } diff --git a/src/generated/metapb.rs b/src/generated/metapb.rs index 8839fb0f..7ee897eb 100644 --- a/src/generated/metapb.rs +++ b/src/generated/metapb.rs @@ -110,6 +110,16 @@ pub struct Buckets { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct BucketMeta { + /// A hint indicate if keys have changed. + #[prost(uint64, tag = "1")] + pub version: u64, + /// keys of buckets, include start/end key of region + #[prost(bytes = "vec", repeated, tag = "2")] + pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Region { #[prost(uint64, tag = "1")] pub id: u64, diff --git a/src/generated/mpp.rs b/src/generated/mpp.rs index e68ab94b..2ee84475 100644 --- a/src/generated/mpp.rs +++ b/src/generated/mpp.rs @@ -50,6 +50,10 @@ pub struct TaskMeta { /// This is the session alias between a client and tidb #[prost(string, tag = "19")] pub connection_alias: ::prost::alloc::string::String, + #[prost(string, tag = "20")] + pub sql_digest: ::prost::alloc::string::String, + #[prost(string, tag = "21")] + pub plan_digest: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -80,6 +84,9 @@ pub struct DispatchTaskRequest { /// Used for partition table scan #[prost(message, repeated, tag = "6")] pub table_regions: ::prost::alloc::vec::Vec, + /// Shard infos for TiCI/FTS routing in MPP dispatch path. + #[prost(message, repeated, tag = "7")] + pub table_shard_infos: ::prost::alloc::vec::Vec, } /// Get response of DispatchTaskRequest. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/src/generated/pdpb.rs b/src/generated/pdpb.rs index 548b333c..77187056 100644 --- a/src/generated/pdpb.rs +++ b/src/generated/pdpb.rs @@ -71,8 +71,17 @@ pub struct RequestHeader { #[prost(uint64, tag = "1")] pub cluster_id: u64, /// sender_id is the ID of the sender server, also member ID or etcd ID. + /// sender_id is used in PD internal communication. #[prost(uint64, tag = "2")] pub sender_id: u64, + /// caller_id is the ID of the client which sends the request, such as tikv, + /// tidb, cdc, etc. + #[prost(string, tag = "3")] + pub caller_id: ::prost::alloc::string::String, + /// caller_component is the component of the client which sends the request, + /// such as ddl, optimizer, etc. + #[prost(string, tag = "4")] + pub caller_component: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -162,6 +171,8 @@ pub struct IsBootstrappedResponse { pub struct AllocIdRequest { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, + #[prost(uint32, tag = "2")] + pub count: u32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -170,6 +181,8 @@ pub struct AllocIdResponse { pub header: ::core::option::Option, #[prost(uint64, tag = "2")] pub id: u64, + #[prost(uint32, tag = "3")] + pub count: u32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -280,6 +293,57 @@ pub struct GetRegionByIdRequest { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct QueryRegionRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + /// Whether to include the buckets info within the response. + #[prost(bool, tag = "2")] + pub need_buckets: bool, + /// The region IDs to query. + #[prost(uint64, repeated, tag = "3")] + pub ids: ::prost::alloc::vec::Vec, + /// The region keys to query. + #[prost(bytes = "vec", repeated, tag = "4")] + pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// The previous region keys to query. + #[prost(bytes = "vec", repeated, tag = "5")] + pub prev_keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct QueryRegionResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + /// This array functions as a map corresponding to the region IDs, + /// preserving the order of the input region keys, if they are present. + #[prost(uint64, repeated, tag = "2")] + pub key_id_map: ::prost::alloc::vec::Vec, + /// This array functions as a map corresponding to the previous region IDs, + /// preserving the order of the input previous region keys, if they are present. + #[prost(uint64, repeated, tag = "3")] + pub prev_key_id_map: ::prost::alloc::vec::Vec, + /// RegionID -> RegionResponse + #[prost(map = "uint64, message", tag = "4")] + pub regions_by_id: ::std::collections::HashMap, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionResponse { + #[prost(message, optional, tag = "1")] + pub region: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub leader: ::core::option::Option, + #[prost(message, repeated, tag = "3")] + pub down_peers: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "4")] + pub pending_peers: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "5")] + pub buckets: ::core::option::Option, +} +/// Use GetRegionResponse as the response of GetRegionByIDRequest. +/// Deprecated: use BatchScanRegionsRequest instead. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ScanRegionsRequest { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -306,6 +370,9 @@ pub struct Region { /// working followers. #[prost(message, repeated, tag = "4")] pub pending_peers: ::prost::alloc::vec::Vec, + /// buckets isn't nil only when need_buckets is true. + #[prost(message, optional, tag = "5")] + pub buckets: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -323,6 +390,44 @@ pub struct ScanRegionsResponse { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyRange { + #[prost(bytes = "vec", tag = "1")] + pub start_key: ::prost::alloc::vec::Vec, + /// end_key is +inf when it is empty. + #[prost(bytes = "vec", tag = "2")] + pub end_key: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BatchScanRegionsRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(bool, tag = "2")] + pub need_buckets: bool, + /// the given ranges must be in order. + #[prost(message, repeated, tag = "3")] + pub ranges: ::prost::alloc::vec::Vec, + /// limit the total number of regions to scan. + #[prost(int32, tag = "4")] + pub limit: i32, + /// If contain_all_key_range is true, the output must contain all + /// key ranges in the request. + /// If the output does not contain all key ranges, the request is considered + /// failed and returns an error(REGIONS_NOT_CONTAIN_ALL_KEY_RANGE). + #[prost(bool, tag = "5")] + pub contain_all_key_range: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BatchScanRegionsResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + /// the returned regions are flattened into a list, because the given ranges can located in the same range, we do not return duplicated regions then. + #[prost(message, repeated, tag = "2")] + pub regions: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct GetClusterConfigRequest { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -468,14 +573,27 @@ pub struct RegionHeartbeatRequest { /// QueryStats reported write query stats, and there are read query stats in store heartbeat #[prost(message, optional, tag = "16")] pub query_stats: ::core::option::Option, - /// cpu_usage is the CPU time usage of the leader region since the last heartbeat, + /// cpu_usage is the total CPU time usage of the leader region since the last heartbeat, /// which is calculated by cpu_time_delta/heartbeat_reported_interval. + /// Deprecated: use cpu_stats instead. + #[deprecated] #[prost(uint64, tag = "17")] pub cpu_usage: u64, - /// (Serverless) Approximate size of key-value pairs for billing. + /// cpu_stats reports CPU usage breakdown for the leader region by kind + /// (e.g. unified read). + #[prost(message, optional, tag = "21")] + pub cpu_stats: ::core::option::Option, + /// Approximate size of row-based key-value pairs for billing. /// It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. #[prost(uint64, tag = "18")] pub approximate_kv_size: u64, + /// Approximate size of column-based key-value pairs for billing. + /// It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. + #[prost(uint64, tag = "19")] + pub approximate_columnar_kv_size: u64, + /// BucketMeta is the bucket version and keys of this region if TiKV enabled the bucket feature + #[prost(message, optional, tag = "20")] + pub bucket_meta: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -530,6 +648,13 @@ pub struct BatchSwitchWitness { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct ChangeSplit { + /// auto_split_enabled configures whether the corresponding Region is allowed to be auto split by size or load. + #[prost(bool, tag = "1")] + pub auto_split_enabled: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct RegionHeartbeatResponse { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -576,6 +701,8 @@ pub struct RegionHeartbeatResponse { pub change_peer_v2: ::core::option::Option, #[prost(message, optional, tag = "10")] pub switch_witnesses: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub change_split: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -624,6 +751,8 @@ pub struct AskBatchSplitRequest { pub region: ::core::option::Option, #[prost(uint32, tag = "3")] pub split_count: u32, + #[prost(enumeration = "SplitReason", tag = "4")] + pub reason: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -688,6 +817,10 @@ pub struct PeerStat { pub written_keys: u64, #[prost(uint64, tag = "6")] pub written_bytes: u64, + /// cpu_stats is the CPU usage of the region's unified read pool since the last heartbeat, + /// which is calculated by cpu_time_delta/heartbeat_reported_interval. + #[prost(message, optional, tag = "7")] + pub cpu_stats: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -777,6 +910,41 @@ pub struct StoreStats { /// Used memory of the store in bytes. #[prost(uint64, tag = "29")] pub used_memory: u64, + /// Network_slow_scores indicate the network status between TiKV nodes, ranging from 1 to 100 (lower is better). + /// StoreID -> score + #[prost(map = "uint64, uint64", tag = "30")] + pub network_slow_scores: ::std::collections::HashMap, + /// The statistics about DFS uploads. + #[prost(message, repeated, tag = "31")] + pub dfs: ::prost::alloc::vec::Vec, + /// True if the store is undergoing graceful shutdown. + #[prost(bool, tag = "32")] + pub is_stopping: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DfsStatScope { + /// When true, the statistic is not tied to any keyspace. + #[prost(bool, tag = "1")] + pub is_global: bool, + /// The keyspace of this statistic. Ignore when is_global is true. + #[prost(uint32, tag = "2")] + pub keyspace_id: u32, + /// The component that provides the statistic. + #[prost(string, tag = "3")] + pub component: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DfsStatItem { + #[prost(message, optional, tag = "1")] + pub scope: ::core::option::Option, + /// Number of bytes written to DFS. + #[prost(uint64, tag = "2")] + pub written_bytes: u64, + /// Number of write requests sent to DFS. + #[prost(uint64, tag = "3")] + pub write_requests: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -822,6 +990,9 @@ pub struct PeerReport { /// The peer has proposed but uncommitted commit merge. #[prost(bool, tag = "4")] pub has_commit_merge: bool, + /// raft applied index + #[prost(uint64, tag = "5")] + pub applied_index: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -928,6 +1099,9 @@ pub struct StoreHeartbeatResponse { /// Pd can return operations to let TiKV forcely PAUSE | RESUME grpc server. #[prost(message, optional, tag = "7")] pub control_grpc: ::core::option::Option, + /// NodeState is going to mark the state of the store. + #[prost(enumeration = "super::metapb::NodeState", tag = "8")] + pub state: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -962,6 +1136,8 @@ pub struct ScatterRegionResponse { pub header: ::core::option::Option, #[prost(uint64, tag = "2")] pub finished_percentage: u64, + #[prost(uint64, repeated, tag = "3")] + pub failed_regions_id: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1135,6 +1311,206 @@ pub struct GetAllGcSafePointV2Response { #[prost(int64, tag = "3")] pub revision: i64, } +/// A wrapper over keyspace_id. +/// When a field is not specified in proto3, its value will be regarded as 0; however, keyspace_id = 0 is regarded as a valid keyspace (which +/// is the "DEFAULT" keyspace). To distinguish unspecified keyspace (NullKeyspace, 0xffffffff) and the default keyspace in some APIs as well +/// as preventing potential misuse, we wrap the keyspace_id into a message type which is nullable. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyspaceScope { + #[prost(uint32, tag = "1")] + pub keyspace_id: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdvanceGcSafePointRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub target: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdvanceGcSafePointResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub old_gc_safe_point: u64, + #[prost(uint64, tag = "3")] + pub new_gc_safe_point: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdvanceTxnSafePointRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub target: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdvanceTxnSafePointResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub old_txn_safe_point: u64, + #[prost(uint64, tag = "3")] + pub new_txn_safe_point: u64, + #[prost(string, tag = "4")] + pub blocker_description: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetGcBarrierRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(string, tag = "3")] + pub barrier_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "4")] + pub barrier_ts: u64, + #[prost(int64, tag = "5")] + pub ttl_seconds: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GcBarrierInfo { + #[prost(string, tag = "1")] + pub barrier_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub barrier_ts: u64, + #[prost(int64, tag = "3")] + pub ttl_seconds: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetGcBarrierResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub new_barrier_info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteGcBarrierRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(string, tag = "3")] + pub barrier_id: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteGcBarrierResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub deleted_barrier_info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetGlobalGcBarrierRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(string, tag = "2")] + pub barrier_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "3")] + pub barrier_ts: u64, + #[prost(int64, tag = "4")] + pub ttl_seconds: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetGlobalGcBarrierResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub new_barrier_info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteGlobalGcBarrierRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(string, tag = "2")] + pub barrier_id: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteGlobalGcBarrierResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub deleted_barrier_info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GlobalGcBarrierInfo { + #[prost(string, tag = "1")] + pub barrier_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub barrier_ts: u64, + #[prost(int64, tag = "3")] + pub ttl_seconds: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetGcStateRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(bool, tag = "3")] + pub exclude_gc_barriers: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GcState { + #[prost(message, optional, tag = "1")] + pub keyspace_scope: ::core::option::Option, + #[prost(bool, tag = "2")] + pub is_keyspace_level_gc: bool, + #[prost(uint64, tag = "3")] + pub txn_safe_point: u64, + #[prost(uint64, tag = "4")] + pub gc_safe_point: u64, + #[prost(message, repeated, tag = "5")] + pub gc_barriers: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetGcStateResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub gc_state: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetAllKeyspacesGcStatesRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(bool, tag = "2")] + pub exclude_gc_barriers: bool, + #[prost(bool, tag = "3")] + pub exclude_global_gc_barriers: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetAllKeyspacesGcStatesResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub gc_states: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub global_gc_barriers: ::prost::alloc::vec::Vec, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct RegionStat { @@ -1330,6 +1706,17 @@ pub struct QueryStats { #[prost(uint64, tag = "11")] pub rollback: u64, } +/// CPU usage breakdown by kind. New kinds may be added in the future. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CpuStats { + /// UnifiedRead is the CPU usage of the unified read pool. + #[prost(uint64, tag = "1")] + pub unified_read: u64, + /// Scheduler is the CPU usage of the scheduler pool, it contains `sched-pool`, `sched-high`, and `sched-pri`. + #[prost(uint64, tag = "2")] + pub scheduler: u64, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ReportBucketsRequest { @@ -1446,6 +1833,7 @@ pub enum ErrorType { InvalidValue = 10, /// required watch revision is smaller than current compact/min revision. DataCompacted = 11, + RegionsNotContainAllKeyRange = 12, } impl ErrorType { /// String value of the enum field names used in the ProtoBuf definition. @@ -1466,6 +1854,9 @@ impl ErrorType { ErrorType::EntryNotFound => "ENTRY_NOT_FOUND", ErrorType::InvalidValue => "INVALID_VALUE", ErrorType::DataCompacted => "DATA_COMPACTED", + ErrorType::RegionsNotContainAllKeyRange => { + "REGIONS_NOT_CONTAIN_ALL_KEY_RANGE" + } } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -1483,6 +1874,9 @@ impl ErrorType { "ENTRY_NOT_FOUND" => Some(Self::EntryNotFound), "INVALID_VALUE" => Some(Self::InvalidValue), "DATA_COMPACTED" => Some(Self::DataCompacted), + "REGIONS_NOT_CONTAIN_ALL_KEY_RANGE" => { + Some(Self::RegionsNotContainAllKeyRange) + } _ => None, } } @@ -1547,6 +1941,35 @@ impl CheckPolicy { } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] +pub enum SplitReason { + Admin = 0, + Size = 1, + Load = 2, +} +impl SplitReason { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + SplitReason::Admin => "ADMIN", + SplitReason::Size => "SIZE", + SplitReason::Load => "LOAD", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "ADMIN" => Some(Self::Admin), + "SIZE" => Some(Self::Size), + "LOAD" => Some(Self::Load), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] pub enum ControlGrpcEvent { /// Pause TiKV grpc server. Pause = 0, @@ -2088,6 +2511,31 @@ pub mod pd_client { req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "GetRegionByID")); self.inner.unary(req, path, codec).await } + pub async fn query_region( + &mut self, + request: impl tonic::IntoStreamingRequest< + Message = super::QueryRegionRequest, + >, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/QueryRegion"); + let mut req = request.into_streaming_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "QueryRegion")); + self.inner.streaming(req, path, codec).await + } + /// Deprecated: use BatchScanRegions instead. pub async fn scan_regions( &mut self, request: impl tonic::IntoRequest, @@ -2110,6 +2558,28 @@ pub mod pd_client { req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "ScanRegions")); self.inner.unary(req, path, codec).await } + pub async fn batch_scan_regions( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/BatchScanRegions"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "BatchScanRegions")); + self.inner.unary(req, path, codec).await + } pub async fn ask_split( &mut self, request: impl tonic::IntoRequest, @@ -2457,6 +2927,197 @@ pub mod pd_client { .insert(GrpcMethod::new("pdpb.PD", "GetAllGCSafePointV2")); self.inner.unary(req, path, codec).await } + pub async fn advance_gc_safe_point( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/AdvanceGCSafePoint", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "AdvanceGCSafePoint")); + self.inner.unary(req, path, codec).await + } + pub async fn advance_txn_safe_point( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/AdvanceTxnSafePoint", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "AdvanceTxnSafePoint")); + self.inner.unary(req, path, codec).await + } + pub async fn set_gc_barrier( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/SetGCBarrier"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "SetGCBarrier")); + self.inner.unary(req, path, codec).await + } + pub async fn delete_gc_barrier( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/DeleteGCBarrier"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "DeleteGCBarrier")); + self.inner.unary(req, path, codec).await + } + pub async fn set_global_gc_barrier( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/SetGlobalGCBarrier", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "SetGlobalGCBarrier")); + self.inner.unary(req, path, codec).await + } + pub async fn delete_global_gc_barrier( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/DeleteGlobalGCBarrier", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "DeleteGlobalGCBarrier")); + self.inner.unary(req, path, codec).await + } + pub async fn get_gc_state( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/GetGCState"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "GetGCState")); + self.inner.unary(req, path, codec).await + } + pub async fn get_all_keyspaces_gc_states( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/GetAllKeyspacesGCStates", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "GetAllKeyspacesGCStates")); + self.inner.unary(req, path, codec).await + } pub async fn sync_regions( &mut self, request: impl tonic::IntoStreamingRequest, diff --git a/src/generated/raft_cmdpb.rs b/src/generated/raft_cmdpb.rs index eb178070..1fc1e66f 100644 --- a/src/generated/raft_cmdpb.rs +++ b/src/generated/raft_cmdpb.rs @@ -312,6 +312,8 @@ pub struct CommitMergeRequest { /// Used in v2. When it's present, `source` and `commit` will not be set. #[prost(message, optional, tag = "4")] pub source_state: ::core::option::Option, + #[prost(bytes = "vec", tag = "100")] + pub source_meta: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -515,6 +517,12 @@ pub struct RaftResponseHeader { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct CustomRequest { + #[prost(bytes = "vec", tag = "1")] + pub data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct RaftCmdRequest { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -526,6 +534,8 @@ pub struct RaftCmdRequest { pub admin_request: ::core::option::Option, #[prost(message, optional, tag = "4")] pub status_request: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub custom_request: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/src/generated/raft_serverpb.rs b/src/generated/raft_serverpb.rs index f6b1f942..c215bcc4 100644 --- a/src/generated/raft_serverpb.rs +++ b/src/generated/raft_serverpb.rs @@ -392,6 +392,10 @@ pub struct ExtraMessage { /// notice the peer to refresh buckets version #[prost(message, optional, tag = "9")] pub refresh_buckets: ::core::option::Option, + /// snap_gen_precheck_passed is used to indicate the precheck result when + /// a follower responds to a leader's snapshot gen precheck request. + #[prost(bool, tag = "10")] + pub snap_gen_precheck_passed: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] @@ -453,6 +457,15 @@ pub enum ExtraMessageType { MsgGcPeerResponse = 12, MsgFlushMemtable = 13, MsgRefreshBuckets = 14, + /// Messages for the snapshot gen precheck process. + MsgSnapGenPrecheckRequest = 15, + MsgSnapGenPrecheckResponse = 16, + /// Used in transfer leader process for leader to inform follower to load the + /// region into in-memory engine if the relevant region is cached. + MsgPreLoadRegionRequest = 17, + /// Used in transfer leader process for follower to inform leader the completes + /// of the region cache + MsgPreLoadRegionResponse = 18, } impl ExtraMessageType { /// String value of the enum field names used in the ProtoBuf definition. @@ -482,6 +495,10 @@ impl ExtraMessageType { ExtraMessageType::MsgGcPeerResponse => "MsgGcPeerResponse", ExtraMessageType::MsgFlushMemtable => "MsgFlushMemtable", ExtraMessageType::MsgRefreshBuckets => "MsgRefreshBuckets", + ExtraMessageType::MsgSnapGenPrecheckRequest => "MsgSnapGenPrecheckRequest", + ExtraMessageType::MsgSnapGenPrecheckResponse => "MsgSnapGenPrecheckResponse", + ExtraMessageType::MsgPreLoadRegionRequest => "MsgPreLoadRegionRequest", + ExtraMessageType::MsgPreLoadRegionResponse => "MsgPreLoadRegionResponse", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -508,6 +525,10 @@ impl ExtraMessageType { "MsgGcPeerResponse" => Some(Self::MsgGcPeerResponse), "MsgFlushMemtable" => Some(Self::MsgFlushMemtable), "MsgRefreshBuckets" => Some(Self::MsgRefreshBuckets), + "MsgSnapGenPrecheckRequest" => Some(Self::MsgSnapGenPrecheckRequest), + "MsgSnapGenPrecheckResponse" => Some(Self::MsgSnapGenPrecheckResponse), + "MsgPreLoadRegionRequest" => Some(Self::MsgPreLoadRegionRequest), + "MsgPreLoadRegionResponse" => Some(Self::MsgPreLoadRegionResponse), _ => None, } } diff --git a/src/generated/resource_manager.rs b/src/generated/resource_manager.rs index caf4b116..e093c6f0 100644 --- a/src/generated/resource_manager.rs +++ b/src/generated/resource_manager.rs @@ -1,7 +1,26 @@ // This file is @generated by prost-build. +/// KeyspaceIDValue is a wrapper for the value of keyspace ID. +/// Because the 0 value is a valid keyspace ID, we need to use a wrapper to distinguish it from the null keyspace ID. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ListResourceGroupsRequest {} +pub struct KeyspaceIdValue { + #[prost(uint32, tag = "1")] + pub value: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListResourceGroupsRequest { + #[prost(bool, tag = "1")] + pub with_ru_stats: bool, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, the listed resource groups will be filtered + /// by the given keyspace ID. + #[prost(message, optional, tag = "2")] + pub keyspace_id: ::core::option::Option, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ListResourceGroupsResponse { @@ -15,6 +34,16 @@ pub struct ListResourceGroupsResponse { pub struct GetResourceGroupRequest { #[prost(string, tag = "1")] pub resource_group_name: ::prost::alloc::string::String, + #[prost(bool, tag = "2")] + pub with_ru_stats: bool, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, it will try to get the resource group within + /// the given keyspace ID. + #[prost(message, optional, tag = "3")] + pub keyspace_id: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -29,6 +58,14 @@ pub struct GetResourceGroupResponse { pub struct DeleteResourceGroupRequest { #[prost(string, tag = "1")] pub resource_group_name: ::prost::alloc::string::String, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, it will try to delete the resource group within + /// the given keyspace ID. + #[prost(message, optional, tag = "2")] + pub keyspace_id: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -75,6 +112,14 @@ pub struct TokenBucketRequest { pub is_background: bool, #[prost(bool, tag = "6")] pub is_tiflash: bool, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, it will try to request the token bucket from + /// the resource group within the given keyspace ID. + #[prost(message, optional, tag = "7")] + pub keyspace_id: ::core::option::Option, #[prost(oneof = "token_bucket_request::Request", tags = "2, 3")] pub request: ::core::option::Option, } @@ -122,6 +167,14 @@ pub struct TokenBucketResponse { /// Raw mode #[prost(message, repeated, tag = "3")] pub granted_resource_tokens: ::prost::alloc::vec::Vec, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version, + /// which can be safely ignored to keep compatibility. + /// * If the keyspace ID is set to a valid value, it means this response is from the resource + /// group within this keyspace ID. + #[prost(message, optional, tag = "4")] + pub keyspace_id: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -162,6 +215,19 @@ pub struct Consumption { pub kv_read_rpc_count: f64, #[prost(double, tag = "8")] pub kv_write_rpc_count: f64, + #[prost(uint64, tag = "9")] + pub read_cross_az_traffic_bytes: u64, + #[prost(uint64, tag = "10")] + pub write_cross_az_traffic_bytes: u64, + /// RUv2 is an experimental v2 RU calculation. + /// For now it only records the consumption without actual token deduction. + #[prost(double, tag = "11")] + pub tikv_r_u_v2: f64, + #[prost(double, tag = "12")] + pub tidb_r_u_v2: f64, + /// tiflash_r_u_v2 equals tiflash RRU + tiflash WRU. + #[prost(double, tag = "13")] + pub tiflash_r_u_v2: f64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -199,8 +265,20 @@ pub struct ResourceGroup { /// Runaway queries settings #[prost(message, optional, tag = "6")] pub runaway_settings: ::core::option::Option, + /// Background task control settings. #[prost(message, optional, tag = "7")] pub background_settings: ::core::option::Option, + /// RU consumption statistics. + #[prost(message, optional, tag = "8")] + pub ru_stats: ::core::option::Option, + /// The keyspace ID that the resource group belongs to. + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, it will directly be used. + #[prost(message, optional, tag = "9")] + pub keyspace_id: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -248,6 +326,10 @@ pub struct Error { pub struct RunawayRule { #[prost(uint64, tag = "1")] pub exec_elapsed_time_ms: u64, + #[prost(int64, tag = "2")] + pub processed_keys: i64, + #[prost(int64, tag = "3")] + pub request_unit: i64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -267,12 +349,20 @@ pub struct RunawaySettings { pub action: i32, #[prost(message, optional, tag = "3")] pub watch: ::core::option::Option, + /// When the runaway action is `SwitchGroup`, + /// this field will be used to indicate which group to switch. + #[prost(string, tag = "4")] + pub switch_group_name: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BackgroundSettings { + /// background task types. #[prost(string, repeated, tag = "1")] pub job_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// the percentage limit of total resource(cpu/io) that background tasks can use. + #[prost(uint64, tag = "2")] + pub utilization_limit: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -379,6 +469,8 @@ pub enum RunawayAction { CoolDown = 2, /// kill the task Kill = 3, + /// switch the task to another group + SwitchGroup = 4, } impl RunawayAction { /// String value of the enum field names used in the ProtoBuf definition. @@ -391,6 +483,7 @@ impl RunawayAction { RunawayAction::DryRun => "DryRun", RunawayAction::CoolDown => "CoolDown", RunawayAction::Kill => "Kill", + RunawayAction::SwitchGroup => "SwitchGroup", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -400,6 +493,7 @@ impl RunawayAction { "DryRun" => Some(Self::DryRun), "CoolDown" => Some(Self::CoolDown), "Kill" => Some(Self::Kill), + "SwitchGroup" => Some(Self::SwitchGroup), _ => None, } } diff --git a/src/generated/resource_usage_agent.rs b/src/generated/resource_usage_agent.rs index a8f7f5fd..26f04ac6 100644 --- a/src/generated/resource_usage_agent.rs +++ b/src/generated/resource_usage_agent.rs @@ -8,7 +8,7 @@ pub struct EmptyResponse {} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ResourceUsageRecord { - #[prost(oneof = "resource_usage_record::RecordOneof", tags = "1")] + #[prost(oneof = "resource_usage_record::RecordOneof", tags = "1, 2")] pub record_oneof: ::core::option::Option, } /// Nested message and enum types in `ResourceUsageRecord`. @@ -18,6 +18,8 @@ pub mod resource_usage_record { pub enum RecordOneof { #[prost(message, tag = "1")] Record(super::GroupTagRecord), + #[prost(message, tag = "2")] + RegionRecord(super::RegionRecord), } } /// GroupTagRecord is a set of resource usage data grouped by resource_group_tag. @@ -40,6 +42,23 @@ pub struct GroupTagRecordItem { pub read_keys: u32, #[prost(uint32, tag = "4")] pub write_keys: u32, + #[prost(uint64, tag = "5")] + pub network_in_bytes: u64, + #[prost(uint64, tag = "6")] + pub network_out_bytes: u64, + #[prost(uint64, tag = "7")] + pub logical_read_bytes: u64, + #[prost(uint64, tag = "8")] + pub logical_write_bytes: u64, +} +/// RegionRecord is a set of resource usage data grouped by region. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionRecord { + #[prost(uint64, tag = "1")] + pub region_id: u64, + #[prost(message, repeated, tag = "2")] + pub items: ::prost::alloc::vec::Vec, } /// Generated client implementations. pub mod resource_usage_agent_client { diff --git a/src/generated/schedulingpb.rs b/src/generated/schedulingpb.rs index 5c6d5de3..e4efe144 100644 --- a/src/generated/schedulingpb.rs +++ b/src/generated/schedulingpb.rs @@ -98,6 +98,9 @@ pub struct RegionHeartbeatRequest { /// Actually reported time interval #[prost(message, optional, tag = "14")] pub interval: ::core::option::Option, + /// BucketMeta is the bucket version and keys of this region if TiKV enabled the bucket feature + #[prost(message, optional, tag = "15")] + pub bucket_meta: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -147,6 +150,8 @@ pub struct RegionHeartbeatResponse { pub change_peer_v2: ::core::option::Option, #[prost(message, optional, tag = "10")] pub switch_witnesses: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub change_split: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -172,6 +177,8 @@ pub struct ScatterRegionsResponse { pub header: ::core::option::Option, #[prost(uint64, tag = "2")] pub finished_percentage: u64, + #[prost(uint64, repeated, tag = "3")] + pub failed_regions_id: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -224,6 +231,8 @@ pub struct AskBatchSplitRequest { pub region: ::core::option::Option, #[prost(uint32, tag = "3")] pub split_count: u32, + #[prost(enumeration = "super::pdpb::SplitReason", tag = "4")] + pub reason: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -233,6 +242,22 @@ pub struct AskBatchSplitResponse { #[prost(message, repeated, tag = "2")] pub ids: ::prost::alloc::vec::Vec, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionBucketsRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub region_epoch: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub buckets: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionBucketsResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum ErrorType { @@ -508,5 +533,32 @@ pub mod scheduling_client { .insert(GrpcMethod::new("schedulingpb.Scheduling", "AskBatchSplit")); self.inner.unary(req, path, codec).await } + pub async fn region_buckets( + &mut self, + request: impl tonic::IntoStreamingRequest< + Message = super::RegionBucketsRequest, + >, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/schedulingpb.Scheduling/RegionBuckets", + ); + let mut req = request.into_streaming_request(); + req.extensions_mut() + .insert(GrpcMethod::new("schedulingpb.Scheduling", "RegionBuckets")); + self.inner.streaming(req, path, codec).await + } } } diff --git a/src/generated/tikvpb.rs b/src/generated/tikvpb.rs index e27b1cbf..f85e37d4 100644 --- a/src/generated/tikvpb.rs +++ b/src/generated/tikvpb.rs @@ -6,6 +6,10 @@ pub struct BatchCommandsRequest { pub requests: ::prost::alloc::vec::Vec, #[prost(uint64, repeated, tag = "2")] pub request_ids: ::prost::alloc::vec::Vec, + /// Unix epoch timestamp in nanoseconds recorded by the client when this + /// batch request is sent to TiKV. + #[prost(uint64, tag = "3")] + pub client_send_time_ns: u64, } /// Nested message and enum types in `BatchCommandsRequest`. pub mod batch_commands_request { @@ -14,7 +18,7 @@ pub mod batch_commands_request { pub struct Request { #[prost( oneof = "request::Cmd", - tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 33, 34, 35, 36, 255" + tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 33, 34, 35, 36, 37, 38, 39, 40, 255" )] pub cmd: ::core::option::Option, } @@ -89,6 +93,14 @@ pub mod batch_commands_request { PrepareFlashbackToVersion( super::super::super::kvrpcpb::PrepareFlashbackToVersionRequest, ), + #[prost(message, tag = "37")] + Flush(super::super::super::kvrpcpb::FlushRequest), + #[prost(message, tag = "38")] + BufferBatchGet(super::super::super::kvrpcpb::BufferBatchGetRequest), + #[prost(message, tag = "39")] + GetHealthFeedback(super::super::super::kvrpcpb::GetHealthFeedbackRequest), + #[prost(message, tag = "40")] + BroadcastTxnStatus(super::super::super::kvrpcpb::BroadcastTxnStatusRequest), /// For some test cases. #[prost(message, tag = "255")] Empty(super::super::BatchCommandsEmptyRequest), @@ -105,6 +117,12 @@ pub struct BatchCommandsResponse { /// 280 means TiKV gRPC cpu usage is 280%. #[prost(uint64, tag = "3")] pub transport_layer_load: u64, + #[prost(message, optional, tag = "4")] + pub health_feedback: ::core::option::Option, + /// Unix epoch timestamp in nanoseconds recorded by TiKV when this batch + /// response is ready to be sent back to the client. + #[prost(uint64, tag = "5")] + pub tikv_send_time_ns: u64, } /// Nested message and enum types in `BatchCommandsResponse`. pub mod batch_commands_response { @@ -113,7 +131,7 @@ pub mod batch_commands_response { pub struct Response { #[prost( oneof = "response::Cmd", - tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 33, 34, 35, 36, 255" + tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 33, 34, 35, 36, 37, 38, 39, 40, 255" )] pub cmd: ::core::option::Option, } @@ -188,6 +206,14 @@ pub mod batch_commands_response { PrepareFlashbackToVersion( super::super::super::kvrpcpb::PrepareFlashbackToVersionResponse, ), + #[prost(message, tag = "37")] + Flush(super::super::super::kvrpcpb::FlushResponse), + #[prost(message, tag = "38")] + BufferBatchGet(super::super::super::kvrpcpb::BufferBatchGetResponse), + #[prost(message, tag = "39")] + GetHealthFeedback(super::super::super::kvrpcpb::GetHealthFeedbackResponse), + #[prost(message, tag = "40")] + BroadcastTxnStatus(super::super::super::kvrpcpb::BroadcastTxnStatusResponse), /// For some test cases. #[prost(message, tag = "255")] Empty(super::super::BatchCommandsEmptyResponse), @@ -199,6 +225,9 @@ pub mod batch_commands_response { pub struct BatchRaftMessage { #[prost(message, repeated, tag = "1")] pub msgs: ::prost::alloc::vec::Vec, + /// Used for measure the send duration. + #[prost(uint64, tag = "13")] + pub last_observed_time: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -762,6 +791,55 @@ pub mod tikv_client { .insert(GrpcMethod::new("tikvpb.Tikv", "KvFlashbackToVersion")); self.inner.unary(req, path, codec).await } + pub async fn kv_flush( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/tikvpb.Tikv/KvFlush"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("tikvpb.Tikv", "KvFlush")); + self.inner.unary(req, path, codec).await + } + pub async fn kv_buffer_batch_get( + &mut self, + request: impl tonic::IntoRequest< + super::super::kvrpcpb::BufferBatchGetRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/KvBufferBatchGet", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "KvBufferBatchGet")); + self.inner.unary(req, path, codec).await + } /// Raw commands; no transaction support. pub async fn raw_get( &mut self, @@ -1256,6 +1334,32 @@ pub mod tikv_client { .insert(GrpcMethod::new("tikvpb.Tikv", "BatchCoprocessor")); self.inner.server_streaming(req, path, codec).await } + /// Command send by remote coprocessor to TiKV for executing coprocessor request. + pub async fn delegate_coprocessor( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/DelegateCoprocessor", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "DelegateCoprocessor")); + self.inner.unary(req, path, codec).await + } /// Command for executing custom user requests in TiKV coprocessor_v2. pub async fn raw_coprocessor( &mut self, @@ -1796,6 +1900,34 @@ pub mod tikv_client { .insert(GrpcMethod::new("tikvpb.Tikv", "GetTiFlashSystemTable")); self.inner.unary(req, path, codec).await } + /// / Get estimate tici count from TiFlash + pub async fn get_estimate_ti_ci_count( + &mut self, + request: impl tonic::IntoRequest< + super::super::coprocessor::TiCiEstimateCountRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/GetEstimateTiCICount", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "GetEstimateTiCICount")); + self.inner.unary(req, path, codec).await + } /// These are for TiFlash disaggregated architecture /// / Try to lock a S3 object, atomically pub async fn try_add_lock( @@ -1963,5 +2095,179 @@ pub mod tikv_client { .insert(GrpcMethod::new("tikvpb.Tikv", "GetDisaggConfig")); self.inner.unary(req, path, codec).await } + /// / Get health feedback info from the TiKV node. + pub async fn get_health_feedback( + &mut self, + request: impl tonic::IntoRequest< + super::super::kvrpcpb::GetHealthFeedbackRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/GetHealthFeedback", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "GetHealthFeedback")); + self.inner.unary(req, path, codec).await + } + /// / Broadcast the transaction status to all TiKV nodes + pub async fn broadcast_txn_status( + &mut self, + request: impl tonic::IntoRequest< + super::super::kvrpcpb::BroadcastTxnStatusRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/BroadcastTxnStatus", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "BroadcastTxnStatus")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated client implementations. +pub mod versioned_kv_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + /// VersionedKv provides versioned coprocessor APIs for TiCI lookup. + /// + /// Invariants: + /// + /// * For `VersionedCoprocessor`, callers should fill `coprocessor.Request.versioned_ranges` + /// (each `VersionedKeyRange.range` must be a point range) and keep `coprocessor.Request.ranges` empty. + #[derive(Debug, Clone)] + pub struct VersionedKvClient { + inner: tonic::client::Grpc, + } + impl VersionedKvClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl VersionedKvClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> VersionedKvClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + VersionedKvClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn versioned_coprocessor( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.VersionedKv/VersionedCoprocessor", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.VersionedKv", "VersionedCoprocessor")); + self.inner.unary(req, path, codec).await + } } } diff --git a/src/generated/tsopb.rs b/src/generated/tsopb.rs index 61a3c6f5..fd84ab11 100644 --- a/src/generated/tsopb.rs +++ b/src/generated/tsopb.rs @@ -14,6 +14,12 @@ pub struct RequestHeader { /// keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. #[prost(uint32, tag = "4")] pub keyspace_group_id: u32, + /// callee_id is the ID of the server which the client expects to receive the request. + /// such as tso-0, tso-1, pd-0, pd-1 etc. + /// This field is used to check if the request is sent to the expected server. + /// If it is not matched, the server will return an error. + #[prost(string, tag = "5")] + pub callee_id: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -105,6 +111,8 @@ pub struct FindGroupByKeyspaceIdRequest { pub header: ::core::option::Option, #[prost(uint32, tag = "2")] pub keyspace_id: u32, + #[prost(uint64, tag = "3")] + pub mod_revision: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -113,6 +121,8 @@ pub struct FindGroupByKeyspaceIdResponse { pub header: ::core::option::Option, #[prost(message, optional, tag = "2")] pub keyspace_group: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub mod_revision: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/src/pd/timestamp.rs b/src/pd/timestamp.rs index a1cc7fbd..30cfd8ee 100644 --- a/src/pd/timestamp.rs +++ b/src/pd/timestamp.rs @@ -156,7 +156,7 @@ impl Stream for TsoRequestStream { let req = TsoRequest { header: Some(RequestHeader { cluster_id: *this.cluster_id, - sender_id: 0, + ..Default::default() }), count: requests.len() as u32, dc_location: String::new(), From 6e07351a5eecc56d2176c4c157c0188959f330a7 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Sat, 25 Jul 2026 13:50:45 +0200 Subject: [PATCH 2/2] transaction: refuse to resolve shared locks instead of mis-handling them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-vendored kvproto exposes shared locks (Op::SharedLock / Op::SharedPessimisticLock). Their contract is unusual: a shared lock's real holders live ONLY in kvrpcpb.LockInfo.shared_lock_infos — the proto comment is explicit that you must "DO NOT read from the wrapper LockInfo", whose own key and lock_version are unset. This client does not implement shared-lock resolution, and every partial handling of one is worse than none: resolving the wrapper checks transaction 0; filtering on the wrapper's fields (use_async_commit, lock_type) silently drops the real members; and the pessimistic-lock special cases in the resolver do not know SharedPessimisticLock. So resolution REFUSES them with an explicit error — in resolve_locks, in LockResolver's crate-public entry point, and in CleanupLocks::execute before any filter runs. Until real support lands, an explicit error is the only answer that cannot roll back a live transaction or skip a dead one. Servers that predate shared locks never produce them, so this is a no-op there. The API-v2 keyspace codecs are made shared-lock-aware separately, because they run on scan results that never reach the resolver. They now take the shape of client-go's codecV2.decodeLockInfo (internal/apicodec/codec_v2.go), which decodes a LockInfo's own Key/PrimaryLock/Secondaries and then recurses into SharedLockInfos — it has no wrapper special case. This client now does the same: convert the lock's OWN key fields, then recurse into the members. An earlier revision skipped a wrapper's own fields entirely, reading the proto's "DO NOT read from the wrapper LockInfo" as covering them. Checking the writer (TiKV's SharedLocks::into_lock_info in components/txn_types/src/lock.rs, identical at v8.5.7 and on master) shows that is only half right, and the half it gets wrong matters: info.set_shared_lock_infos(shared_locks.into()); info.set_key(raw_key); A wrapper sets lock_type, shared_lock_infos and key — the key it locks — and leaves primary_lock and lock_version at their defaults; each member is built from that SAME raw key plus its own primary and version. So the caveat scopes the per-transaction fields, not the locked key. Skipping the wrapper wholesale would have handed scan_locks a physical key beside decoded member keys, while converting it wholesale would have hit the length assertion on the unset primary. Both fields are exercised by tests. The two directions guard differently, on purpose. Truncating, empty bytes can only mean "unset" — an encoded key always carries its 4-byte prefix — so unset fields are skipped rather than run into pretruncate_bytes' length assertion. Encoding, the input is a LOGICAL key and the empty logical key is valid in API v2, so nothing is skipped: scan_locks -> resolve_locks round-trips locks through truncate-then-encode, and skipping empties there would strand a lock on the empty key with no prefix and send resolution after an empty physical key. Wrappers need no encode-side guard because resolution refuses them first. That panic hazard predates the re-vendor: a wrapper arrives the same way whichever proto vintage parses it. Full shared-lock support is deliberately follow-up work. Split out of the kvproto re-vendor at pingyu's suggestion (#550). Tests: 4 new — the resolver refusal (a plain lock passes; a wrapper carrying members and a lock marked shared only by its op are both refused); the codec converting a wrapper's own key alongside its members; a wrapper's unset fields surviving truncation (the other reading, so both are pinned); and a lock on the empty logical key round-tripping through truncate-then-encode. 71 lib tests green. The coverage is unit-level by necessity: CI pins TIKV_VERSION v8.5.5, which predates shared locks, so no integration test in this repo can produce one, and the refusal path is unreachable against that server. v8.5.6 is the first release carrying shared locks, so a CI pin bump within the same release family would make these paths reachable — left to a separate change. Signed-off-by: Eduard R. --- src/request/keyspace.rs | 157 ++++++++++++++++++++++++++++++++++++++-- src/request/plan.rs | 4 + src/transaction/lock.rs | 62 ++++++++++++++++ src/transaction/mod.rs | 1 + 4 files changed, 218 insertions(+), 6 deletions(-) diff --git a/src/request/keyspace.rs b/src/request/keyspace.rs index 79f7225d..366e995c 100644 --- a/src/request/keyspace.rs +++ b/src/request/keyspace.rs @@ -160,23 +160,57 @@ impl TruncateKeyspace for Vec { } } +/// Is this key field UNSET, as opposed to carrying the empty logical key? +/// +/// An encoded key always carries its 4-byte keyspace prefix, so on the wire an empty +/// byte string means "not set" — never "the empty logical key", which encodes to the +/// bare prefix. Guarding on this keeps the codecs off `pretruncate_bytes`' length +/// assertion for fields a shared-lock wrapper leaves unset, and mirrors client-go's +/// `codecV2.DecodeKey`, which returns early on a zero-length key. +fn is_unset_key(key: &[u8]) -> bool { + key.is_empty() +} + impl TruncateKeyspace for Vec { fn truncate_keyspace(mut self, keyspace: Keyspace) -> Self { if !matches!(keyspace, Keyspace::Enable { .. }) { return self; } for lock in &mut self { - take_mut::take(&mut lock.key, |key| { - Key::from(key).truncate_keyspace(keyspace).into() - }); - take_mut::take(&mut lock.primary_lock, |primary| { - Key::from(primary).truncate_keyspace(keyspace).into() - }); + // Convert this lock's OWN key fields, then recurse into any shared-lock + // members — the shape of client-go's `codecV2.decodeLockInfo`, which + // decodes Key/PrimaryLock/Secondaries and *then* walks SharedLockInfos, + // with no wrapper special case. + // + // Per TiKV's writer (`SharedLocks::into_lock_info`, txn_types/src/lock.rs) + // a wrapper sets `lock_type`, `shared_lock_infos` and `key` — the key it + // locks — and leaves `primary_lock`/`lock_version` at their defaults. Each + // member is built from that SAME raw key plus its own primary and version. + // So the wrapper's key must be converted like any other (skipping it would + // hand `scan_locks` a physical key beside decoded member keys), while its + // unset fields must be left alone — hence no wrapper special case, just the + // per-field guard below. + if !is_unset_key(&lock.key) { + take_mut::take(&mut lock.key, |key| { + Key::from(key).truncate_keyspace(keyspace).into() + }); + } + if !is_unset_key(&lock.primary_lock) { + take_mut::take(&mut lock.primary_lock, |primary| { + Key::from(primary).truncate_keyspace(keyspace).into() + }); + } for secondary in lock.secondaries.iter_mut() { + if is_unset_key(secondary) { + continue; + } take_mut::take(secondary, |secondary| { Key::from(secondary).truncate_keyspace(keyspace).into() }); } + take_mut::take(&mut lock.shared_lock_infos, |members| { + members.truncate_keyspace(keyspace) + }); } self } @@ -188,6 +222,17 @@ impl EncodeKeyspace for Vec { return self; } for lock in &mut self { + // Deliberately NOT symmetric with the TruncateKeyspace impl above. There, + // empty bytes can only mean "unset", because an encoded key always carries + // its 4-byte prefix. Here the input is a LOGICAL key, and the empty logical + // key is valid in API v2 — it encodes to the bare prefix. Skipping empties + // would strand a lock on the empty key with no prefix, and `scan_locks` -> + // `resolve_locks` (transaction/client.rs) round-trips exactly that way, so + // its region lookup would then use an empty physical key. + // + // Shared-lock wrappers do not need the unset-field guard here: resolution + // refuses them (`reject_shared_locks`) before anything acts on the encoded + // result. take_mut::take(&mut lock.key, |key| { Key::from(key).encode_keyspace(keyspace, key_mode).into() }); @@ -203,6 +248,9 @@ impl EncodeKeyspace for Vec { .into() }); } + take_mut::take(&mut lock.shared_lock_infos, |members| { + members.encode_keyspace(keyspace, key_mode) + }); } self } @@ -477,4 +525,101 @@ mod tests { let locks = vec![lock]; assert_eq!(locks.clone().truncate_keyspace(keyspace), locks); } + + /// A shared-lock wrapper carries the key it locks, and its members are built from + /// that same raw key (TiKV's `SharedLocks::into_lock_info`). Both must be converted: + /// skipping the wrapper would hand `scan_locks` a physical key beside decoded member + /// keys. The wrapper's *other* fields are a different matter — see + /// [`unset_lock_key_fields_survive_truncation`]. + #[test] + fn shared_lock_wrapper_keys_are_converted_alongside_their_members() { + use crate::proto::kvrpcpb::{LockInfo, Op}; + let keyspace = Keyspace::Enable { keyspace_id: 0 }; + + let wrapper = LockInfo { + key: vec![b'x', 0, 0, 0, b'k'], + lock_type: Op::SharedLock as i32, + shared_lock_infos: vec![LockInfo { + key: vec![b'x', 0, 0, 0, b'm'], + primary_lock: vec![b'x', 0, 0, 0, b'p'], + lock_version: 8, + ..Default::default() + }], + ..Default::default() + }; + + let out = vec![wrapper].truncate_keyspace(keyspace); + assert_eq!( + out[0].key, + vec![b'k'], + "the wrapper's own key must be decoded, not left physical" + ); + assert_eq!(out[0].shared_lock_infos[0].key, vec![b'm']); + assert_eq!(out[0].shared_lock_infos[0].primary_lock, vec![b'p']); + + let back = out.encode_keyspace(keyspace, KeyMode::Txn); + assert_eq!(back[0].key, vec![b'x', 0, 0, 0, b'k']); + assert_eq!(back[0].shared_lock_infos[0].key, vec![b'x', 0, 0, 0, b'm']); + } + + /// The empty logical key is VALID in API v2: it encodes to the bare keyspace + /// prefix. `scan_locks` -> `resolve_locks` round-trips locks through + /// truncate-then-encode, so a lock on the empty key must regain its prefix — + /// otherwise resolution would look up the region for an empty physical key. + /// This is why the encode side does not share the truncate side's unset guard. + #[test] + fn a_lock_on_the_empty_logical_key_round_trips() { + use crate::proto::kvrpcpb::LockInfo; + let keyspace = Keyspace::Enable { keyspace_id: 0 }; + + let physical = vec![LockInfo { + key: vec![b'x', 0, 0, 0], + primary_lock: vec![b'x', 0, 0, 0], + ..Default::default() + }]; + let logical = physical.clone().truncate_keyspace(keyspace); + assert!(logical[0].key.is_empty(), "the empty logical key"); + + let back = logical.encode_keyspace(keyspace, KeyMode::Txn); + assert_eq!( + back, physical, + "a lock on the empty logical key must regain its keyspace prefix" + ); + } + + /// TiKV's `SharedLocks::into_lock_info` leaves a wrapper's `primary_lock` at its + /// default, so the codec meets genuinely unset fields in practice — they must be + /// skipped, not run into `pretruncate_bytes`' length assertion. + #[test] + fn unset_lock_key_fields_survive_truncation() { + use crate::proto::kvrpcpb::{LockInfo, Op}; + let keyspace = Keyspace::Enable { keyspace_id: 0 }; + + // The writer's actual shape: key and members set, primary_lock left default. + let realistic = LockInfo { + key: vec![b'x', 0, 0, 0, b'k'], + lock_type: Op::SharedLock as i32, + shared_lock_infos: vec![LockInfo { + key: vec![b'x', 0, 0, 0, b'k'], + primary_lock: vec![b'x', 0, 0, 0, b'p'], + ..Default::default() + }], + ..Default::default() + }; + let out = vec![realistic].truncate_keyspace(keyspace); + assert_eq!(out[0].key, vec![b'k']); + assert!( + out[0].primary_lock.is_empty(), + "the wrapper's unset primary must be skipped, not truncated" + ); + assert_eq!(out[0].shared_lock_infos[0].primary_lock, vec![b'p']); + + // Defensively, a wrapper with no key at all must not panic either. + let keyless = LockInfo { + lock_type: Op::SharedLock as i32, + ..Default::default() + }; + let out = vec![keyless].truncate_keyspace(keyspace); + assert!(out[0].key.is_empty()); + } } diff --git a/src/request/plan.rs b/src/request/plan.rs index b7fac56f..6ef78f93 100644 --- a/src/request/plan.rs +++ b/src/request/plan.rs @@ -841,6 +841,10 @@ where has_more_batch = false; } + // BEFORE any filter: a shared-lock wrapper's fields (including + // `use_async_commit`) must not be read — filtering on them would silently + // drop the real member locks. Refuse instead; see `reject_shared_locks`. + crate::transaction::reject_shared_locks(&locks)?; if self.options.async_commit_only { locks = locks .into_iter() diff --git a/src/transaction/lock.rs b/src/transaction/lock.rs index 4a8adb63..98bb3683 100644 --- a/src/transaction/lock.rs +++ b/src/transaction/lock.rs @@ -43,6 +43,35 @@ pub(crate) fn format_key_for_log(key: &[u8]) -> String { format!("len={}, prefix={}", key.len(), HexRepr(&key[..prefix_len])) } +/// Refuse to resolve SHARED locks — loudly, before any of them can be mis-handled. +/// +/// The contract (`kvrpcpb.LockInfo.shared_lock_infos`) is explicit: a shared lock's +/// real holders live ONLY in `shared_lock_infos` — "DO NOT read from the wrapper +/// LockInfo", whose own `key`/`lock_version` are unset. This client does not implement +/// shared-lock resolution yet, and every partial handling is worse than none: +/// resolving the wrapper checks transaction 0; filtering on wrapper fields silently +/// drops the members; and the pessimistic-lock special cases in this resolver do not +/// know `SharedPessimisticLock`. Until support lands, an explicit error is the only +/// answer that cannot roll back a live transaction or skip a dead one. +/// +/// Servers that predate shared locks never produce them, so this is a no-op there. +pub(crate) fn reject_shared_locks(locks: &[kvrpcpb::LockInfo]) -> Result<()> { + let shared = |l: &kvrpcpb::LockInfo| { + !l.shared_lock_infos.is_empty() + || l.lock_type == kvrpcpb::Op::SharedLock as i32 + || l.lock_type == kvrpcpb::Op::SharedPessimisticLock as i32 + }; + if locks.iter().any(shared) { + return Err(Error::StringError( + "shared locks (SharedLock/SharedPessimisticLock) are not supported by this \ + client yet; refusing to resolve them — resolving the wrapper would target \ + the wrong transaction" + .to_owned(), + )); + } + Ok(()) +} + /// _Resolves_ the given locks. Returns locks still live. When there is no live locks, all the given locks are resolved. /// /// If a key has a lock, the latest status of the key is unknown. We need to "resolve" the lock, @@ -56,6 +85,7 @@ pub async fn resolve_locks( keyspace: Keyspace, ) -> Result /* live_locks */> { debug!("resolving locks"); + reject_shared_locks(&locks)?; let ts = pd_client.clone().get_timestamp().await?; let caller_start_ts = timestamp.version(); let current_ts = ts.version(); @@ -300,6 +330,9 @@ impl LockResolver { pd_client: Arc, // TODO: make pd_client a member of LockResolver keyspace: Keyspace, ) -> Result<()> { + // Defense in depth: CleanupLocks::execute refuses these before its filters, + // but this entry point is public within the crate. + reject_shared_locks(&locks)?; if locks.is_empty() { return Ok(()); } @@ -619,6 +652,35 @@ mod tests { use crate::mock::MockPdClient; use crate::proto::errorpb; + #[test] + fn shared_locks_are_refused_never_misresolved() { + let plain = kvrpcpb::LockInfo { + key: b"k1".to_vec(), + lock_version: 7, + ..Default::default() + }; + assert!(reject_shared_locks(std::slice::from_ref(&plain)).is_ok()); + + // A wrapper: key/lock_version deliberately unset per the contract — resolving + // it would check transaction 0. Must be refused, not resolved or filtered. + let wrapper = kvrpcpb::LockInfo { + shared_lock_infos: vec![kvrpcpb::LockInfo { + key: b"k2".to_vec(), + lock_version: 8, + ..Default::default() + }], + ..Default::default() + }; + assert!(reject_shared_locks(&[plain.clone(), wrapper]).is_err()); + + // Also refused when only the op marks it shared (empty member list). + let by_op = kvrpcpb::LockInfo { + lock_type: kvrpcpb::Op::SharedPessimisticLock as i32, + ..Default::default() + }; + assert!(reject_shared_locks(&[by_op]).is_err()); + } + #[rstest::rstest] #[case(Keyspace::Disable)] #[case(Keyspace::Enable { keyspace_id: 0 })] diff --git a/src/transaction/mod.rs b/src/transaction/mod.rs index 1f2cc181..573992b0 100644 --- a/src/transaction/mod.rs +++ b/src/transaction/mod.rs @@ -28,6 +28,7 @@ mod client; mod lock; pub mod lowering; mod requests; +pub(crate) use lock::reject_shared_locks; pub use lock::LockResolver; pub use lock::ResolveLocksContext; pub use lock::ResolveLocksOptions;