From 12f2aa84925234b1844e079ad45df36b703f2d57 Mon Sep 17 00:00:00 2001 From: ekexium Date: Wed, 24 Mar 2021 14:26:03 +0800 Subject: [PATCH] update proto files Signed-off-by: ekexium --- src/timestamp.rs | 2 + tikv-client-pd/src/timestamp.rs | 1 + tikv-client-proto/proto/backup.proto | 20 ++++++ tikv-client-proto/proto/coprocessor_v2.proto | 33 ++++++++++ tikv-client-proto/proto/encryptionpb.proto | 4 ++ tikv-client-proto/proto/import_sstpb.proto | 5 ++ tikv-client-proto/proto/kvrpcpb.proto | 68 ++++++++++++++++++++ tikv-client-proto/proto/metapb.proto | 2 + tikv-client-proto/proto/pdpb.proto | 18 ++++-- tikv-client-proto/proto/raft_cmdpb.proto | 3 + tikv-client-proto/proto/raft_serverpb.proto | 6 ++ tikv-client-proto/proto/tikvpb.proto | 19 +++++- 12 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 tikv-client-proto/proto/coprocessor_v2.proto diff --git a/src/timestamp.rs b/src/timestamp.rs index 720be77..a3a9a09 100644 --- a/src/timestamp.rs +++ b/src/timestamp.rs @@ -35,6 +35,8 @@ impl TimestampExt for Timestamp { Self { physical: version >> PHYSICAL_SHIFT_BITS, logical: version & LOGICAL_MASK, + // We only support global transactions + suffix_bits: 0, } } diff --git a/tikv-client-pd/src/timestamp.rs b/tikv-client-pd/src/timestamp.rs index da31b87..56a095d 100644 --- a/tikv-client-pd/src/timestamp.rs +++ b/tikv-client-pd/src/timestamp.rs @@ -207,6 +207,7 @@ fn allocate_timestamps( let ts = Timestamp { physical: tail_ts.physical, logical: tail_ts.logical - offset as i64, + suffix_bits: tail_ts.get_suffix_bits(), }; let _ = request.send(ts); } diff --git a/tikv-client-proto/proto/backup.proto b/tikv-client-proto/proto/backup.proto index b739ec4..a31914a 100644 --- a/tikv-client-proto/proto/backup.proto +++ b/tikv-client-proto/proto/backup.proto @@ -43,6 +43,9 @@ message BackupMeta { // In incremental backup, DDLs which are completed in (lastBackupTS, backupTS] will be stored here. bytes ddls = 10; + + // Save the version of BR running backup jobs. + string br_version = 11; } message File { @@ -140,6 +143,7 @@ message StorageBackend { Local local = 2; S3 s3 = 3; GCS gcs = 4; + CloudDynamic cloud_dynamic = 5; } } @@ -184,6 +188,22 @@ message GCS { string credentials_blob = 6; } +message Bucket { + string endpoint = 1; + string region = 3; + string bucket = 4; + string prefix = 5; + string storage_class = 6; +} + +// CloudDynamic allows testing new cloud providers and new fields without changing protobuf definitions +message CloudDynamic { + Bucket bucket = 1; + string provider_name = 2; // s3 and gcs are supported + map attrs = 3; +} + + message BackupResponse { Error error = 1; bytes start_key = 2; diff --git a/tikv-client-proto/proto/coprocessor_v2.proto b/tikv-client-proto/proto/coprocessor_v2.proto new file mode 100644 index 0000000..b321a1f --- /dev/null +++ b/tikv-client-proto/proto/coprocessor_v2.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; +package coprocessor_v2; + +import "errorpb.proto"; +import "kvrpcpb.proto"; +import "gogoproto/gogo.proto"; +import "rustproto.proto"; + +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; + +option java_package = "org.tikv.kvproto"; + +message RawCoprocessorRequest { + kvrpcpb.Context context = 1; + + string copr_name = 2; + string copr_version_constraint = 3; + + bytes data = 4; +} + +message RawCoprocessorResponse { + bytes data = 1; + + errorpb.Error region_error = 2; + + // Error message for cases like if no coprocessor with a matching name is found + // or on a version mismatch between plugin_api and the coprocessor. + string other_error = 4; +} diff --git a/tikv-client-proto/proto/encryptionpb.proto b/tikv-client-proto/proto/encryptionpb.proto index 880bc5d..ddae704 100644 --- a/tikv-client-proto/proto/encryptionpb.proto +++ b/tikv-client-proto/proto/encryptionpb.proto @@ -1,3 +1,7 @@ +// These encryption protobufs are not sent over the network. +// Protobufs are used to define a stable backwards compatible persistent storage format. +// These definitions are used by both PD and TiKV to keep their implementations similar. + syntax = "proto3"; package encryptionpb; diff --git a/tikv-client-proto/proto/import_sstpb.proto b/tikv-client-proto/proto/import_sstpb.proto index 7824528..c3a9d4a 100644 --- a/tikv-client-proto/proto/import_sstpb.proto +++ b/tikv-client-proto/proto/import_sstpb.proto @@ -76,6 +76,11 @@ message SSTMeta { uint64 region_id = 6; metapb.RegionEpoch region_epoch = 7; bool end_key_exclusive = 8; + + // total_kvs and total_bytes is equivalent to PD's approximate_keys and approximate_size + // set these values can save time from tikv upload keys and size to PD through Heartbeat. + uint64 total_kvs = 9; + uint64 total_bytes = 10; } // A rewrite rule is applied on the *encoded* keys (the internal storage diff --git a/tikv-client-proto/proto/kvrpcpb.proto b/tikv-client-proto/proto/kvrpcpb.proto index 1ea1b12..308c761 100644 --- a/tikv-client-proto/proto/kvrpcpb.proto +++ b/tikv-client-proto/proto/kvrpcpb.proto @@ -216,6 +216,14 @@ message CheckTxnStatusRequest { // If true, then TiKV will leave a rollback tombstone in the write CF for `primary_key`, even if // that key is not locked. bool rollback_if_not_exist = 6; + // This field is set to true only if the transaction is known to fall back from async commit. + // Then, CheckTxnStatus treats the transaction as non-async-commit even if the use_async_commit + // field in the primary lock is true. + bool force_sync_commit = 7; + // If the check request is used to resolve or decide the transaction status for a input pessimistic + // lock, the transaction status could not be decided if the primary lock is pessimistic too and + // it's still uncertain. + bool resolving_pessimistic_lock = 8; } message CheckTxnStatusResponse { @@ -349,6 +357,8 @@ message ScanLockRequest { bytes start_key = 3; // The maximum number of locks to return. uint32 limit = 4; + // The exclusive upperbound for scanning. + bytes end_key = 5; } message ScanLockResponse { @@ -435,6 +445,8 @@ message RawPutRequest { bytes key = 2; bytes value = 3; string cf = 4; + uint64 ttl = 5; + bool for_cas = 6; } message RawPutResponse { @@ -446,6 +458,8 @@ message RawBatchPutRequest { Context context = 1; repeated KvPair pairs = 2; string cf = 3; + uint64 ttl = 4; + bool for_cas = 5; } message RawBatchPutResponse { @@ -457,6 +471,7 @@ message RawDeleteRequest { Context context = 1; bytes key = 2; string cf = 3; + bool for_cas = 4; } message RawDeleteResponse { @@ -468,6 +483,7 @@ message RawBatchDeleteRequest { Context context = 1; repeated bytes keys = 2; string cf = 3; + bool for_cas = 4; } message RawBatchDeleteResponse { @@ -759,6 +775,10 @@ message Context { // A hint for TiKV to schedule tasks more fairly. Query with same task ID // may share same priority and resource quota. uint64 task_id = 16; + + // Not required to read the most up-to-date data, replicas with `safe_ts` >= `start_ts` + // can handle read request directly + bool stale_read = 17; } message LockInfo { @@ -843,11 +863,14 @@ message TimeDetail { // cannot be excluded for now, like Mutex wait time, which is included in this field, so that // this field is called wall time instead of CPU time. int64 process_wall_time_ms = 2; + // KV read wall Time means the time used in key/value scan and get. + int64 kv_read_wall_time_ms = 3; } message ScanInfo { int64 total = 1; int64 processed = 2; + int64 read_bytes = 3; } // Only reserved for compatibility. @@ -974,6 +997,8 @@ enum Action { TTLExpireRollback = 1; LockNotExistRollback = 2; MinCommitTSPushed = 3; + TTLExpirePessimisticRollback = 4; + LockNotExistDoNothing = 5; } message KeyRange { @@ -1003,3 +1028,46 @@ message CheckLeaderResponse { repeated uint64 regions = 1; uint64 ts = 2; } + +message StoreSafeTSRequest { + // Get the minimal `safe_ts` from regions that overlap with the key range [`start_key`, `end_key`) + // An empty key range means all regions in the store + KeyRange key_range = 1; +} + +message StoreSafeTSResponse { + uint64 safe_ts = 1; +} + +message RawGetKeyTTLRequest { + Context context = 1; + bytes key = 2; + string cf = 3; +} + +message RawGetKeyTTLResponse { + errorpb.Error region_error = 1; + string error = 2; + uint64 ttl = 3; + bool not_found = 4; +} + +message RawCASRequest { + Context context = 1; + bytes key = 2; + bytes value = 3; + bool previous_not_exist = 4; + bytes previous_value = 5; + string cf = 6; + uint64 ttl = 7; +} + +message RawCASResponse { + errorpb.Error region_error = 1; + string error = 2; + bool succeed = 3; + // The previous value regardless of whether the comparison is succeed. + bool previous_not_exist = 4; + bytes previous_value = 5; +} + diff --git a/tikv-client-proto/proto/metapb.proto b/tikv-client-proto/proto/metapb.proto index a26ba92..f59de84 100644 --- a/tikv-client-proto/proto/metapb.proto +++ b/tikv-client-proto/proto/metapb.proto @@ -50,6 +50,8 @@ message Store { string deploy_path = 10; // The last heartbeat timestamp of the store. int64 last_heartbeat = 11; + // If the store is physically destroyed, which means it can never up again. + bool physically_destroyed = 12; } message RegionEpoch { diff --git a/tikv-client-proto/proto/pdpb.proto b/tikv-client-proto/proto/pdpb.proto index 056b8ef..caba2ef 100644 --- a/tikv-client-proto/proto/pdpb.proto +++ b/tikv-client-proto/proto/pdpb.proto @@ -80,7 +80,7 @@ service PD { rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} - rpc GetDCLocations(GetDCLocationsRequest) returns (GetDCLocationsResponse) {} + rpc GetDCLocationInfo(GetDCLocationInfoRequest) returns (GetDCLocationInfoResponse) {} } message RequestHeader { @@ -121,7 +121,9 @@ message TsoRequest { message Timestamp { int64 physical = 1; int64 logical = 2; -} + // Number of suffix bits used for global distinction, + // PD client will use this to compute a TSO's logical part. + uint32 suffix_bits = 3;} message TsoResponse { ResponseHeader header = 1; @@ -286,6 +288,7 @@ message Member { string deploy_path = 6; string binary_version = 7; string git_hash = 8; + string dc_location = 9; } message GetMembersRequest { @@ -652,11 +655,16 @@ message SplitRegionsResponse { repeated uint64 regions_id = 3; } -message GetDCLocationsRequest { +message GetDCLocationInfoRequest { RequestHeader header = 1; + string dc_location = 2; } -message GetDCLocationsResponse { +message GetDCLocationInfoResponse { ResponseHeader header = 1; - repeated string dc_locations = 2; + // suffix sign + int32 suffix = 2; + // max_ts will be included into this response if PD leader think the receiver needs, + // which it's set when the number of the max suffix bits changes. + Timestamp max_ts = 3; } diff --git a/tikv-client-proto/proto/raft_cmdpb.proto b/tikv-client-proto/proto/raft_cmdpb.proto index 8c5ea5d..ca073c1 100644 --- a/tikv-client-proto/proto/raft_cmdpb.proto +++ b/tikv-client-proto/proto/raft_cmdpb.proto @@ -313,6 +313,9 @@ message RaftRequestHeader { // Read requests can be responsed directly after the Raft applys to `applied_index`. uint64 applied_index = 9; + + // Custom flags for this raft request. + uint64 flags = 10; } message RaftResponseHeader { diff --git a/tikv-client-proto/proto/raft_serverpb.proto b/tikv-client-proto/proto/raft_serverpb.proto index 01d9ab5..63191ad 100644 --- a/tikv-client-proto/proto/raft_serverpb.proto +++ b/tikv-client-proto/proto/raft_serverpb.proto @@ -38,6 +38,8 @@ message SnapshotCFFile { message SnapshotMeta { repeated SnapshotCFFile cf_files = 1; + // true means this snapshot is triggered for load balance + bool for_balance = 2; } message SnapshotChunk { @@ -102,6 +104,10 @@ enum ExtraMessageType { MsgWantRollbackMerge = 1; MsgCheckStalePeer = 2; MsgCheckStalePeerResponse = 3; + // If leader is going to sleep, it will send requests to all its followers + // to make sure they all agree to sleep. + MsgHibernateRequest = 4; + MsgHibernateResponse = 5; } message ExtraMessage { diff --git a/tikv-client-proto/proto/tikvpb.proto b/tikv-client-proto/proto/tikvpb.proto index 7f98606..d2f3f8a 100644 --- a/tikv-client-proto/proto/tikvpb.proto +++ b/tikv-client-proto/proto/tikvpb.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package tikvpb; import "coprocessor.proto"; +import "coprocessor_v2.proto"; import "kvrpcpb.proto"; import "mpp.proto"; import "raft_serverpb.proto"; @@ -47,6 +48,10 @@ service Tikv { rpc RawScan(kvrpcpb.RawScanRequest) returns (kvrpcpb.RawScanResponse) {} rpc RawDeleteRange(kvrpcpb.RawDeleteRangeRequest) returns (kvrpcpb.RawDeleteRangeResponse) {} rpc RawBatchScan(kvrpcpb.RawBatchScanRequest) returns (kvrpcpb.RawBatchScanResponse) {} + // Get TTL of the key. Returns 0 if TTL is not set for the key. + rpc RawGetKeyTTL(kvrpcpb.RawGetKeyTTLRequest) returns (kvrpcpb.RawGetKeyTTLResponse) {} + // Compare if the value in database equals to `RawCASRequest.previous_value` before putting the new value. If not, this request will have no effect and the value in the database will be returned. + rpc RawCompareAndSwap(kvrpcpb.RawCASRequest) returns (kvrpcpb.RawCASResponse) {} // VerKV commands. rpc VerGet(kvrpcpb.VerGetRequest) returns (kvrpcpb.VerGetResponse) {} @@ -66,9 +71,12 @@ service Tikv { // Commands for executing SQL in the TiKV coprocessor (i.e., 'pushed down' to TiKV rather than // executed in TiDB). rpc Coprocessor(coprocessor.Request) returns (coprocessor.Response) {} - rpc CoprocessorStream(coprocessor.Request) returns (stream coprocessor.Response) {} + rpc CoprocessorStream(coprocessor.Request) returns (stream coprocessor.Response) {} rpc BatchCoprocessor(coprocessor.BatchRequest) returns (stream coprocessor.BatchResponse) {} + // Command for executing custom user requests in TiKV coprocessor_v2. + rpc CoprocessorV2(coprocessor_v2.RawCoprocessorRequest) returns (coprocessor_v2.RawCoprocessorResponse) {} + // Raft commands (sent between TiKV nodes). rpc Raft(stream raft_serverpb.RaftMessage) returns (raft_serverpb.Done) {} rpc BatchRaft(stream BatchRaftMessage) returns (raft_serverpb.Done) {} @@ -96,7 +104,10 @@ service Tikv { /// term and epoch match with local information in the store. /// After the client collected all responses from all stores, it checks if got a quorum of responses from /// other stores for every region, and decides to advance resolved ts from these regions. - rpc CheckLeader(kvrpcpb.CheckLeaderRequest) returns(kvrpcpb.CheckLeaderResponse); + rpc CheckLeader(kvrpcpb.CheckLeaderRequest) returns (kvrpcpb.CheckLeaderResponse); + + /// Get the minimal `safe_ts` from regions at the store + rpc GetStoreSafeTS(kvrpcpb.StoreSafeTSRequest) returns (kvrpcpb.StoreSafeTSResponse); } message BatchCommandsRequest { @@ -144,6 +155,8 @@ message BatchCommandsRequest { kvrpcpb.CheckSecondaryLocksRequest CheckSecondaryLocks = 33; + coprocessor_v2.RawCoprocessorRequest CoprocessorV2 = 34; + // For some test cases. BatchCommandsEmptyRequest Empty = 255; } @@ -197,6 +210,8 @@ message BatchCommandsResponse { kvrpcpb.CheckSecondaryLocksResponse CheckSecondaryLocks = 33; + coprocessor_v2.RawCoprocessorResponse CoprocessorV2 = 34; + // For some test cases. BatchCommandsEmptyResponse Empty = 255; }