update proto files

Signed-off-by: ekexium <ekexium@gmail.com>

src/timestamp.rs

@@ -35,6 +35,8 @@ impl TimestampExt for Timestamp {
         Self {
             physical: version >> PHYSICAL_SHIFT_BITS,
             logical: version & LOGICAL_MASK,
+            // We only support global transactions
+            suffix_bits: 0,
         }
     }
 

tikv-client-pd/src/timestamp.rs

@@ -207,6 +207,7 @@ fn allocate_timestamps(
             let ts = Timestamp {
                 physical: tail_ts.physical,
                 logical: tail_ts.logical - offset as i64,
+                suffix_bits: tail_ts.get_suffix_bits(),
             };
             let _ = request.send(ts);
         }

tikv-client-proto/proto/backup.proto

@@ -43,6 +43,9 @@ message BackupMeta {
 
     // In incremental backup, DDLs which are completed in (lastBackupTS, backupTS] will be stored here.
     bytes ddls = 10;
+
+    // Save the version of BR running backup jobs.
+    string br_version = 11;
 }
 
 message File {
@@ -140,6 +143,7 @@ message StorageBackend {
         Local local = 2;
         S3 s3 = 3;
         GCS gcs = 4;
+        CloudDynamic cloud_dynamic = 5;
     }
 }
 
@@ -184,6 +188,22 @@ message GCS {
     string credentials_blob = 6;
 }
 
+message Bucket {
+    string endpoint = 1;
+    string region = 3;
+    string bucket = 4;
+    string prefix = 5;
+    string storage_class = 6;
+}
+
+// CloudDynamic allows testing new cloud providers and new fields without changing protobuf definitions
+message CloudDynamic {
+    Bucket bucket = 1;
+    string provider_name = 2; // s3 and gcs are supported
+    map<string, string> attrs = 3;
+}
+
+
 message BackupResponse {
     Error error = 1;
     bytes start_key = 2;

tikv-client-proto/proto/coprocessor_v2.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+package coprocessor_v2;
+
+import "errorpb.proto";
+import "kvrpcpb.proto";
+import "gogoproto/gogo.proto";
+import "rustproto.proto";
+
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (rustproto.lite_runtime_all) = true;
+
+option java_package = "org.tikv.kvproto";
+
+message RawCoprocessorRequest {
+    kvrpcpb.Context context = 1;
+
+    string copr_name = 2;
+    string copr_version_constraint = 3;
+
+    bytes data = 4;
+}
+
+message RawCoprocessorResponse {
+    bytes data = 1;
+
+    errorpb.Error region_error = 2;
+    
+    // Error message for cases like if no coprocessor with a matching name is found
+    // or on a version mismatch between plugin_api and the coprocessor.
+    string other_error = 4;
+}

tikv-client-proto/proto/encryptionpb.proto

@@ -1,3 +1,7 @@
+// These encryption protobufs are not sent over the network.
+// Protobufs are used to define a stable backwards compatible persistent storage format.
+// These definitions are used by both PD and TiKV to keep their implementations similar.
+
 syntax = "proto3";
 package encryptionpb;
 

tikv-client-proto/proto/import_sstpb.proto

@@ -76,6 +76,11 @@ message SSTMeta {
     uint64 region_id = 6;
     metapb.RegionEpoch region_epoch = 7;
     bool end_key_exclusive = 8;
+
+    // total_kvs and total_bytes is equivalent to PD's approximate_keys and approximate_size
+    // set these values can save time from tikv upload keys and size to PD through Heartbeat.
+    uint64 total_kvs = 9;
+    uint64 total_bytes = 10;
 }
 
 // A rewrite rule is applied on the *encoded* keys (the internal storage

tikv-client-proto/proto/kvrpcpb.proto

@@ -216,6 +216,14 @@ message CheckTxnStatusRequest {
     // If true, then TiKV will leave a rollback tombstone in the write CF for `primary_key`, even if
     // that key is not locked.
     bool rollback_if_not_exist = 6;
+    // This field is set to true only if the transaction is known to fall back from async commit.
+    // Then, CheckTxnStatus treats the transaction as non-async-commit even if the use_async_commit
+    // field in the primary lock is true.
+    bool force_sync_commit = 7;
+    // If the check request is used to resolve or decide the transaction status for a input pessimistic
+    // lock, the transaction status could not be decided if the primary lock is pessimistic too and
+    // it's still uncertain.
+    bool resolving_pessimistic_lock = 8;
 }
 
 message CheckTxnStatusResponse {
@@ -349,6 +357,8 @@ message ScanLockRequest {
     bytes start_key = 3;
     // The maximum number of locks to return.
     uint32 limit = 4;
+    // The exclusive upperbound for scanning.
+    bytes end_key = 5;
 }
 
 message ScanLockResponse {
@@ -435,6 +445,8 @@ message RawPutRequest {
     bytes key = 2;
     bytes value = 3;
     string cf = 4;
+    uint64 ttl = 5;
+    bool for_cas = 6;
 }
 
 message RawPutResponse {
@@ -446,6 +458,8 @@ message RawBatchPutRequest {
     Context context = 1;
     repeated KvPair pairs = 2;
     string cf = 3;
+    uint64 ttl = 4;
+    bool for_cas = 5;
 }
 
 message RawBatchPutResponse {
@@ -457,6 +471,7 @@ message RawDeleteRequest {
     Context context = 1;
     bytes key = 2;
     string cf = 3;
+    bool for_cas = 4;
 }
 
 message RawDeleteResponse {
@@ -468,6 +483,7 @@ message RawBatchDeleteRequest {
     Context context = 1;
     repeated bytes keys = 2;
     string cf = 3;
+    bool for_cas = 4;
 }
 
 message RawBatchDeleteResponse {
@@ -759,6 +775,10 @@ message Context {
     // A hint for TiKV to schedule tasks more fairly. Query with same task ID
     // may share same priority and resource quota.
     uint64 task_id = 16;
+
+    // Not required to read the most up-to-date data, replicas with `safe_ts` >= `start_ts`
+    // can handle read request directly
+    bool stale_read = 17;
 }
 
 message LockInfo {
@@ -843,11 +863,14 @@ message TimeDetail {
     // cannot be excluded for now, like Mutex wait time, which is included in this field, so that
     // this field is called wall time instead of CPU time.
     int64 process_wall_time_ms = 2;
+    // KV read wall Time means the time used in key/value scan and get.
+    int64 kv_read_wall_time_ms = 3;
 }
 
 message ScanInfo {
     int64 total = 1;
     int64 processed = 2;
+    int64 read_bytes = 3;
 }
 
 // Only reserved for compatibility.
@@ -974,6 +997,8 @@ enum Action {
     TTLExpireRollback = 1;
     LockNotExistRollback = 2;
     MinCommitTSPushed = 3;
+    TTLExpirePessimisticRollback = 4;
+    LockNotExistDoNothing = 5;
 }
 
 message KeyRange {
@@ -1003,3 +1028,46 @@ message CheckLeaderResponse {
     repeated uint64 regions = 1;
     uint64 ts = 2;
 }
+
+message StoreSafeTSRequest {
+    // Get the minimal `safe_ts` from regions that overlap with the key range [`start_key`, `end_key`)
+    // An empty key range means all regions in the store
+    KeyRange key_range = 1;
+}
+
+message StoreSafeTSResponse {
+    uint64 safe_ts = 1;
+}
+
+message RawGetKeyTTLRequest {
+    Context context = 1;
+    bytes key = 2;
+    string cf = 3;
+}
+
+message RawGetKeyTTLResponse {
+    errorpb.Error region_error = 1;
+    string error = 2;
+    uint64 ttl = 3;
+    bool not_found = 4;
+}
+
+message RawCASRequest {
+    Context context = 1;
+    bytes key = 2;
+    bytes value = 3;
+    bool previous_not_exist = 4;
+    bytes previous_value = 5;
+    string cf = 6;
+    uint64 ttl = 7;
+}
+
+message RawCASResponse {
+    errorpb.Error region_error = 1;
+    string error = 2;
+    bool succeed = 3;
+    // The previous value regardless of whether the comparison is succeed.
+    bool previous_not_exist = 4;
+    bytes previous_value = 5;
+}
+

tikv-client-proto/proto/metapb.proto

@@ -50,6 +50,8 @@ message Store {
     string deploy_path = 10;
     // The last heartbeat timestamp of the store.
     int64 last_heartbeat = 11;
+    // If the store is physically destroyed, which means it can never up again.
+    bool physically_destroyed = 12;
 }
 
 message RegionEpoch {

tikv-client-proto/proto/pdpb.proto

@@ -80,7 +80,7 @@ service PD {
 
     rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {}
 
-    rpc GetDCLocations(GetDCLocationsRequest) returns (GetDCLocationsResponse) {}
+    rpc GetDCLocationInfo(GetDCLocationInfoRequest) returns (GetDCLocationInfoResponse) {}
 }
 
 message RequestHeader {
@@ -121,7 +121,9 @@ message TsoRequest {
 message Timestamp {
     int64 physical = 1;
     int64 logical = 2;
-}
+    // Number of suffix bits used for global distinction,
+    // PD client will use this to compute a TSO's logical part.
+    uint32 suffix_bits = 3;}
 
 message TsoResponse {
     ResponseHeader header = 1;
@@ -286,6 +288,7 @@ message Member {
     string deploy_path = 6;
     string binary_version = 7;
     string git_hash = 8;
+    string dc_location = 9;
 }
 
 message GetMembersRequest {
@@ -652,11 +655,16 @@ message SplitRegionsResponse {
     repeated uint64 regions_id = 3;
 }
 
-message GetDCLocationsRequest {
+message GetDCLocationInfoRequest {
     RequestHeader header = 1;
+    string dc_location = 2;
 }
 
-message GetDCLocationsResponse {
+message GetDCLocationInfoResponse {
     ResponseHeader header = 1;
-    repeated string dc_locations = 2;
+    // suffix sign
+    int32 suffix = 2;
+    // max_ts will be included into this response if PD leader think the receiver needs,
+    // which it's set when the number of the max suffix bits changes.
+    Timestamp max_ts = 3;
 }

tikv-client-proto/proto/raft_cmdpb.proto

@@ -313,6 +313,9 @@ message RaftRequestHeader {
 
     // Read requests can be responsed directly after the Raft applys to `applied_index`.
     uint64 applied_index = 9;
+
+    // Custom flags for this raft request.
+    uint64 flags = 10;
 }
 
 message RaftResponseHeader {

tikv-client-proto/proto/raft_serverpb.proto

@@ -38,6 +38,8 @@ message SnapshotCFFile {
 
 message SnapshotMeta {
     repeated SnapshotCFFile cf_files = 1;
+    // true means this snapshot is triggered for load balance
+    bool for_balance = 2; 
 }
 
 message SnapshotChunk {
@@ -102,6 +104,10 @@ enum ExtraMessageType {
     MsgWantRollbackMerge = 1;
     MsgCheckStalePeer = 2;
     MsgCheckStalePeerResponse = 3;
+    // If leader is going to sleep, it will send requests to all its followers
+    // to make sure they all agree to sleep.
+    MsgHibernateRequest = 4;
+    MsgHibernateResponse = 5;
 }
 
 message ExtraMessage {

tikv-client-proto/proto/tikvpb.proto

@@ -2,6 +2,7 @@ syntax = "proto3";
 package tikvpb;
 
 import "coprocessor.proto";
+import "coprocessor_v2.proto";
 import "kvrpcpb.proto";
 import "mpp.proto";
 import "raft_serverpb.proto";
@@ -47,6 +48,10 @@ service Tikv {
     rpc RawScan(kvrpcpb.RawScanRequest) returns (kvrpcpb.RawScanResponse) {}
     rpc RawDeleteRange(kvrpcpb.RawDeleteRangeRequest) returns (kvrpcpb.RawDeleteRangeResponse) {}
     rpc RawBatchScan(kvrpcpb.RawBatchScanRequest) returns (kvrpcpb.RawBatchScanResponse) {}
+    // Get TTL of the key. Returns 0 if TTL is not set for the key.
+    rpc RawGetKeyTTL(kvrpcpb.RawGetKeyTTLRequest) returns (kvrpcpb.RawGetKeyTTLResponse) {}
+    // Compare if the value in database equals to `RawCASRequest.previous_value` before putting the new value. If not, this request will have no effect and the value in the database will be returned.
+    rpc RawCompareAndSwap(kvrpcpb.RawCASRequest) returns (kvrpcpb.RawCASResponse) {}
 
     // VerKV commands.
     rpc VerGet(kvrpcpb.VerGetRequest) returns (kvrpcpb.VerGetResponse) {}
@@ -66,9 +71,12 @@ service Tikv {
     // Commands for executing SQL in the TiKV coprocessor (i.e., 'pushed down' to TiKV rather than
     // executed in TiDB).
     rpc Coprocessor(coprocessor.Request) returns (coprocessor.Response) {}
-    rpc CoprocessorStream(coprocessor.Request)  returns (stream coprocessor.Response) {}
+    rpc CoprocessorStream(coprocessor.Request) returns (stream coprocessor.Response) {}
     rpc BatchCoprocessor(coprocessor.BatchRequest) returns (stream coprocessor.BatchResponse) {}
 
+    // Command for executing custom user requests in TiKV coprocessor_v2.
+    rpc CoprocessorV2(coprocessor_v2.RawCoprocessorRequest) returns (coprocessor_v2.RawCoprocessorResponse) {}
+
     // Raft commands (sent between TiKV nodes).
     rpc Raft(stream raft_serverpb.RaftMessage) returns (raft_serverpb.Done) {}
     rpc BatchRaft(stream BatchRaftMessage) returns (raft_serverpb.Done) {}
@@ -96,7 +104,10 @@ service Tikv {
     /// term and epoch match with local information in the store.
     /// After the client collected all responses from all stores, it checks if got a quorum of responses from
     /// other stores for every region, and decides to advance resolved ts from these regions.
-    rpc CheckLeader(kvrpcpb.CheckLeaderRequest) returns(kvrpcpb.CheckLeaderResponse);
+    rpc CheckLeader(kvrpcpb.CheckLeaderRequest) returns (kvrpcpb.CheckLeaderResponse);
+
+    /// Get the minimal `safe_ts` from regions at the store
+    rpc GetStoreSafeTS(kvrpcpb.StoreSafeTSRequest) returns (kvrpcpb.StoreSafeTSResponse);
 }
 
 message BatchCommandsRequest {
@@ -144,6 +155,8 @@ message BatchCommandsRequest {
 
             kvrpcpb.CheckSecondaryLocksRequest CheckSecondaryLocks = 33;
 
+            coprocessor_v2.RawCoprocessorRequest CoprocessorV2 = 34;
+
             // For some test cases.
             BatchCommandsEmptyRequest Empty = 255;
         }
@@ -197,6 +210,8 @@ message BatchCommandsResponse {
 
             kvrpcpb.CheckSecondaryLocksResponse CheckSecondaryLocks = 33;
 
+            coprocessor_v2.RawCoprocessorResponse CoprocessorV2 = 34;
+
             // For some test cases.
             BatchCommandsEmptyResponse Empty = 255;
         }