[to #788] Reduce the number of ObjectMapper creations to improve performance (#787)

Signed-off-by: Jia Fan <fanjiaeminem@qq.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: "\U0001F41B Bug Report"
+about: As a User, I want to report a Bug.
+labels: type/bug
+---
+
+## Bug Report
+
+### 1. Describe the bug
+
+<!-- A clear and concise description of what the bug is. -->
+
+### 2. Minimal reproduce step (Required)
+
+<!-- a step by step guide for reproducing the bug. -->
+
+### 3. What did you see instead (Required)
+
+### 4. What did you expect to see? (Required)
+
+### 5. What are your Java Client and TiKV versions? (Required)
+
+- Client Java:
+- TiKV:

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,23 @@
+---
+name: "\U0001F680 Feature Request"
+about: As a user, I want to request a New Feature on the product.
+labels: type/feature-request
+---
+
+## Feature Request
+
+**Is your feature request related to a problem? Please describe:**
+<!-- A clear and concise description of what the problem is. Ex. I'm always
+frustrated when [...] -->
+
+**Describe the feature you'd like:**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered:**
+<!-- A clear and concise description of any alternative solutions or features
+you've considered. -->
+
+**Teachability, Documentation, Adoption, Migration Strategy:**
+<!-- If you can, explain some scenarios of how users might use this, situations it
+would be helpful in. Any API designs, mockups, or diagrams are also helpful.
+-->

.github/config/pd.toml

@@ -0,0 +1,4 @@
+# PD Configuration.
+[replication]
+enable-placement-rules = true
+max-replicas = 1

.github/config/tikv_rawkv.toml

@@ -0,0 +1,21 @@
+# TiKV Configuration.
+
+[raftstore]
+# set store capacity, if no set, use disk capacity.
+capacity = "6G"
+pd-heartbeat-tick-interval = "2s"
+pd-store-heartbeat-tick-interval = "5s"
+split-region-check-tick-interval = "1s"
+
+[rocksdb]
+max-open-files = 10000
+
+[raftdb]
+max-open-files = 10000
+
+[storage.block-cache]
+capacity = "128MB"
+
+[storage]
+reserve-space = "0MB"
+enable-ttl = true

.github/config/tikv_txnkv.toml

@@ -0,0 +1,20 @@
+# TiKV Configuration.
+
+[raftstore]
+# set store capacity, if no set, use disk capacity.
+capacity = "6G"
+pd-heartbeat-tick-interval = "2s"
+pd-store-heartbeat-tick-interval = "5s"
+split-region-check-tick-interval = "1s"
+
+[rocksdb]
+max-open-files = 10000
+
+[raftdb]
+max-open-files = 10000
+
+[storage.block-cache]
+capacity = "128MB"
+
+[storage]
+reserve-space = "0MB"

.github/config/tikv_v2.toml

@@ -0,0 +1,17 @@
+# TiKV Configuration.
+
+[raftstore]
+pd-heartbeat-tick-interval = "2s"
+pd-store-heartbeat-tick-interval = "5s"
+split-region-check-tick-interval = "1s"
+
+[rocksdb]
+max-open-files = 10000
+
+[raftdb]
+max-open-files = 10000
+
+[storage]
+reserve-space = "0MB"
+api-version = 2
+enable-ttl = true

.github/labeler.yml

@@ -0,0 +1,17 @@
+component/config:
+  - config/*
+
+component/br:
+  - src/main/java/org/tikv/br/*
+
+component/cdc:
+  - src/main/java/org/tikv/cdc/*
+
+component/common:
+  - src/main/java/org/tikv/common/*
+
+component/rawkv:
+  - src/main/java/org/tikv/raw/*
+
+component/txnkv:
+  - src/main/java/org/tikv/txn/*

.github/license-checker.yml

@@ -0,0 +1,26 @@
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: TiKV Project Authors.
+  paths-ignore:
+    - 'config/'
+    - 'dev/'
+    - 'docs/'
+    - 'metrics/'
+    - 'LICENSE'
+    - 'Makefile'
+    - 'pom.xml'
+    - 'shell.nix'
+    - '.ci/'
+    - '.gitignore'
+    - '.gitattributes'
+    - '.github/'
+    - '**/*.md'
+    - '**/*.properties'
+    - '**/*.json'
+    - '**/*.pem'
+    - '**/*.crt'
+    - '**/*.g4'
+    - 'src/main/java/io/grpc/'
+    - 'src/main/java/io/netty/'
+  comment: on-failure

.github/pull_request_template.md

@@ -0,0 +1,46 @@
+<!-- Thank you for contributing to TiKV Java Client!
+
+PR Title Format: "[close/to/fix #issue_number] summary" -->
+
+### What problem does this PR solve?
+
+Issue Number: close #issue_number
+
+Problem Description: **TBD**
+
+### What is changed and how does it work?
+
+
+
+### Code changes
+
+<!-- REMOVE the items that are not applicable -->
+- Has exported function/method change
+- Has exported variable/fields change
+- Has methods of interface change
+- Has persistent data change
+- No code
+
+### Check List for Tests
+
+This PR has been tested by at least one of the following methods:
+- Unit test
+- Integration test
+- Manual test (add detailed scripts or steps below)
+- No code
+
+### Side effects
+
+<!-- REMOVE the items that are not applicable -->
+- Possible performance regression, WHY: **TBD**
+- Increased code complexity, WHY: **TBD**
+- Breaking backward compatibility, WHY: **TBD**
+- NO side effects
+
+### Related changes
+
+<!-- REMOVE the items that are not applicable -->
+- Need to cherry-pick to the release branch
+- Need to update the documentation
+- Need to be included in the release note
+- NO related changes

.github/workflows/ci.yml

@@ -0,0 +1,84 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  format:
+    name: Check Format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8.0'
+          distribution: 'adopt'
+      - run: |
+          ./dev/javafmt
+          if [[ $(git diff) != "" ]]
+          then
+              echo "code format error, please run the following commands:"
+              echo "   ./dev/javafmt"
+              exit 1
+          fi
+
+  integration-test:
+    name: Integration Test - ${{ matrix.tikv_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tikv_version: [v5.0.6, v5.3.4, v5.4.3]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8.0'
+          distribution: 'adopt'
+      - name: Install TiUP
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+          /home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
+      - name: Start TiUP Playground
+        run: |
+          # Start TiKV in APIV1TTL
+          touch tiup-v1ttl.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag rawkv --mode tikv-slim --kv 1 --without-monitor --kv.port 20160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup-v1ttl.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1ttl.log)
+          cat tiup-v1ttl.log
+          echo "Wait for bootstrap"
+          sleep 10s
+
+          # Start TiKV in APIV1
+          touch tiup-v1.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag txnkv --mode tikv-slim --kv 1 --without-monitor --kv.port 30160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2381 2>&1 >> tiup-v1.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1.log)
+          cat tiup-v1.log
+          echo "Wait for bootstrap"
+          sleep 10s
+
+          # Get PD address
+          echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+          echo "TXNKV_PD_ADDRESSES=127.0.0.1:2381" >> $GITHUB_ENV
+
+      - name: Run Integration Test
+        run: mvn clean test
+      - name: Print TiKV logs
+        if: failure()
+        run: |
+          echo "RawKV TiKV logs"
+          cat /home/runner/.tiup/data/rawkv/tikv-0/tikv.log
+          
+          echo "TxnKV TiKV logs"
+          cat /home/runner/.tiup/data/txnkv/tikv-0/tikv.log
+      - name: Upload coverage
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/ci_v2.yml

@@ -0,0 +1,52 @@
+name: CI (APIv2)
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  integration-test:
+    name: Integration Test - ${{ matrix.tikv_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tikv_version: [v6.5.3, v7.1.1, nightly]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8.0'
+          distribution: 'adopt'
+      - name: Install TiUP
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+          /home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
+      - name: Start TiUP Playground
+        run: |
+          # Start TiKV in APIV2
+          touch tiup.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --tag kv --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_v2.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup.log)
+          cat tiup.log
+
+          # Get PD address
+          echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+          echo "TXNKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+
+      - name: Run Integration Test
+        run: mvn clean test
+      - name: Print TiKV logs
+        if: failure()
+        run: |
+          echo "TiKV logs"
+          cat /home/runner/.tiup/data/kv/tikv-0/tikv.log
+      - name: Upload coverage
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/gh-pages.yml

@@ -0,0 +1,38 @@
+name: GitHub Pages
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  deploy:
+    runs-on: ubuntu-20.04
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup mdBook
+        uses: peaceiris/actions-mdbook@v1
+        with:
+          mdbook-version: '0.4.8'
+          # mdbook-version: 'latest'
+
+      - run: mdbook build ./docs
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
+      - name: Maven site
+        run: mvn clean compile site && mv ./target/site ./docs/book/site
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.ref == 'refs/heads/master' }}
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/book

.github/workflows/labeler.yml

@@ -0,0 +1,11 @@
+name: "PR Labeler"
+on:
+- pull_request_target
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v3
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/license-checker.yml

@@ -0,0 +1,24 @@
+name: License checker
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check-license:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check License Header
+        uses: apache/skywalking-eyes@v0.3.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          log: info
+          config: .github/license-checker.yml
+

.github/workflows/pr-title-checker.yml

@@ -0,0 +1,14 @@
+name: 'PR Title Checker'
+on:
+  pull_request:
+    types: [edited, opened, synchronize, reopened]
+
+jobs:
+  title-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: naveenk1223/action-pr-title@master
+        with:
+          regex: '\[to|fix|close #[0-9]+\] .+'
+          prefix_case_sensitive: true
+          max_length: -1

.github/workflows/spotbugs.yml

@@ -0,0 +1,30 @@
+name: Spotbugs
+
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - uses: actions/cache@v1
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Build with Maven
+        run: mvn -B verify spotbugs:spotbugs -Dmaven.test.skip=true
+      - uses: jwgmeligmeyling/spotbugs-github-action@v1
+        with:
+          path: "**/spotbugsXml.xml"

.gitignore

@@ -5,6 +5,7 @@ pub.sh
 
 # ignore compiled classes
 target
+.classpath
 
 # ignore version info
 src/main/java/com/pingcap/tikv/TiVersion.java
@@ -70,3 +71,9 @@ out/
 
 # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898
 # gradle/wrapper/gradle-wrapper.properties
+
+# vscode
+.settings
+
+# mdbook
+docs/book

Makefile

@@ -0,0 +1,16 @@
+.DEFAULT: build
+
+.PHONY: clean
+
+build:
+	mvn clean package -Dmaven.test.skip=true
+
+fmt:
+	./dev/javafmt
+
+test:
+	mvn clean test
+
+clean:
+	mvn clean
+	rm -rf target tipb raft-rs kvproto

README.md

@@ -1,137 +1,89 @@
+[![Maven Central](https://img.shields.io/maven-central/v/org.tikv/tikv-client-java.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22org.tikv%22%20AND%20a:%22tikv-client-java%22)
+[![Slack](https://img.shields.io/badge/chat-on%20Slack-brightgreen.svg)](https://slack.tidb.io/invite?team=tikv-wg&channel=client)
+[![codecov](https://codecov.io/gh/tikv/client-java/branch/master/graph/badge.svg?token=nSAjGaN0EH)](https://codecov.io/gh/tikv/client-java)
+
 ## TiKV JAVA Client
 
-A Java client for [TiDB](https://github.com/pingcap/tidb)/[TiKV](https://github.com/tikv/tikv).
+A Java client for [TiKV](https://github.com/tikv/tikv).
 It is supposed to:
 + Communicate via [gRPC](http://www.grpc.io/)
 + Talk to Placement Driver searching for a region
-+ Talk to TiKV for reading/writing data and the resulted data is encoded/decoded just like what we do in TiDB.
++ Talk to TiKV for reading/writing data
-+ Talk to Coprocessor for calculation pushdown
 
-## How to build
+## Quick Start
 
-### Maven
+> TiKV Java Client is designed to communicate with [PD](https://github.com/tikv/pd) and [TiKV](https://github.com/tikv/tikv), please run PD and TiKV in advance.
 
-The alternative way to build a usable jar for testing will be
+Build Java client from source file:
 
-```
+```sh
 mvn clean install -Dmaven.test.skip=true
 ```
 
-The following command can install dependencies for you.
+Add maven dependency to `pom.xml`:
-
-```
-mvn package
-```
-
-The jar can be found in `./target/`
-
-## Usage
-
-This project is designed to hook with `pd` and `tikv` which you can find in `PingCAP` github page.
-
-When you work with this project, you have to communicate with `pd` and `tikv`. Please run TiKV and PD in advance.
-
-## Component: Raw Ti-Client in Java
-
-Java Implementation of Raw TiKV-Client to support RawKVClient commands.
-
-Demo is avaliable in [KVRawClientTest](https://github.com/birdstorm/KVRawClientTest/)
-
-### Build
-```
-mvn clean install -Dmaven.test.skip=true
-```
-
-### Add to dependency
-
-#### Use jar for binary
-
-Add your jar built with all dependencies into you project's library to use `tikv-client-java` as dependency
-
-#### Use as maven dependency
-
-After building, add following lines into your `pom.xml` if you are using Maven
 
 ```xml
 <dependency>
 	<groupId>org.tikv</groupId>
 	<artifactId>tikv-client-java</artifactId>
-	<version>3.0.0</version>
+	<version>3.3.0</version>
 </dependency>
 ```
 
-### Entrance
+Create a transactional `KVClient` and communicates with TiKV:
-`org.tikv.raw.RawKVClient`
-
-### Create a RawKVClient
 
 ```java
+import org.tikv.common.TiConfiguration;
 import org.tikv.common.TiSession;
-import org.tikv.raw.RawKVClient;
+import org.tikv.txn.KVClient;
 
 public class Main {
-	public static void main() {
+	public static void main(String[] args) throws Exception {
-		// You MUST create a raw configuration if you are using RawKVClient.
+		TiConfiguration conf = TiConfiguration.createDefault(YOUR_PD_ADDRESSES);
-		TiConfiguration conf = TiConfiguration.createRawDefault(YOUR_PD_ADDRESSES);
 		TiSession session = TiSession.create(conf);
-		RawKVClient = session.createRawKVClient();
+		KVClient client = session.createKVClient();
 	}
 }
 ```
 
-### API
+Or create a `RawKVClient` if you don't need the transaction semantic:
 
 ```java
-/**
+import org.tikv.common.TiConfiguration;
- * Put a raw key-value pair to TiKV
+import org.tikv.common.TiSession;
- *
+import org.tikv.raw.RawKVClient;
- * @param key raw key
+
- * @param value raw value
+public class Main {
- */
+	public static void main(String[] args) throws Exception {
-void put(ByteString key, ByteString value)
+		TiConfiguration conf = TiConfiguration.createRawDefault(YOUR_PD_ADDRESSES);
+		TiSession session = TiSession.create(conf);
+		RawKVClient client = session.createRawClient();
+	}
+}
 ```
 
-```java
+Find more demo in [TiKV Java Client User Documents](https://tikv.github.io/client-java/examples/introduction.html)
-/**
- * Get a raw key-value pair from TiKV if key exists
- *
- * @param key raw key
- * @return a ByteString value if key exists, ByteString.EMPTY if key does not exist
- */
-ByteString get(ByteString key)
-```
 
-```java
+## Documentation
-/**
- * Scan raw key-value pairs from TiKV in range [startKey, endKey)
- *
- * @param startKey raw start key, inclusive
- * @param endKey raw end key, exclusive
- * @param limit limit of key-value pairs scanned, should be less than {@link #MAX_RAW_SCAN_LIMIT}
- * @return list of key-value pairs in range
- */
-List<Kvrpcpb.KvPair> scan(ByteString startKey, ByteString endKey, int limit)
-```
 
-```java
+See [Java Client Documents](/docs/README.md) for references about how to config and monitor Java Client.
-/**
- * Scan raw key-value pairs from TiKV in range [startKey, endKey)
- *
- * @param startKey raw start key, inclusive
- * @param limit limit of key-value pairs scanned, should be less than {@link #MAX_RAW_SCAN_LIMIT}
- * @return list of key-value pairs in range
- */
-List<Kvrpcpb.KvPair> scan(ByteString startKey, int limit)
-```
 
-```java
+A [Maven site](https://tikv.github.io/client-java/site) is also available. It includes:
-/**
+1. [API reference](https://tikv.github.io/client-java/site/apidocs/index.html)
- * Delete a raw key-value pair from TiKV if key exists
+2. [Spotbugs Reports](https://tikv.github.io/client-java/site/spotbugs.html)
- *
+3. [Source Code Xref](https://tikv.github.io/client-java/site/xref/index.html)
- * @param key raw key to be deleted
+
- */
+## Community
-void delete(ByteString key)
+
-```
+### Forum
+
+- User forum: [AskTUG](https://asktug.com/)
+- Contributor forum: [https://internals.tidb.io/](https://internals.tidb.io/)
+
+### Contribute to TiKV Java Client
+
+See [Contribution Guide](https://tikv.github.io/client-java/contribution/introduction.html) for references about how to contribute to this project.
 
 ## License
+
 Apache 2.0 license. See the [LICENSE](./LICENSE) file for details.

dev/README.md

@@ -1,8 +1,6 @@
-# TiSpark Dev Tools Guide
+# TiKV Java Client Dev Tools
 
-## Formatting
+## Code Formatting
-
-### Java Format
 
 TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https://github.com/coveooss/fmt-maven-plugin) which follows Google's code styleguide. It is also checked on CI before build.
 
@@ -18,4 +16,4 @@ TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https:
 
     ```shell script
    ./dev/javafmt
-    ```
+    ```

dev/intellij-java-google-style.xml

@@ -2,222 +2,222 @@
 <code_scheme name="GoogleStyle">
     <option name="OTHER_INDENT_OPTIONS">
         <value>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
-            <option name="USE_TAB_CHARACTER" value="false" />
+            <option name="USE_TAB_CHARACTER" value="false"/>
-            <option name="SMART_TABS" value="false" />
+            <option name="SMART_TABS" value="false"/>
-            <option name="LABEL_INDENT_SIZE" value="0" />
+            <option name="LABEL_INDENT_SIZE" value="0"/>
-            <option name="LABEL_INDENT_ABSOLUTE" value="false" />
+            <option name="LABEL_INDENT_ABSOLUTE" value="false"/>
-            <option name="USE_RELATIVE_INDENTS" value="false" />
+            <option name="USE_RELATIVE_INDENTS" value="false"/>
         </value>
     </option>
-    <option name="INSERT_INNER_CLASS_IMPORTS" value="true" />
+    <option name="INSERT_INNER_CLASS_IMPORTS" value="true"/>
-    <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
+    <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999"/>
-    <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
+    <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999"/>
     <option name="PACKAGES_TO_USE_IMPORT_ON_DEMAND">
-        <value />
+        <value/>
     </option>
     <option name="IMPORT_LAYOUT_TABLE">
         <value>
-            <package name="" withSubpackages="true" static="true" />
+            <package name="" withSubpackages="true" static="true"/>
-            <emptyLine />
+            <emptyLine/>
-            <package name="" withSubpackages="true" static="false" />
+            <package name="" withSubpackages="true" static="false"/>
         </value>
     </option>
-    <option name="RIGHT_MARGIN" value="100" />
+    <option name="RIGHT_MARGIN" value="100"/>
-    <option name="JD_ALIGN_PARAM_COMMENTS" value="false" />
+    <option name="JD_ALIGN_PARAM_COMMENTS" value="false"/>
-    <option name="JD_ALIGN_EXCEPTION_COMMENTS" value="false" />
+    <option name="JD_ALIGN_EXCEPTION_COMMENTS" value="false"/>
-    <option name="JD_P_AT_EMPTY_LINES" value="false" />
+    <option name="JD_P_AT_EMPTY_LINES" value="false"/>
-    <option name="JD_KEEP_EMPTY_PARAMETER" value="false" />
+    <option name="JD_KEEP_EMPTY_PARAMETER" value="false"/>
-    <option name="JD_KEEP_EMPTY_EXCEPTION" value="false" />
+    <option name="JD_KEEP_EMPTY_EXCEPTION" value="false"/>
-    <option name="JD_KEEP_EMPTY_RETURN" value="false" />
+    <option name="JD_KEEP_EMPTY_RETURN" value="false"/>
-    <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
+    <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false"/>
-    <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="0" />
+    <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="0"/>
-    <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
+    <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
-    <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="0" />
+    <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="0"/>
-    <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
+    <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
-    <option name="ALIGN_MULTILINE_FOR" value="false" />
+    <option name="ALIGN_MULTILINE_FOR" value="false"/>
-    <option name="CALL_PARAMETERS_WRAP" value="1" />
+    <option name="CALL_PARAMETERS_WRAP" value="1"/>
-    <option name="METHOD_PARAMETERS_WRAP" value="1" />
+    <option name="METHOD_PARAMETERS_WRAP" value="1"/>
-    <option name="EXTENDS_LIST_WRAP" value="1" />
+    <option name="EXTENDS_LIST_WRAP" value="1"/>
-    <option name="THROWS_KEYWORD_WRAP" value="1" />
+    <option name="THROWS_KEYWORD_WRAP" value="1"/>
-    <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
+    <option name="METHOD_CALL_CHAIN_WRAP" value="1"/>
-    <option name="BINARY_OPERATION_WRAP" value="1" />
+    <option name="BINARY_OPERATION_WRAP" value="1"/>
-    <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
+    <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
-    <option name="TERNARY_OPERATION_WRAP" value="1" />
+    <option name="TERNARY_OPERATION_WRAP" value="1"/>
-    <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
+    <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
-    <option name="FOR_STATEMENT_WRAP" value="1" />
+    <option name="FOR_STATEMENT_WRAP" value="1"/>
-    <option name="ARRAY_INITIALIZER_WRAP" value="1" />
+    <option name="ARRAY_INITIALIZER_WRAP" value="1"/>
-    <option name="WRAP_COMMENTS" value="true" />
+    <option name="WRAP_COMMENTS" value="true"/>
-    <option name="IF_BRACE_FORCE" value="3" />
+    <option name="IF_BRACE_FORCE" value="3"/>
-    <option name="DOWHILE_BRACE_FORCE" value="3" />
+    <option name="DOWHILE_BRACE_FORCE" value="3"/>
-    <option name="WHILE_BRACE_FORCE" value="3" />
+    <option name="WHILE_BRACE_FORCE" value="3"/>
-    <option name="FOR_BRACE_FORCE" value="3" />
+    <option name="FOR_BRACE_FORCE" value="3"/>
-    <option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true" />
+    <option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true"/>
     <AndroidXmlCodeStyleSettings>
-        <option name="USE_CUSTOM_SETTINGS" value="true" />
+        <option name="USE_CUSTOM_SETTINGS" value="true"/>
         <option name="LAYOUT_SETTINGS">
             <value>
-                <option name="INSERT_BLANK_LINE_BEFORE_TAG" value="false" />
+                <option name="INSERT_BLANK_LINE_BEFORE_TAG" value="false"/>
             </value>
         </option>
     </AndroidXmlCodeStyleSettings>
     <JSCodeStyleSettings>
-        <option name="INDENT_CHAINED_CALLS" value="false" />
+        <option name="INDENT_CHAINED_CALLS" value="false"/>
     </JSCodeStyleSettings>
     <Python>
-        <option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true" />
+        <option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true"/>
     </Python>
     <TypeScriptCodeStyleSettings>
-        <option name="INDENT_CHAINED_CALLS" value="false" />
+        <option name="INDENT_CHAINED_CALLS" value="false"/>
     </TypeScriptCodeStyleSettings>
     <XML>
-        <option name="XML_ALIGN_ATTRIBUTES" value="false" />
+        <option name="XML_ALIGN_ATTRIBUTES" value="false"/>
-        <option name="XML_LEGACY_SETTINGS_IMPORTED" value="true" />
+        <option name="XML_LEGACY_SETTINGS_IMPORTED" value="true"/>
     </XML>
     <codeStyleSettings language="CSS">
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="ECMA Script Level 4">
-        <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
+        <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
-        <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
+        <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
-        <option name="ALIGN_MULTILINE_FOR" value="false" />
+        <option name="ALIGN_MULTILINE_FOR" value="false"/>
-        <option name="CALL_PARAMETERS_WRAP" value="1" />
+        <option name="CALL_PARAMETERS_WRAP" value="1"/>
-        <option name="METHOD_PARAMETERS_WRAP" value="1" />
+        <option name="METHOD_PARAMETERS_WRAP" value="1"/>
-        <option name="EXTENDS_LIST_WRAP" value="1" />
+        <option name="EXTENDS_LIST_WRAP" value="1"/>
-        <option name="BINARY_OPERATION_WRAP" value="1" />
+        <option name="BINARY_OPERATION_WRAP" value="1"/>
-        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
+        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
-        <option name="TERNARY_OPERATION_WRAP" value="1" />
+        <option name="TERNARY_OPERATION_WRAP" value="1"/>
-        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
+        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
-        <option name="FOR_STATEMENT_WRAP" value="1" />
+        <option name="FOR_STATEMENT_WRAP" value="1"/>
-        <option name="ARRAY_INITIALIZER_WRAP" value="1" />
+        <option name="ARRAY_INITIALIZER_WRAP" value="1"/>
-        <option name="IF_BRACE_FORCE" value="3" />
+        <option name="IF_BRACE_FORCE" value="3"/>
-        <option name="DOWHILE_BRACE_FORCE" value="3" />
+        <option name="DOWHILE_BRACE_FORCE" value="3"/>
-        <option name="WHILE_BRACE_FORCE" value="3" />
+        <option name="WHILE_BRACE_FORCE" value="3"/>
-        <option name="FOR_BRACE_FORCE" value="3" />
+        <option name="FOR_BRACE_FORCE" value="3"/>
-        <option name="PARENT_SETTINGS_INSTALLED" value="true" />
+        <option name="PARENT_SETTINGS_INSTALLED" value="true"/>
     </codeStyleSettings>
     <codeStyleSettings language="HTML">
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="JAVA">
-        <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
+        <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false"/>
-        <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
+        <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
-        <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
+        <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1"/>
-        <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
+        <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
-        <option name="ALIGN_MULTILINE_RESOURCES" value="false" />
+        <option name="ALIGN_MULTILINE_RESOURCES" value="false"/>
-        <option name="ALIGN_MULTILINE_FOR" value="false" />
+        <option name="ALIGN_MULTILINE_FOR" value="false"/>
-        <option name="CALL_PARAMETERS_WRAP" value="1" />
+        <option name="CALL_PARAMETERS_WRAP" value="1"/>
-        <option name="METHOD_PARAMETERS_WRAP" value="1" />
+        <option name="METHOD_PARAMETERS_WRAP" value="1"/>
-        <option name="EXTENDS_LIST_WRAP" value="1" />
+        <option name="EXTENDS_LIST_WRAP" value="1"/>
-        <option name="THROWS_KEYWORD_WRAP" value="1" />
+        <option name="THROWS_KEYWORD_WRAP" value="1"/>
-        <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
+        <option name="METHOD_CALL_CHAIN_WRAP" value="1"/>
-        <option name="BINARY_OPERATION_WRAP" value="1" />
+        <option name="BINARY_OPERATION_WRAP" value="1"/>
-        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
+        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
-        <option name="TERNARY_OPERATION_WRAP" value="1" />
+        <option name="TERNARY_OPERATION_WRAP" value="1"/>
-        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
+        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
-        <option name="FOR_STATEMENT_WRAP" value="1" />
+        <option name="FOR_STATEMENT_WRAP" value="1"/>
-        <option name="ARRAY_INITIALIZER_WRAP" value="1" />
+        <option name="ARRAY_INITIALIZER_WRAP" value="1"/>
-        <option name="WRAP_COMMENTS" value="true" />
+        <option name="WRAP_COMMENTS" value="true"/>
-        <option name="IF_BRACE_FORCE" value="3" />
+        <option name="IF_BRACE_FORCE" value="3"/>
-        <option name="DOWHILE_BRACE_FORCE" value="3" />
+        <option name="DOWHILE_BRACE_FORCE" value="3"/>
-        <option name="WHILE_BRACE_FORCE" value="3" />
+        <option name="WHILE_BRACE_FORCE" value="3"/>
-        <option name="FOR_BRACE_FORCE" value="3" />
+        <option name="FOR_BRACE_FORCE" value="3"/>
-        <option name="PARENT_SETTINGS_INSTALLED" value="true" />
+        <option name="PARENT_SETTINGS_INSTALLED" value="true"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="JSON">
         <indentOptions>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="JavaScript">
-        <option name="RIGHT_MARGIN" value="80" />
+        <option name="RIGHT_MARGIN" value="80"/>
-        <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
+        <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
-        <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
+        <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
-        <option name="ALIGN_MULTILINE_FOR" value="false" />
+        <option name="ALIGN_MULTILINE_FOR" value="false"/>
-        <option name="CALL_PARAMETERS_WRAP" value="1" />
+        <option name="CALL_PARAMETERS_WRAP" value="1"/>
-        <option name="METHOD_PARAMETERS_WRAP" value="1" />
+        <option name="METHOD_PARAMETERS_WRAP" value="1"/>
-        <option name="BINARY_OPERATION_WRAP" value="1" />
+        <option name="BINARY_OPERATION_WRAP" value="1"/>
-        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
+        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
-        <option name="TERNARY_OPERATION_WRAP" value="1" />
+        <option name="TERNARY_OPERATION_WRAP" value="1"/>
-        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
+        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
-        <option name="FOR_STATEMENT_WRAP" value="1" />
+        <option name="FOR_STATEMENT_WRAP" value="1"/>
-        <option name="ARRAY_INITIALIZER_WRAP" value="1" />
+        <option name="ARRAY_INITIALIZER_WRAP" value="1"/>
-        <option name="IF_BRACE_FORCE" value="3" />
+        <option name="IF_BRACE_FORCE" value="3"/>
-        <option name="DOWHILE_BRACE_FORCE" value="3" />
+        <option name="DOWHILE_BRACE_FORCE" value="3"/>
-        <option name="WHILE_BRACE_FORCE" value="3" />
+        <option name="WHILE_BRACE_FORCE" value="3"/>
-        <option name="FOR_BRACE_FORCE" value="3" />
+        <option name="FOR_BRACE_FORCE" value="3"/>
-        <option name="PARENT_SETTINGS_INSTALLED" value="true" />
+        <option name="PARENT_SETTINGS_INSTALLED" value="true"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="PROTO">
-        <option name="RIGHT_MARGIN" value="80" />
+        <option name="RIGHT_MARGIN" value="80"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="2" />
+            <option name="CONTINUATION_INDENT_SIZE" value="2"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="protobuf">
-        <option name="RIGHT_MARGIN" value="80" />
+        <option name="RIGHT_MARGIN" value="80"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="2" />
+            <option name="CONTINUATION_INDENT_SIZE" value="2"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="Python">
-        <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
+        <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
-        <option name="RIGHT_MARGIN" value="80" />
+        <option name="RIGHT_MARGIN" value="80"/>
-        <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
+        <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
-        <option name="PARENT_SETTINGS_INSTALLED" value="true" />
+        <option name="PARENT_SETTINGS_INSTALLED" value="true"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="SASS">
         <indentOptions>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="SCSS">
         <indentOptions>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="TypeScript">
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="XML">
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="2" />
+            <option name="CONTINUATION_INDENT_SIZE" value="2"/>
-            <option name="TAB_SIZE" value="2" />
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
         <arrangement>
             <rules>
@@ -226,7 +226,7 @@
                         <match>
                             <AND>
                                 <NAME>xmlns:android</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>^$</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -237,7 +237,7 @@
                         <match>
                             <AND>
                                 <NAME>xmlns:.*</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>^$</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -249,7 +249,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:id</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -260,7 +260,7 @@
                         <match>
                             <AND>
                                 <NAME>style</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>^$</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -271,7 +271,7 @@
                         <match>
                             <AND>
                                 <NAME>.*</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>^$</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -283,7 +283,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:.*Style</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -295,7 +295,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_width</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -306,7 +306,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_height</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -317,7 +317,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_weight</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -328,7 +328,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_margin</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -339,7 +339,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginTop</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -350,7 +350,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginBottom</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -361,7 +361,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginStart</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -372,7 +372,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginEnd</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -383,7 +383,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginLeft</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -394,7 +394,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginRight</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -405,7 +405,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_.*</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -417,7 +417,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:padding</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -428,7 +428,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingTop</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -439,7 +439,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingBottom</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -450,7 +450,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingStart</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -461,7 +461,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingEnd</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -472,7 +472,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingLeft</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -483,7 +483,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingRight</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -537,62 +537,62 @@
         </arrangement>
     </codeStyleSettings>
     <Objective-C>
-        <option name="INDENT_NAMESPACE_MEMBERS" value="0" />
+        <option name="INDENT_NAMESPACE_MEMBERS" value="0"/>
-        <option name="INDENT_C_STRUCT_MEMBERS" value="2" />
+        <option name="INDENT_C_STRUCT_MEMBERS" value="2"/>
-        <option name="INDENT_CLASS_MEMBERS" value="2" />
+        <option name="INDENT_CLASS_MEMBERS" value="2"/>
-        <option name="INDENT_VISIBILITY_KEYWORDS" value="1" />
+        <option name="INDENT_VISIBILITY_KEYWORDS" value="1"/>
-        <option name="INDENT_INSIDE_CODE_BLOCK" value="2" />
+        <option name="INDENT_INSIDE_CODE_BLOCK" value="2"/>
-        <option name="KEEP_STRUCTURES_IN_ONE_LINE" value="true" />
+        <option name="KEEP_STRUCTURES_IN_ONE_LINE" value="true"/>
-        <option name="FUNCTION_PARAMETERS_WRAP" value="5" />
+        <option name="FUNCTION_PARAMETERS_WRAP" value="5"/>
-        <option name="FUNCTION_CALL_ARGUMENTS_WRAP" value="5" />
+        <option name="FUNCTION_CALL_ARGUMENTS_WRAP" value="5"/>
-        <option name="TEMPLATE_CALL_ARGUMENTS_WRAP" value="5" />
+        <option name="TEMPLATE_CALL_ARGUMENTS_WRAP" value="5"/>
-        <option name="TEMPLATE_CALL_ARGUMENTS_ALIGN_MULTILINE" value="true" />
+        <option name="TEMPLATE_CALL_ARGUMENTS_ALIGN_MULTILINE" value="true"/>
-        <option name="ALIGN_INIT_LIST_IN_COLUMNS" value="false" />
+        <option name="ALIGN_INIT_LIST_IN_COLUMNS" value="false"/>
-        <option name="SPACE_BEFORE_SUPERCLASS_COLON" value="false" />
+        <option name="SPACE_BEFORE_SUPERCLASS_COLON" value="false"/>
     </Objective-C>
     <Objective-C-extensions>
-        <option name="GENERATE_INSTANCE_VARIABLES_FOR_PROPERTIES" value="ASK" />
+        <option name="GENERATE_INSTANCE_VARIABLES_FOR_PROPERTIES" value="ASK"/>
-        <option name="RELEASE_STYLE" value="IVAR" />
+        <option name="RELEASE_STYLE" value="IVAR"/>
-        <option name="TYPE_QUALIFIERS_PLACEMENT" value="BEFORE" />
+        <option name="TYPE_QUALIFIERS_PLACEMENT" value="BEFORE"/>
         <file>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Import" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Import"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Macro" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Macro"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Typedef" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Typedef"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Enum" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Enum"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Constant" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Constant"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Global" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Global"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Struct" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Struct"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="FunctionPredecl" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="FunctionPredecl"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Function" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Function"/>
         </file>
         <class>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Property" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Property"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Synthesize" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Synthesize"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InitMethod" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InitMethod"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="StaticMethod" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="StaticMethod"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InstanceMethod" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InstanceMethod"/>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="DeallocMethod" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="DeallocMethod"/>
         </class>
         <extensions>
-            <pair source="cc" header="h" />
+            <pair source="cc" header="h"/>
-            <pair source="c" header="h" />
+            <pair source="c" header="h"/>
         </extensions>
     </Objective-C-extensions>
     <codeStyleSettings language="ObjectiveC">
-        <option name="RIGHT_MARGIN" value="80" />
+        <option name="RIGHT_MARGIN" value="80"/>
-        <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />
+        <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1"/>
-        <option name="BLANK_LINES_BEFORE_IMPORTS" value="0" />
+        <option name="BLANK_LINES_BEFORE_IMPORTS" value="0"/>
-        <option name="BLANK_LINES_AFTER_IMPORTS" value="0" />
+        <option name="BLANK_LINES_AFTER_IMPORTS" value="0"/>
-        <option name="BLANK_LINES_AROUND_CLASS" value="0" />
+        <option name="BLANK_LINES_AROUND_CLASS" value="0"/>
-        <option name="BLANK_LINES_AROUND_METHOD" value="0" />
+        <option name="BLANK_LINES_AROUND_METHOD" value="0"/>
-        <option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="0" />
+        <option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="0"/>
-        <option name="ALIGN_MULTILINE_BINARY_OPERATION" value="false" />
+        <option name="ALIGN_MULTILINE_BINARY_OPERATION" value="false"/>
-        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
+        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
-        <option name="FOR_STATEMENT_WRAP" value="1" />
+        <option name="FOR_STATEMENT_WRAP" value="1"/>
-        <option name="ASSIGNMENT_WRAP" value="1" />
+        <option name="ASSIGNMENT_WRAP" value="1"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
         </indentOptions>
     </codeStyleSettings>
 </code_scheme>

dev/javafmt

@@ -2,4 +2,5 @@
 
 echo $MAVEN_HOME
 
-mvn com.coveo:fmt-maven-plugin:format
+mvn com.coveo:fmt-maven-plugin:format
+mvn xml-format:xml-format

dev/proto.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+#   Copyright 2017 PingCAP, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+proto_dir="proto"
+
+if [ -d $proto_dir ]; then
+	rm -r $proto_dir
+fi
+
+repos=("https://github.com/pingcap/kvproto" "https://github.com/pingcap/raft-rs" "https://github.com/pingcap/tipb")
+commits=(3056ca36e6f2a71a9fc7ba7135e6b119fd977553 b9891b673573fad77ebcf9bbe0969cf945841926 c4d518eb1d60c21f05b028b36729e64610346dac)
+
+for i in "${!repos[@]}"; do 
+	repo_name=$(basename ${repos[$i]})
+	git_command="git -C $repo_name"
+
+	if [ -d "$repo_name" ]; then
+		$git_command checkout `basename $($git_command symbolic-ref --short refs/remotes/origin/HEAD)`
+		$git_command fetch --all
+		$git_command pull --all
+	else
+		git clone ${repos[$i]} $repo_name
+		$git_command fetch -p
+	fi
+
+	$git_command checkout ${commits[$i]}
+done

dev/spotbugs-include.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<FindBugsFilter>
+    <Match>
+        <Package name="~org\.tikv\.(?!kvproto).*"/>
+        <Not>
+            <Package name="~org\.tikv\.common\.parser.*"/>
+        </Not>
+        <Not>
+            <Or>
+                <Bug pattern="EI_EXPOSE_REP"/>
+                <Bug pattern="EI_EXPOSE_REP2"/>
+            </Or>
+        </Not>
+        <Rank value="1"/>
+    </Match>
+</FindBugsFilter>

docs/README.md

@@ -0,0 +1,28 @@
+# Client Java Docs
+
+NOTE: See [tikv.github.io/client-java](https://tikv.github.io/client-java/) for
+the rendered developer and contributor documents.
+
+This directory contains all the source of developer and contributor
+documentations in markdown format. The table of content can be found in
+[./src/SUMMARY.md](./src/SUMMARY.md).
+
+## How to contribute to the document
+
+The rendered document is generated by `mdbook`, to get it:
+
+```sh
+cargo install mdbook
+```
+
+Build the source in your development machine:
+
+```sh
+mdbook build
+```
+
+Get more information about how to use `mdbook`:
+
+```sh
+mdbook help
+```

docs/book.toml

@@ -0,0 +1,6 @@
+[book]
+authors = ["Jian Zhang"]
+language = "en"
+multilingual = false
+src = "src"
+title = "TiKV Java Client User Documents"

docs/src/SUMMARY.md

@@ -0,0 +1,33 @@
+# Summary
+
+<!-- Links to empty page is replaced by empty link `() `, so that we can prevent readers to click into these unfinished page. Once the corresponding page is done, we can delete the empty parenthesis and then the pages will be visible. -->
+
+- [Introduction](./introduction/introduction.md)
+
+- [Production Readiness](./production-readiness.md)
+
+- [Start With Examples](./examples/introduction.md)
+    - [Quick Start](./examples/quick-start.md)
+    - [Interact with TiKV RawKV API](./examples/rawkv.md)
+    - [Interact with TiKV TxnKV API](./examples/txnkv.md)
+    - [TiKV RawKV Bulk Load]() (./examples/bulk-load.md)
+
+- [Performance]() (./performance/introduction.md)
+    - [YCSB Benchmarks]() (./performance/ycsb.md)
+
+- [Administration](./administration/introduction.md)
+    - [Configuration](./administration/configuration.md)
+    - [Monitoring](./administration/monitoring.md)
+
+- [Troubleshooting](./troubleshooting/introduction.md)
+    - [Slow Request Diagnosis](./troubleshooting/slow-request.md)
+    - [Error Request Diagnosis]() (./troubleshooting/error-request.md)
+
+- [Architecture and Internals](./architecture/introduction.md)
+    - [The Lifecycle of A Request](./architecture/request-lifecycle.md)
+    - [Backoff and Retry Policy](./architecture/availability.md)
+    - [Slow Log and Metrics]() (./architecture/observability.md)
+    - [Region Cache]() (./architecture/region-cache.md)
+
+- [Contribution Guide](./contribution/introduction.md)
+    - [Bug Severity Guidelines](./contribution/bug-severity-guide.md)

docs/src/administration/configuration.md

@@ -0,0 +1,121 @@
+## Java Client Configuration Parameter
+
+### JVM Parameter
+
+The following includes JVM related parameters.
+
+#### tikv.pd.addresses
+- pd addresses, separated by comma
+- default: 127.0.0.1:2379
+
+#### tikv.grpc.timeout_in_ms
+- timeout of grpc request  
+- default: 600ms
+
+#### tikv.grpc.scan_timeout_in_ms
+- timeout of scan/delete range grpc request
+- default: 20s
+
+#### tikv.importer.max_kv_batch_bytes
+- Maximal package size transporting from clients to TiKV Server (ingest API)
+- default: 1048576 (1M)
+
+#### tikv.importer.max_kv_batch_size
+- Maximal batch size transporting from clients to TiKV Server (ingest API)
+- default: 32768 (32K)
+
+#### tikv.scatter_wait_seconds
+- time to wait for scattering regions
+- default: 300 (5min)
+
+#### tikv.rawkv.default_backoff_in_ms
+- RawKV default backoff in milliseconds
+- default: 20000 (20 seconds)
+
+### Metrics Parameter
+
+#### tikv.metrics.enable
+- whether to enable metrics exporting
+- default: false
+
+#### tikv.metrics.port
+- the metrics exporting http port
+- default: 3140
+
+### ThreadPool Parameter
+
+The following includes ThreadPool related parameters, which can be passed in through JVM parameters.
+
+#### tikv.batch_get_concurrency
+- the thread pool size of batchGet on client side
+- default: 20
+
+#### tikv.batch_put_concurrency
+- the thread pool size of batchPut on client side
+- default: 20
+
+#### tikv.batch_delete_concurrency
+- the thread pool size of batchDelete on client side
+- default: 20
+
+#### tikv.batch_scan_concurrency
+- the thread pool size of batchScan on client side
+- default: 5
+
+#### tikv.delete_range_concurrency
+- the thread pool size of deleteRange on client side
+- default: 20
+
+#### tikv.enable_atomic_for_cas
+- whether to enable `Compare And Set`, set true if using `RawKVClient.compareAndSet` or `RawKVClient.putIfAbsent`
+- default: false
+
+### TLS
+
+#### tikv.tls_enable
+- whether to enable TLS
+- default: false
+
+#### tikv.trust_cert_collection
+- Trusted certificates for verifying the remote endpoint's certificate, e.g. /home/tidb/ca.pem. The file should contain an X.509 certificate collection in PEM format.
+- default: null
+
+#### tikv.key_cert_chain
+- an X.509 certificate chain file in PEM format, e.g. /home/tidb/client.pem.
+- default: null
+
+#### tikv.key_file
+- a PKCS#8 private key file in PEM format. e.g. /home/tidb/client-key.pem.
+- default: null
+
+#### tikv.tls.reload_interval
+- The interval in seconds to poll the change of TLS context, if a change is detected, the TLS context will be rebuilded.
+- default: `"10s"`, `"0s"` means disable TLS context reload.
+
+#### tikv.conn.recycle_time
+- After a TLS context reloading, the old connections will be forced to shutdown after `tikv.conn.recycle_time` to prevent channel leak.
+- default: `"60s"`.
+
+#### tikv.rawkv.read_timeout_in_ms
+- RawKV read timeout in milliseconds. This parameter controls the timeout of `get` `getKeyTTL`.
+- default: 2000 (2 seconds)
+
+#### tikv.rawkv.write_timeout_in_ms
+- RawKV write timeout in milliseconds. This parameter controls the timeout of `put` `putAtomic` `putIfAbsent` `delete` `deleteAtomic`.
+- default: 2000 (2 seconds)
+
+#### tikv.rawkv.batch_read_timeout_in_ms
+- RawKV batch read timeout in milliseconds. This parameter controls the timeout of `batchGet`.
+- default: 2000 (2 seconds)
+
+#### tikv.rawkv.batch_write_timeout_in_ms
+- RawKV batch write timeout in milliseconds. This parameter controls the timeout of `batchPut` `batchDelete` `batchDeleteAtomic`.
+- default: 2000 (2 seconds)
+
+#### tikv.rawkv.scan_timeout_in_ms
+- RawKV scan timeout in milliseconds. This parameter controls the timeout of `batchScan` `scan` `scanPrefix`.
+- default: 10000 (10 seconds)
+
+#### tikv.rawkv.clean_timeout_in_ms
+- RawKV clean timeout in milliseconds. This parameter controls the timeout of `deleteRange` `deletePrefix`.
+- default: 600000 (10 minutes)

docs/src/administration/introduction.md

@@ -0,0 +1 @@
+# Administration

docs/src/administration/monitoring.md

@@ -0,0 +1,33 @@
+## Java Client Metrics
+
+Client Java supports exporting metrics to Prometheus using poll mode and viewing on Grafana. The following steps shows how to enable this function.
+
+### Step 1: Enable metrics exporting
+
+- set the config `tikv.metrics.enable` to `true`
+- call TiConfiguration.setMetricsEnable(true)
+
+### Step 2: Set the metrics port
+
+- set the config `tikv.metrics.port`
+- call TiConfiguration.setMetricsPort
+
+Default port is 3140.
+
+### Step 3: Config Prometheus
+
+Add the following config to `conf/prometheus.yml` and restart Prometheus.
+
+```yaml
+- job_name: "tikv-client"
+    honor_labels: true
+    static_configs:
+    - targets:
+        - '127.0.0.1:3140'
+        - '127.0.0.2:3140'
+        - '127.0.0.3:3140'
+```
+
+### Step 4: Config Grafana
+
+Import the [Client-Java-Summary dashboard config](/metrics/grafana/client_java_summary.json) to Grafana.

docs/src/architecture/availability.md

@@ -0,0 +1,47 @@
+# Availability: Backoff and Retry Policy
+
+## BackOffer
+
+The retry and timeout mechanism for a request is controlled by a `BackOffer` object, which is created one per `RawKVClient` method. The `BackOffer` will decide how much time the next sleep and retry should spend, and whether to timeout the request if not enough time is left for retrying the request.
+If we need a back off sleep, we call backOffer.doBackOff(funcType, exception), and the current thread will sleep for a decided time. If the current operation will timeout after sleep, the doBackOff simply throw an exception to abort the operation.
+
+## callWithRetry
+
+RegionStoreClient.callWithRetry inherits from AbstractGRPCClient.callWithRetry. The concrete logic is in RetryPolicy.callWithRetry, which implements a retry mechanism, but the specific retry strategy is determined by the ErrorHandler.
+ErrorHandler’s handler{Request, Response}Error function returns a boolean value indicating whether to retry inside callWithRetry.
+The control flow for callWithRetry is as follows:
+
+![callWithRetry](./callWithRetry.jpg)
+
+The error handler is chosen obeying the following table:
+
+| gPRC request	| the result	| handler |
+| -- | -- | -- |
+| throws exception	| -	| handleRequestError
+| no exception	| is null	| handleRequestError
+| no exception	| is error	| handleResponseError
+| no exception	| normal	| normal return
+
+The handleRequestError function copes with the following situations:
+
+| situation  | retry within callWithRetry | note   |
+|----------|---------------|------------------------------|
+| invalid store in region manager  | true  | refresh ClientStub  |
+| region has not got multiple copies  | false | |
+| successfully switched to new leader | true | |
+| seekProxyStore | true if success | only when `tikv.enable_grpc_forward` is set |
+| other | false | |
+
+The handleResponseError function copes with the following gRPC errors:
+
+| error                | retry within callWithRetry |
+|----------------------|----------------------------|
+| NotLeader            | true if leader unchanged   |
+| StoreNotMatch        | false                      |
+| EphochNotMatch       | true if region epoch in `ctx` is ahead of TiKV's |
+| ServerIsBusy         | true                       |
+| StaleCommand         | true                       |
+| RaftEntryTooLarge    | throw                      |
+| KeyNotInRegion       | throw                      |
+| Raft ProposalDropped | true                       |
+| other                | false                      |

docs/src/architecture/introduction.md

@@ -0,0 +1,3 @@
+# Architecture
+
+This section includes in-depthA description of the client architecture.

docs/src/architecture/observability.md

@@ -0,0 +1 @@
+# Observability: Slow Log and Metrics

docs/src/architecture/region-cache.md

@@ -0,0 +1 @@
+# Region Cache

docs/src/architecture/request-lifecycle.md

@@ -0,0 +1,7 @@
+# The Lifecycle of A Request
+
+![time graph](./timegraph.png)
+
+The client talks to TiKV store directly using gRPC requests, which are created in RegionStoreClient. If a request failed, the client could retry after a back off sleep. The retry logic is delegated to AbstractGRPCClient::callWithRetry method. callWithRetry may decide to retry request within the function, or, if the RegionStoreClient must be recreated (due to, for example, region split), return a failure and let outermost RawKVClient to do the retry.
+
+![request-overview](./request-overview.jpg)

docs/src/contribution/bug-severity-guide.md

@@ -0,0 +1,49 @@
+## Bug Severity Guidelines
+
+This is a **working-in-progress** guide about determining defects severity on
+TiKV Java Client according to the impact on the online service. The higher
+effect the defect has on the overall functionality or performance, the higher
+the severity is. There are 4 severity levels:
+
+1. Critical
+2. Major
+3. Moderate
+4. Minor
+
+Each severity is described with examples in the remaining contents.
+
+### Critical Defects
+
+A defect that affects critical data or functionality and leaves users
+with no workaround is classified as a critical defect. These defects are
+labeled with `type/bug` and `severity/critical`, can be found
+[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fcritical)
+
+Guideline 1. A defect that breaks the API definition is regarded as critical.
+For example:
+
+* [client-java/issues/412](https://github.com/tikv/client-java/issues/412)
+in this defect, gRPC timeout is not set for certain requests, which causes the
+requests can not be terminated as expected when the processing time is too long.
+
+### Major Defects
+
+A defect that affects critical data or functionality and forces users to employ
+a workaround is classified as a major defect. These defects are labeled with
+`type/bug` and `severity/major`, can be found
+[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fmajor)
+
+### Moderate Defects
+
+A defect that affects non-critical data or functionality and forces users to
+employ a workaround is classified as moderate defect. These defects are labeled
+with `type/bug` and `severity/moderate`, can be found
+[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fmoderate)
+
+### Minor Defects
+
+A defect that does not affect data or functionality. It does not even need a
+workaround. It does not impact productivity or efficiency. It is merely an
+inconvenience. These defects are labeled with `type/bug` and `severity/minor`,
+can be found
+[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fminor)

docs/src/contribution/introduction.md

@@ -0,0 +1,21 @@
+# Contribution Guide
+
+## Build the package
+
+```
+mvn clean package -Dmaven.test.skip=true
+```
+
+## Install the package to local maven repository
+
+```
+mvn clean install -Dmaven.test.skip=true
+```
+
+## Run tests
+
+```
+export RAWKV_PD_ADDRESSES=127.0.0.1:2379
+export TXNKV_PD_ADDRESSES=127.0.0.1:2379
+mvn clean test
+```

docs/src/examples/introduction.md

@@ -0,0 +1,3 @@
+# Start With Examples
+
+This section contains examples to demonstrate basic usages of the java client.

docs/src/examples/quick-start.md

@@ -0,0 +1,109 @@
+# Quick Start
+
+The package is hosted on maven central repository. To build from source, refer to the [Contribution Guide](../contribution/introduction.html).
+
+## Create a maven project
+
+First download [maven] and follow the [installation instructoins][install]. Then `mvn` command should be available in the `$PATH`.
+
+[maven]: https://maven.apache.org/download.html
+[install]: https://maven.apache.org/install.html
+
+create a maven project by following command:
+
+```
+mvn archetype:generate -DgroupId=com.example -DartifactId=java-client-example -DarchetypeArtifactId=maven-archetype-quickstart -DinteractiveMode=false
+cd java-client-example
+```
+
+## Add dependency
+
+Add maven dependency to `pom.xml`.
+
+```xml
+<dependency>
+  <groupId>org.tikv</groupId>
+  <artifactId>tikv-client-java</artifactId>
+  <version>3.3.0</version>
+</dependency>
+<dependency>
+  <groupId>org.slf4j</groupId>
+  <artifactId>slf4j-log4j12</artifactId>
+  <version>1.7.32</version>
+</dependency>
+```
+
+Now `pom.xml` should look like this:
+
+```xml
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.example</groupId>
+  <artifactId>java-project</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>java-project</name>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.12</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.tikv</groupId>
+      <artifactId>tikv-client-java</artifactId>
+      <version>3.1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>1.7.32</version>
+    </dependency>
+  </dependencies>
+</project>
+```
+
+## Writing code
+
+To interact with TiKV, we should first create a `TiConfiguration` with PD address, create a `TiSession` using `TiSession.create`, and then create a client.
+For example, if we want to put a `World` in `Hello` key in RawKV, write the following code in `src/main/java/com/example/App.java`.
+
+```java
+import org.tikv.common.TiConfiguration;
+import org.tikv.common.TiSession;
+import org.tikv.raw.RawKVClient;
+import org.tikv.shade.com.google.protobuf.ByteString;
+
+public class App {
+  public static void main(String[] args) throws Exception {
+    String pdAddr = "127.0.0.1:2379";
+    // You MUST create a raw configuration if you are using RawKVClient.
+    TiConfiguration conf = TiConfiguration.createRawDefault(pdAddr);
+    try (TiSession session = TiSession.create(conf)) {
+      try (RawKVClient client = session.createRawClient()) {
+        client.put(ByteString.copyFromUtf8("Hello"), ByteString.copyFromUtf8("World"));
+        ByteString value = client.get(ByteString.copyFromUtf8("Hello"));
+        System.out.println(value);
+      }
+    }
+  }
+}
+```
+
+More examples for RawKV and TxnKV are in following chapters.
+
+## Running program
+
+Run following command:
+
+```
+mvn assembly:assembly -DdescriptorId=jar-with-dependencies
+java -cp target/java-client-example-1.0-SNAPSHOT-jar-with-dependencies.jar com.example.App
+```

docs/src/examples/rawkv.md

@@ -0,0 +1,100 @@
+# RawKV
+
+Below is the basic usages of RawKV. See [API document] to see a full list of methods available.
+
+[API document]: https://tikv.github.io/client-java/apidocs/org/tikv/raw/RawKVClient
+
+```java
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import org.tikv.common.TiConfiguration;
+import org.tikv.common.TiSession;
+import org.tikv.kvproto.Kvrpcpb;
+import org.tikv.raw.RawKVClient;
+import org.tikv.shade.com.google.protobuf.ByteString;
+
+public class Main {
+  public static void main(String[] args) throws Exception {
+    // You MUST create a raw configuration if you are using RawKVClient.
+    TiConfiguration conf = TiConfiguration.createRawDefault("127.0.0.1:2379");
+    TiSession session = TiSession.create(conf);
+    RawKVClient client = session.createRawClient();
+
+    // put
+    client.put(ByteString.copyFromUtf8("k1"), ByteString.copyFromUtf8("Hello"));
+    client.put(ByteString.copyFromUtf8("k2"), ByteString.copyFromUtf8(","));
+    client.put(ByteString.copyFromUtf8("k3"), ByteString.copyFromUtf8("World"));
+    client.put(ByteString.copyFromUtf8("k4"), ByteString.copyFromUtf8("!"));
+    client.put(ByteString.copyFromUtf8("k5"), ByteString.copyFromUtf8("Raw KV"));
+
+    // get
+    Optional<ByteString> result = client.get(ByteString.copyFromUtf8("k1"));
+    System.out.println(result.get().toStringUtf8());
+
+    // batch get
+    List<Kvrpcpb.KvPair> list = client.batchGet(new ArrayList<ByteString>() {{
+        add(ByteString.copyFromUtf8("k1"));
+        add(ByteString.copyFromUtf8("k3"));
+    }});
+    System.out.println(list);
+
+    // scan
+    list = client.scan(ByteString.copyFromUtf8("k1"), ByteString.copyFromUtf8("k6"), 10);
+    System.out.println(list);
+
+    // close
+    client.close();
+    session.close();
+  }
+}
+```
+
+
+## API V2
+With TiKV version >= 6.1.0, we release a new feature called "TiKV API V2" which provides a new raw key-value storage format allowing the coexistence of RawKV and TxnKV. Please refer to [v6.10 release notes](https://docs.pingcap.com/tidb/stable/release-6.1.0#ease-of-use) for detail.
+
+To enable the API V2 mode, users need to specify the API version of the client.
+
+```java
+// import ...
+import org.tikv.common.TiConfiguration.ApiVersion;
+
+public class Main {
+  public static void main(String[] args) throws Exception {
+    TiConfiguration conf = TiConfiguration.createRawDefault("127.0.0.1:2379");
+    conf.setApiVersion(ApiVersion.V2);
+    try(TiSession session = TiSession.create(conf)) {
+      try(RawKVClient client = session.createRawClient()) {
+        // The client will read and write date in the format of API V2, which is
+        // transparent to the users.
+        client.put(ByteString.copyFromUtf8("hello"), ByteString.copyFromUtf8("world"));
+        // other client operations.
+      }
+    }
+  }
+}
+```
+
+### Compatibility
+
+The V2 Client should not access the cluster other than V2, this requires users to [enable the API V2](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#api-version-new-in-v610) for the cluster:
+
+```toml
+[storage]
+# The V2 cluster must enable ttl for RawKV explicitly
+enable-ttl = true
+api-version = 2
+```
+
+If V2 client accesses a V1 cluster or V1 cluster accesses a V2 cluster, the requests will be denied by the cluster. You can check the compatibility via the following matrix.
+
+
+|                       | V1 Server | V1TTL Server | V2 Server |
+| --------------------- | --------- | ------------ | --------- |
+| V1 RawClient          | Raw       | Raw          | Error     |
+| V1 RawClient with TTL | Error     | Raw          | Error     |
+| V1 TxnClient          | Txn       | Error        | Error     |
+| V1 TiDB               | TiDB Data | Error        | TiDB Data |
+| V2 RawClient          | Error     | Error        | Raw       |
+| V2 TxnClient          | Error     | Error        | Txn       |

docs/src/examples/txnkv.md

@@ -0,0 +1,69 @@
+# TxnKV
+
+Below is the basic usages of TxnKV.
+Data should be written into TxnKV using [`TwoPhaseCommitter`], and be read using [`org.tikv.txn.KVClient`][KVClient].
+
+[`TwoPhaseCommitter`]: https://tikv.github.io/client-java/apidocs/org/tikv/txn/TwoPhaseCommitter.html
+[KVClient]: https://tikv.github.io/client-java/apidocs/org/tikv/txn/KVClient.html
+
+```java
+import java.util.Arrays;
+import java.util.List;
+
+import org.tikv.common.BytePairWrapper;
+import org.tikv.common.ByteWrapper;
+import org.tikv.common.TiConfiguration;
+import org.tikv.common.TiSession;
+import org.tikv.common.util.BackOffer;
+import org.tikv.common.util.ConcreteBackOffer;
+import org.tikv.kvproto.Kvrpcpb.KvPair;
+import org.tikv.shade.com.google.protobuf.ByteString;
+import org.tikv.txn.KVClient;
+import org.tikv.txn.TwoPhaseCommitter;
+
+public class App {
+    public static void main(String[] args) throws Exception {
+        TiConfiguration conf = TiConfiguration.createDefault("127.0.0.1:2379");
+        try (TiSession session = TiSession.create(conf)) {
+            // two-phrase write
+            long startTS = session.getTimestamp().getVersion();
+            try (TwoPhaseCommitter twoPhaseCommitter = new TwoPhaseCommitter(session, startTS)) {
+                BackOffer backOffer = ConcreteBackOffer.newCustomBackOff(1000);
+                byte[] primaryKey = "key1".getBytes("UTF-8");
+                byte[] key2 = "key2".getBytes("UTF-8");
+
+                // first phrase: prewrite
+                twoPhaseCommitter.prewritePrimaryKey(backOffer, primaryKey, "val1".getBytes("UTF-8"));
+                List<BytePairWrapper> pairs = Arrays
+                        .asList(new BytePairWrapper(key2, "val2".getBytes("UTF-8")));
+                twoPhaseCommitter.prewriteSecondaryKeys(primaryKey, pairs.iterator(), 1000);
+
+                // second phrase: commit
+                long commitTS = session.getTimestamp().getVersion();
+                twoPhaseCommitter.commitPrimaryKey(backOffer, primaryKey, commitTS);
+                List<ByteWrapper> keys = Arrays.asList(new ByteWrapper(key2));
+                twoPhaseCommitter.commitSecondaryKeys(keys.iterator(), commitTS, 1000);
+            }
+
+            try (KVClient kvClient = session.createKVClient()) {
+                long version = session.getTimestamp().getVersion();
+                ByteString key1 = ByteString.copyFromUtf8("key1");
+                ByteString key2 = ByteString.copyFromUtf8("key2");
+
+                // get value of a single key
+                ByteString val = kvClient.get(key1, version);
+                System.out.println(val);
+
+                // get value of multiple keys
+                BackOffer backOffer = ConcreteBackOffer.newCustomBackOff(1000);
+                List<KvPair> kvPairs = kvClient.batchGet(backOffer, Arrays.asList(key1, key2), version);
+                System.out.println(kvPairs);
+
+                // get value of a range of keys
+                kvPairs = kvClient.scan(key1, ByteString.copyFromUtf8("key3"), version);
+                System.out.println(kvPairs);
+            }
+        }
+    }
+}
+```

docs/src/introduction/introduction.md

@@ -0,0 +1,15 @@
+# Introduction
+
+Welcome to the documents of TiKV Java Client. This document is oriented to:
+
+1. Application developers who needs to integrate java client to their
+   application code. There are documents about best practices, administrations,
+   troubleshootings, and internals of TiKV Java Client for developers.
+
+2. TiKV contributors who wish to contribute to TiKV Java Client, become a TiKV
+   reviewer, committer, or maintainer. There are documents about setting up
+   environment, submitting PR, triage issues, and manage releases in the
+   community.
+
+Wish you have a good journey in developing application or contributing to the
+TiKV Java Client.

docs/src/performance/introduction.md

@@ -0,0 +1 @@
+# Performance

docs/src/performance/ycsb.md

@@ -0,0 +1 @@
+# YCSB Benchmarks

docs/src/production-readiness.md

@@ -0,0 +1,18 @@
+# Production Readiness
+
+In general, the latest [release](https://github.com/tikv/client-java/releases) of TiKV Java Client is ready for production use. But it is not battle-tested as full featured client for TiKV in all use cases. This page will give you more details.
+
+## RawKV
+All RawKV APIs are covered by [CI](https://github.com/tikv/client-java/actions/workflows/ci.yml).
+
+At this time, RawKV has been used in the production environment of some commercial customers in latency sensitive systems. But they only use part of the RawKV APIs (mainly including `raw_put`, `raw_get`, `raw_compare_and_swap`, and `raw_batch_put`).
+
+## TxnKV
+All TxnKV APIs are covered by [CI](https://github.com/tikv/client-java/actions/workflows/ci.yml).
+
+In addition, TxnKV has been used in the [TiSpark](https://docs.pingcap.com/tidb/stable/tispark-overview) and [TiBigData](https://github.com/tidb-incubator/TiBigData) project to integrate data from TiDB to Big Data ecosystem. TiSpark and TiBigData are used in the production system of some commercial customers and internet companies.
+
+Similar to RawKV, only part of APIs are used in this scenario (mainly including `prewrite/commit` and `coprocessor`). And this use case doesn't care about latency but throughput and reliability.
+
+## TiDB Cloud
+Directly using TiKV is not possible on TiDB Cloud due to the fact that client has to access the whole cluster, which has security issues. And TiKV managed service is not coming soon as it's not contained in [roadmap](https://docs.pingcap.com/tidbcloud/tidb-cloud-roadmap) yet.

docs/src/troubleshooting/error-request.md

@@ -0,0 +1 @@
+# Error Request Diagnosis

docs/src/troubleshooting/introduction.md

@@ -0,0 +1 @@
+# Troubleshooting

docs/src/troubleshooting/slow-request.md

@@ -0,0 +1,34 @@
+# Slow Request Diagnosis
+
+If a request take too much time, we can collect the detailed time spend in each component in a “slow log”.
+
+<!-- wrap text in the code block -->
+<pre>
+<code class="hljs" style="white-space: pre-wrap;">
+2022-02-11 11:07:56 WARN  SlowLogImpl:88 - A request spent 55 ms. start=11:07:56.938, end=11:07:56.993, SlowLog:{"trace_id":4361090673996453790,"spans":[{"event":"put","begin":"11:07:56.938","duration_ms":55,"properties":{"region":"{Region[2] ConfVer[5] Version[60] Store[1] KeyRange[]:[]}","key":"Hello"}},{"event":"getRegionByKey","begin":"11:07:56.938","duration_ms":0},{"event":"callWithRetry","begin":"11:07:56.943","duration_ms":49,"properties":{"method":"tikvpb.Tikv/RawPut"}},{"event":"gRPC","begin":"11:07:56.943","duration_ms":49,"properties":{"method":"tikvpb.Tikv/RawPut"}}]}
+</code>
+</pre>
+
+## Slow log configurations
+
+| SlowLog settings | default value |
+| -- | -- |
+| tikv.rawkv.read_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
+| tikv.rawkv.write_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
+| tikv.rawkv.batch_read_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
+| tikv.rawkv.batch_write_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
+| tikv.rawkv.scan_slowlog_in_ms | 5s |
+
+Each settings can be set by system properties, configuration files or `set...` method of `TiConfiguration`.
+
+System properties can be set by `-D` parameter of `java` command.
+
+```
+java -cp target/java-client-example-1.0-SNAPSHOT-jar-with-dependencies.jar -Dtikv.rawkv.read_slowlog_in_ms=100 com.example.App
+```
+
+Configuration file is `src/main/resources/tikv.properties` in maven projects.
+
+## Visualize slow log
+
+TBD

pom.xml

@@ -1,16 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
-
     <groupId>org.tikv</groupId>
     <artifactId>tikv-client-java</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.3.4-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>TiKV Java Client</name>
     <description>A Java Client for TiKV</description>
     <url>http://github.com/tikv/client-java</url>
-
     <licenses>
         <license>
             <name>Apache 2.0 License</name>
@@ -18,11 +15,9 @@
             <distribution>repo</distribution>
         </license>
     </licenses>
-
     <organization>
         <name>PingCAP</name>
     </organization>
-
     <developers>
         <developer>
             <name>Xiaoyu Ma</name>
@@ -49,33 +44,54 @@
             <organizationUrl>https://www.pingcap.com</organizationUrl>
         </developer>
     </developers>
-
     <scm>
         <connection>scm:git:git://github.com/tikv/client-java.git</connection>
         <developerConnection>scm:git:ssh://github.com:tikv/client-java.git</developerConnection>
         <url>https://github.com/tikv/client-java/tree/master</url>
     </scm>
-
     <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <protobuf.version>3.5.1</protobuf.version>
+        <protobuf.version>3.18.0</protobuf.version>
         <log4j.version>1.2.17</log4j.version>
         <slf4j.version>1.7.16</slf4j.version>
-        <grpc.version>1.24.0</grpc.version>
+        <grpc.version>1.60.0</grpc.version>
+        <netty.tcnative.version>2.0.34.Final</netty.tcnative.version>
+        <gson.version>2.8.9</gson.version>
         <powermock.version>1.6.6</powermock.version>
-        <jackson.version>2.10.0</jackson.version>
+        <jackson-annotations.version>2.13.2</jackson-annotations.version>
+        <jackson.version>2.13.4.2</jackson.version>
         <trove4j.version>3.0.1</trove4j.version>
-        <jetcd.version>0.4.1</jetcd.version>
+        <jetcd.version>0.7.7</jetcd.version>
         <joda-time.version>2.9.9</joda-time.version>
         <joda-convert.version>1.9.2</joda-convert.version>
         <proto.folder>${basedir}/proto</proto.folder>
         <gpg.keyname>fake gpg key name</gpg.keyname>
         <gpg.skip>true</gpg.skip>
-        <javadoc.skip>true</javadoc.skip>
     </properties>
-
     <dependencies>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.19.6</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java-util</artifactId>
+            <version>3.19.6</version>
+        </dependency>
+        <dependency>
+            <groupId>io.perfmark</groupId>
+            <artifactId>perfmark-api</artifactId>
+            <version>0.24.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.perfmark</groupId>
+            <artifactId>perfmark-traceviewer</artifactId>
+            <version>0.24.0</version>
+        </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
@@ -120,22 +136,49 @@
             <groupId>io.grpc</groupId>
             <artifactId>grpc-protobuf</artifactId>
             <version>${grpc.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-stub</artifactId>
             <version>${grpc.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-services</artifactId>
+            <version>${grpc.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java-util</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-tcnative-boringssl-static</artifactId>
+            <version>${netty.tcnative.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-testing</artifactId>
             <version>${grpc.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>${gson.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
-            <version>${jackson.version}</version>
+            <version>${jackson-annotations.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
@@ -145,20 +188,6 @@
         <dependency>
             <groupId>io.etcd</groupId>
             <artifactId>jetcd-core</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>io.etcd</groupId>
-                    <artifactId>jetcd-resolver</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>io.etcd</groupId>
-                    <artifactId>jetcd-common</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>io.grpc</groupId>
-                    <artifactId>grpc-grpclb</artifactId>
-                </exclusion>
-            </exclusions>
             <version>${jetcd.version}</version>
         </dependency>
         <dependency>
@@ -180,7 +209,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.9</version>
+            <version>3.10</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -189,7 +218,65 @@
             <version>3.9</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.15</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.13</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient</artifactId>
+            <version>0.10.0</version>
+        </dependency>
+        <!-- Hotspot JVM metrics-->
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_hotspot</artifactId>
+            <version>0.10.0</version>
+        </dependency>
+        <!-- Exposition HTTPServer-->
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_httpserver</artifactId>
+            <version>0.10.0</version>
+        </dependency>
+        <!-- Pushgateway exposition-->
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_pushgateway</artifactId>
+            <version>0.10.0</version>
+        </dependency>
     </dependencies>
+    <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>3.3.1</version>
+            </plugin>
+            <plugin>
+                <groupId>com.github.spotbugs</groupId>
+                <artifactId>spotbugs-maven-plugin</artifactId>
+                <version>4.5.2.0</version>
+                <configuration>
+                    <xmlOutput>true</xmlOutput>
+                    <!-- Optional directory to put spotbugs xdoc xml report -->
+                    <xmlOutputDirectory>target/site</xmlOutputDirectory>
+                    <includeFilterFile>dev/spotbugs-include.xml</includeFilterFile>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jxr-plugin</artifactId>
+                <version>3.1.1</version>
+            </plugin>
+        </plugins>
+    </reporting>
     <build>
         <resources>
             <resource>
@@ -233,7 +320,7 @@
                     <execution>
                         <id>clone proto files</id>
                         <configuration>
-                            <executable>${basedir}/scripts/proto.sh</executable>
+                            <executable>${basedir}/dev/proto.sh</executable>
                         </configuration>
                         <phase>validate</phase>
                         <goals>
@@ -284,7 +371,6 @@
                     </execution>
                 </executions>
             </plugin>
-
             <plugin>
                 <groupId>org.xolstice.maven.plugins</groupId>
                 <artifactId>protobuf-maven-plugin</artifactId>
@@ -307,7 +393,6 @@
                     </execution>
                 </executions>
             </plugin>
-
             <!-- Compiler Plug-in -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -359,26 +444,6 @@
                     </filesets>
                 </configuration>
             </plugin>
-            <!-- Javadoc Plug-in -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
-                <configuration>
-                    <skip>${javadoc.skip}</skip>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>attach-javadocs</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                        <configuration> <!-- add this to disable checking -->
-                            <additionalparam>-Xdoclint:none</additionalparam>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
             <!--- Needs to shade Protobuf 3 since other projects might use other version -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -397,44 +462,33 @@
                             <relocations>
                                 <relocation>
                                     <pattern>io.grpc</pattern>
-                                    <shadedPattern>shade.io.grpc</shadedPattern>
+                                    <shadedPattern>org.tikv.shade.io.grpc</shadedPattern>
                                 </relocation>
                                 <relocation>
                                     <pattern>com.google</pattern>
-                                    <shadedPattern>shade.com.google</shadedPattern>
+                                    <shadedPattern>org.tikv.shade.com.google</shadedPattern>
                                 </relocation>
                                 <relocation>
                                     <pattern>io.netty</pattern>
-                                    <shadedPattern>shade.io.netty</shadedPattern>
+                                    <shadedPattern>org.tikv.shade.io.netty</shadedPattern>
                                 </relocation>
                                 <relocation>
                                     <pattern>io.opencensus</pattern>
-                                    <shadedPattern>shade.io.opencensus</shadedPattern>
+                                    <shadedPattern>org.tikv.shade.io.opencensus</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>io.prometheus</pattern>
+                                    <shadedPattern>org.tikv.shade.io.prometheus</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>com.fasterxml.jackson</pattern>
+                                    <shadedPattern>org.tikv.shade.com.fasterxml.jackson</shadedPattern>
                                 </relocation>
                             </relocations>
                         </configuration>
                     </execution>
                 </executions>
             </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>3.1.0</version>
-                <executions>
-                    <execution>
-                        <id>jar</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                        <configuration>
-                            <descriptorRefs>
-                                <descriptorRef>jar-with-dependencies</descriptorRef>
-                            </descriptorRefs>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
             <!-- Code coverage test -->
             <plugin>
                 <groupId>org.jacoco</groupId>
@@ -449,7 +503,7 @@
                     </execution>
                     <execution>
                         <id>jacoco-site</id>
-                        <phase>package</phase>
+                        <phase>test</phase>
                         <goals>
                             <goal>report</goal>
                         </goals>
@@ -506,9 +560,110 @@
                 <configuration>
                     <serverId>ossrh</serverId>
                     <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-                    <autoReleaseAfterClose>false</autoReleaseAfterClose>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>io.github.git-commit-id</groupId>
+                <artifactId>git-commit-id-maven-plugin</artifactId>
+                <version>4.9.9</version>
+                <executions>
+                    <execution>
+                        <id>get-the-git-infos</id>
+                        <goals>
+                            <goal>revision</goal>
+                        </goals>
+                        <phase>initialize</phase>
+                    </execution>
+                    <execution>
+                        <id>validate-the-git-infos</id>
+                        <goals>
+                            <goal>validateRevision</goal>
+                        </goals>
+                        <phase>package</phase>
+                    </execution>
+                </executions>
+                <configuration>
+                    <verbose>true</verbose>
+                    <generateGitPropertiesFile>true</generateGitPropertiesFile>
+                    <generateGitPropertiesFilename>${project.build.outputDirectory}/git.properties</generateGitPropertiesFilename>
+                    <includeOnlyProperties>
+                        <includeOnlyProperty>^git.build.(time|version)$</includeOnlyProperty>
+                        <includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
+                    </includeOnlyProperties>
+                    <commitIdGenerationMode>full</commitIdGenerationMode>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.0.0-M5</version>
+                <configuration>
+                    <forkMode>always</forkMode>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-site-plugin</artifactId>
+                <version>3.7.1</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-project-info-reports-plugin</artifactId>
+                <version>3.0.0</version>
+            </plugin>
+            <plugin>
+                <groupId>com.github.spotbugs</groupId>
+                <artifactId>spotbugs-maven-plugin</artifactId>
+                <version>4.5.2.0</version>
+                <configuration>
+                    <includeFilterFile>dev/spotbugs-include.xml</includeFilterFile>
+                    <xmlOutput>true</xmlOutput>
+                    <failOnError>false</failOnError>
+                </configuration>
+                <dependencies>
+                    <!-- overwrite dependency on spotbugs if you want to specify the version of spotbugs -->
+                    <dependency>
+                        <groupId>com.github.spotbugs</groupId>
+                        <artifactId>spotbugs</artifactId>
+                        <version>4.5.3</version>
+                    </dependency>
+                </dependencies>
+            </plugin>
+            <plugin>
+                <groupId>au.com.acegi</groupId>
+                <artifactId>xml-format-maven-plugin</artifactId>
+                <version>3.2.0</version>
+                <executions>
+                    <execution>
+                        <id>xml-format</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>xml-format</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <indentSize>4</indentSize>
                 </configuration>
             </plugin>
         </plugins>
     </build>
+    <profiles>
+        <profile>
+            <id>jdk9plus</id>
+            <activation>
+                <!-- activated when building with JDK9+ -->
+                <jdk>!1.8</jdk>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>javax.annotation</groupId>
+                    <artifactId>javax.annotation-api</artifactId>
+                    <version>1.3.2</version>
+                    <scope>provided</scope>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
 </project>

scripts/proto.sh

@@ -1,45 +0,0 @@
-#!/bin/sh
-#
-#   Copyright 2017 PingCAP, Inc.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-
-CURRENT_DIR=`pwd`
-TIKV_CLIENT_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-cd $TIKV_CLIENT_HOME
-
-kvproto_hash=2cf9a243b8d589f345de1dbaa9eeffec6afbdc06
-
-raft_rs_hash=b9891b673573fad77ebcf9bbe0969cf945841926
-
-tipb_hash=c4d518eb1d60c21f05b028b36729e64610346dac
-
-if [ -d "kvproto" ]; then
-	cd kvproto; git fetch -p; git checkout ${kvproto_hash}; cd ..
-else
-	git clone https://github.com/pingcap/kvproto; cd kvproto; git checkout ${kvproto_hash}; cd ..
-fi
-
-if [ -d "raft-rs" ]; then
-	cd raft-rs; git fetch -p; git checkout ${raft_rs_hash}; cd ..
-else
-	git clone https://github.com/pingcap/raft-rs; cd raft-rs; git checkout ${raft_rs_hash}; cd ..
-fi
-
-if [ -d "tipb" ]; then
-	cd tipb; git fetch -p; git checkout ${tipb_hash}; cd ..
-else
-	git clone https://github.com/pingcap/tipb; cd tipb; git checkout ${tipb_hash}; cd ..
-fi
-
-cd $CURRENT_DIR

scripts/rstats.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+#!coding:utf-8
+
+# Copyright 2022 TiKV Project Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import re
+import json
+import argparse
+
+def main():
+    pattern = r'.*SlowLog:.*'
+    slowstr = "SlowLog:"
+    grpc_pattern = "gRPC tikvpb.Tikv"
+    backoff_pattern = "backoff "
+
+    args = parse_args()
+    items = []
+    with open(args.slowlog, encoding = 'utf-8') as f:
+        for line in f.readlines():
+            matched = re.match(pattern, line, re.M|re.I)
+            if matched is not None:
+                log = json.loads(line[(line.index(slowstr) + len(slowstr)):])
+                item = {
+                    'req': log['func'],
+                    'start': log['start'],
+                    'tot_lat': latency_ms(log),
+                    'tot_grpc': 0,
+                    'tot_bo': 0,
+                }
+                items.append(item)
+                for span in log['spans']:
+                    if grpc_pattern in span['name'] and span['duration'] != 'N/A':
+                        item['tot_grpc'] += latency_ms(span)
+                    elif backoff_pattern in span['name'] and span['duration'] != 'N/A':
+                        item['tot_bo'] += latency_ms(span)
+
+        if args.order == "total":
+            items = sorted(items, key=lambda d: d['tot_lat'], reverse=True)
+        elif args.order == "grpc":
+            items = sorted(items, key=lambda d: d['tot_grpc'], reverse=True)
+        elif args.order == "backoff":
+            items = sorted(items, key=lambda d: d['tot_bo'], reverse=True)
+        else:
+            print("unsupported order option, use default value: total")
+            items = sorted(items, key=lambda d: d['tot_lat'], reverse=True)
+
+    fmtStr = "{:<12} {:<14} {:<14} {:<20} {:<20}"
+    print(fmtStr.format("Request", "Start", "Total Lat(ms)", "Total gRPC Lat(ms)", "Total Backoff Lat(ms)"))
+    for item in items:
+        print(fmtStr.format(item['req'], item['start'], item['tot_lat'], item['tot_grpc'], item['tot_bo']))
+
+def latency_ms(span):
+    return int(span['duration'][:len(span['duration'])-2])
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="rstats: A TiKV Java Client Request Stats Analyzer")
+    parser.add_argument("-o", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff")
+    parser.add_argument("slowlog", help="slow log file")
+    return parser.parse_args()
+
+if __name__ == '__main__':
+    main()
+

src/main/java/org/tikv/BUILD

@@ -1,34 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-java_library(
-    name = "tikv-java-client-lib",
-    srcs = glob(
-        ["**/*.java"],
-    ),
-    deps = [
-        "//:java",
-        "@com_fasterxml_jackson_core_jackson_annotations//jar",
-        "@com_fasterxml_jackson_core_jackson_core//jar",
-        "@com_fasterxml_jackson_core_jackson_databind//jar",
-        "@com_google_code_findbugs_jsr305//jar",
-        "@com_google_code_gson_gson//jar",
-        "@com_google_errorprone_error_prone_annotations//jar",
-        "@com_google_guava_guava//jar",
-        "@com_google_protobuf_protobuf_java//jar",
-        "@joda_time//jar",
-        # the following are defined in rules_protobuf
-        "@org_pubref_rules_protobuf//java:grpc_compiletime_deps",
-        "@org_pubref_rules_protobuf//java:netty_runtime_deps",
-
-        "@org_slf4j_slf4j_api//jar",
-        "@org_slf4j_jcl_over_slf4j//jar",
-        "@org_slf4j_jul_to_slf4j//jar",
-        "@log4j_log4j//jar",
-        "@net_sf_trove4j_trove4j//jar",
-    ],
-)
-
-filegroup(
-    name = "srcs",
-    srcs = ["BUILD"] + glob(["**/*.java"]),
-)

src/main/java/org/tikv/Main.java

@@ -1,5 +0,0 @@
-package org.tikv;
-
-public class Main {
-  public static void main(String args[]) throws Exception {}
-}

src/main/java/org/tikv/common/AbstractGRPCClient.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -18,10 +20,16 @@ package org.tikv.common;
 import static io.grpc.stub.ClientCalls.asyncBidiStreamingCall;
 import static io.grpc.stub.ClientCalls.blockingServerStreamingCall;
 
+import io.grpc.ManagedChannel;
 import io.grpc.MethodDescriptor;
+import io.grpc.health.v1.HealthCheckRequest;
+import io.grpc.health.v1.HealthCheckResponse;
+import io.grpc.health.v1.HealthGrpc;
+import io.grpc.stub.AbstractFutureStub;
 import io.grpc.stub.AbstractStub;
 import io.grpc.stub.ClientCalls;
 import io.grpc.stub.StreamObserver;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -29,20 +37,24 @@ import org.tikv.common.operation.ErrorHandler;
 import org.tikv.common.policy.RetryMaxMs.Builder;
 import org.tikv.common.policy.RetryPolicy;
 import org.tikv.common.streaming.StreamingResponse;
+import org.tikv.common.util.BackOffFunction.BackOffFuncType;
 import org.tikv.common.util.BackOffer;
 import org.tikv.common.util.ChannelFactory;
 
 public abstract class AbstractGRPCClient<
-        BlockingStubT extends AbstractStub<BlockingStubT>, StubT extends AbstractStub<StubT>>
+        BlockingStubT extends AbstractStub<BlockingStubT>,
+        FutureStubT extends AbstractFutureStub<FutureStubT>>
     implements AutoCloseable {
   protected final Logger logger = LoggerFactory.getLogger(this.getClass());
   protected final ChannelFactory channelFactory;
   protected TiConfiguration conf;
+  protected long timeout;
   protected BlockingStubT blockingStub;
-  protected StubT asyncStub;
+  protected FutureStubT asyncStub;
 
   protected AbstractGRPCClient(TiConfiguration conf, ChannelFactory channelFactory) {
     this.conf = conf;
+    this.timeout = conf.getTimeout();
     this.channelFactory = channelFactory;
   }
 
@@ -50,8 +62,9 @@ public abstract class AbstractGRPCClient<
       TiConfiguration conf,
       ChannelFactory channelFactory,
       BlockingStubT blockingStub,
-      StubT asyncStub) {
+      FutureStubT asyncStub) {
     this.conf = conf;
+    this.timeout = conf.getTimeout();
     this.channelFactory = channelFactory;
     this.blockingStub = blockingStub;
     this.asyncStub = asyncStub;
@@ -70,17 +83,16 @@ public abstract class AbstractGRPCClient<
     if (logger.isTraceEnabled()) {
       logger.trace(String.format("Calling %s...", method.getFullMethodName()));
     }
-    RetryPolicy.Builder<RespT> builder = new Builder<>(backOffer);
+    RetryPolicy<RespT> policy = new Builder<RespT>(backOffer).create(handler);
     RespT resp =
-        builder
+        policy.callWithRetry(
-            .create(handler)
+            () -> {
-            .callWithRetry(
+              BlockingStubT stub = getBlockingStub();
-                () -> {
+              return ClientCalls.blockingUnaryCall(
-                  BlockingStubT stub = getBlockingStub();
+                  stub.getChannel(), method, stub.getCallOptions(), requestFactory.get());
-                  return ClientCalls.blockingUnaryCall(
+            },
-                      stub.getChannel(), method, stub.getCallOptions(), requestFactory.get());
+            method.getFullMethodName(),
-                },
+            backOffer);
-                method.getFullMethodName());
 
     if (logger.isTraceEnabled()) {
       logger.trace(String.format("leaving %s...", method.getFullMethodName()));
@@ -96,19 +108,18 @@ public abstract class AbstractGRPCClient<
       ErrorHandler<RespT> handler) {
     logger.debug(String.format("Calling %s...", method.getFullMethodName()));
 
-    RetryPolicy.Builder<RespT> builder = new Builder<>(backOffer);
+    RetryPolicy<RespT> policy = new Builder<RespT>(backOffer).create(handler);
-    builder
+    policy.callWithRetry(
-        .create(handler)
+        () -> {
-        .callWithRetry(
+          FutureStubT stub = getAsyncStub();
-            () -> {
+          ClientCalls.asyncUnaryCall(
-              StubT stub = getAsyncStub();
+              stub.getChannel().newCall(method, stub.getCallOptions()),
-              ClientCalls.asyncUnaryCall(
+              requestFactory.get(),
-                  stub.getChannel().newCall(method, stub.getCallOptions()),
+              responseObserver);
-                  requestFactory.get(),
+          return null;
-                  responseObserver);
+        },
-              return null;
+        method.getFullMethodName(),
-            },
+        backOffer);
-            method.getFullMethodName());
     logger.debug(String.format("leaving %s...", method.getFullMethodName()));
   }
 
@@ -119,17 +130,17 @@ public abstract class AbstractGRPCClient<
       ErrorHandler<StreamObserver<ReqT>> handler) {
     logger.debug(String.format("Calling %s...", method.getFullMethodName()));
 
-    RetryPolicy.Builder<StreamObserver<ReqT>> builder = new Builder<>(backOffer);
+    RetryPolicy<StreamObserver<ReqT>> policy =
+        new Builder<StreamObserver<ReqT>>(backOffer).create(handler);
     StreamObserver<ReqT> observer =
-        builder
+        policy.callWithRetry(
-            .create(handler)
+            () -> {
-            .callWithRetry(
+              FutureStubT stub = getAsyncStub();
-                () -> {
+              return asyncBidiStreamingCall(
-                  StubT stub = getAsyncStub();
+                  stub.getChannel().newCall(method, stub.getCallOptions()), responseObserver);
-                  return asyncBidiStreamingCall(
+            },
-                      stub.getChannel().newCall(method, stub.getCallOptions()), responseObserver);
+            method.getFullMethodName(),
-                },
+            backOffer);
-                method.getFullMethodName());
     logger.debug(String.format("leaving %s...", method.getFullMethodName()));
     return observer;
   }
@@ -141,23 +152,58 @@ public abstract class AbstractGRPCClient<
       ErrorHandler<StreamingResponse> handler) {
     logger.debug(String.format("Calling %s...", method.getFullMethodName()));
 
-    RetryPolicy.Builder<StreamingResponse> builder = new Builder<>(backOffer);
+    RetryPolicy<StreamingResponse> policy =
+        new Builder<StreamingResponse>(backOffer).create(handler);
     StreamingResponse response =
-        builder
+        policy.callWithRetry(
-            .create(handler)
+            () -> {
-            .callWithRetry(
+              BlockingStubT stub = getBlockingStub();
-                () -> {
+              return new StreamingResponse(
-                  BlockingStubT stub = getBlockingStub();
+                  blockingServerStreamingCall(
-                  return new StreamingResponse(
+                      stub.getChannel(), method, stub.getCallOptions(), requestFactory.get()));
-                      blockingServerStreamingCall(
+            },
-                          stub.getChannel(), method, stub.getCallOptions(), requestFactory.get()));
+            method.getFullMethodName(),
-                },
+            backOffer);
-                method.getFullMethodName());
     logger.debug(String.format("leaving %s...", method.getFullMethodName()));
     return response;
   }
 
+  public void setTimeout(long timeout) {
+    this.timeout = timeout;
+  }
+
+  public long getTimeout() {
+    return this.timeout;
+  }
+
   protected abstract BlockingStubT getBlockingStub();
 
-  protected abstract StubT getAsyncStub();
+  protected abstract FutureStubT getAsyncStub();
+
+  private boolean doCheckHealth(BackOffer backOffer, String addressStr, HostMapping hostMapping) {
+    while (true) {
+      backOffer.checkTimeout();
+
+      try {
+        ManagedChannel channel = channelFactory.getChannel(addressStr, hostMapping);
+        HealthGrpc.HealthBlockingStub stub =
+            HealthGrpc.newBlockingStub(channel)
+                .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
+        HealthCheckRequest req = HealthCheckRequest.newBuilder().build();
+        HealthCheckResponse resp = stub.check(req);
+        return resp.getStatus() == HealthCheckResponse.ServingStatus.SERVING;
+      } catch (Exception e) {
+        logger.warn("check health failed, addr: {}, caused by: {}", addressStr, e.getMessage());
+        backOffer.doBackOff(BackOffFuncType.BoCheckHealth, e);
+      }
+    }
+  }
+
+  protected boolean checkHealth(BackOffer backOffer, String addressStr, HostMapping hostMapping) {
+    try {
+      return doCheckHealth(backOffer, addressStr, hostMapping);
+    } catch (Exception e) {
+      return false;
+    }
+  }
 }

src/main/java/org/tikv/common/BytePairWrapper.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/ByteWrapper.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/ConfigUtils.java

@@ -0,0 +1,212 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+import org.tikv.common.util.BackOffer;
+import org.tikv.kvproto.Kvrpcpb;
+
+public class ConfigUtils {
+  public static final String TIKV_CONFIGURATION_FILENAME = "tikv.properties";
+
+  public static final String TIKV_PD_ADDRESSES = "tikv.pd.addresses";
+  public static final String TIKV_GRPC_TIMEOUT = "tikv.grpc.timeout_in_ms";
+  public static final String TIKV_GRPC_INGEST_TIMEOUT = "tikv.grpc.ingest_timeout_in_ms";
+  public static final String TIKV_GRPC_FORWARD_TIMEOUT = "tikv.grpc.forward_timeout_in_ms";
+  public static final String TIKV_GRPC_WARM_UP_TIMEOUT = "tikv.grpc.warm_up_timeout_in_ms";
+  public static final String TIKV_PD_FIRST_GET_MEMBER_TIMEOUT =
+      "tikv.grpc.pd_first_get_member_timeout_in_ms";
+  public static final String TIKV_GRPC_SCAN_TIMEOUT = "tikv.grpc.scan_timeout_in_ms";
+  public static final String TIKV_GRPC_SCAN_BATCH_SIZE = "tikv.grpc.scan_batch_size";
+  public static final String TIKV_GRPC_MAX_FRAME_SIZE = "tikv.grpc.max_frame_size";
+  public static final String TIKV_GRPC_KEEPALIVE_TIME = "tikv.grpc.keepalive_time";
+  public static final String TIKV_GRPC_KEEPALIVE_TIMEOUT = "tikv.grpc.keepalive_timeout";
+  public static final String TIKV_GRPC_IDLE_TIMEOUT = "tikv.grpc.idle_timeout";
+  public static final String TIKV_CONN_RECYCLE_TIME = "tikv.conn.recycle_time";
+
+  public static final String TIKV_INDEX_SCAN_BATCH_SIZE = "tikv.index.scan_batch_size";
+  public static final String TIKV_INDEX_SCAN_CONCURRENCY = "tikv.index.scan_concurrency";
+  public static final String TIKV_TABLE_SCAN_CONCURRENCY = "tikv.table.scan_concurrency";
+
+  public static final String TIKV_BATCH_GET_CONCURRENCY = "tikv.batch_get_concurrency";
+  public static final String TIKV_BATCH_PUT_CONCURRENCY = "tikv.batch_put_concurrency";
+  public static final String TIKV_BATCH_DELETE_CONCURRENCY = "tikv.batch_delete_concurrency";
+  public static final String TIKV_BATCH_SCAN_CONCURRENCY = "tikv.batch_scan_concurrency";
+  public static final String TIKV_DELETE_RANGE_CONCURRENCY = "tikv.delete_range_concurrency";
+
+  public static final String TIKV_REQUEST_COMMAND_PRIORITY = "tikv.request.command.priority";
+  public static final String TIKV_REQUEST_ISOLATION_LEVEL = "tikv.request.isolation.level";
+
+  public static final String TIKV_SHOW_ROWID = "tikv.show_rowid";
+  public static final String TIKV_DB_PREFIX = "tikv.db_prefix";
+  public static final String TIKV_KV_CLIENT_CONCURRENCY = "tikv.kv_client_concurrency";
+
+  public static final String TIKV_KV_MODE = "tikv.kv_mode";
+  public static final String TIKV_REPLICA_READ = "tikv.replica_read";
+
+  public static final String TIKV_METRICS_ENABLE = "tikv.metrics.enable";
+  public static final String TIKV_METRICS_PORT = "tikv.metrics.port";
+
+  public static final String TIKV_NETWORK_MAPPING_NAME = "tikv.network.mapping";
+  public static final String TIKV_ENABLE_GRPC_FORWARD = "tikv.enable_grpc_forward";
+  public static final String TIKV_GRPC_HEALTH_CHECK_TIMEOUT = "tikv.grpc.health_check_timeout";
+  public static final String TIKV_HEALTH_CHECK_PERIOD_DURATION =
+      "tikv.health_check_period_duration";
+
+  public static final String TIKV_ENABLE_ATOMIC_FOR_CAS = "tikv.enable_atomic_for_cas";
+
+  public static final String TIKV_IMPORTER_MAX_KV_BATCH_BYTES = "tikv.importer.max_kv_batch_bytes";
+  public static final String TIKV_IMPORTER_MAX_KV_BATCH_SIZE = "tikv.importer.max_kv_batch_size";
+
+  public static final String TIKV_SCATTER_WAIT_SECONDS = "tikv.scatter_wait_seconds";
+
+  public static final String TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS = "tikv.rawkv.default_backoff_in_ms";
+  public static final String TIKV_RAWKV_READ_TIMEOUT_IN_MS = "tikv.rawkv.read_timeout_in_ms";
+  public static final String TIKV_RAWKV_WRITE_TIMEOUT_IN_MS = "tikv.rawkv.write_timeout_in_ms";
+  public static final String TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS =
+      "tikv.rawkv.batch_read_timeout_in_ms";
+  public static final String TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS =
+      "tikv.rawkv.batch_write_timeout_in_ms";
+  public static final String TIKV_RAWKV_SCAN_TIMEOUT_IN_MS = "tikv.rawkv.scan_timeout_in_ms";
+  public static final String TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS = "tikv.rawkv.clean_timeout_in_ms";
+  public static final String TIKV_BO_REGION_MISS_BASE_IN_MS = "tikv.bo_region_miss_base_in_ms";
+  public static final String TIKV_RAWKV_READ_SLOWLOG_IN_MS = "tikv.rawkv.read_slowlog_in_ms";
+  public static final String TIKV_RAWKV_WRITE_SLOWLOG_IN_MS = "tikv.rawkv.write_slowlog_in_ms";
+  public static final String TIKV_RAWKV_BATCH_READ_SLOWLOG_IN_MS =
+      "tikv.rawkv.batch_read_slowlog_in_ms";
+  public static final String TIKV_RAWKV_BATCH_WRITE_SLOWLOG_IN_MS =
+      "tikv.rawkv.batch_write_slowlog_in_ms";
+  public static final String TIKV_RAWKV_SCAN_SLOWLOG_IN_MS = "tikv.rawkv.scan_slowlog_in_ms";
+  public static final String TIKV_RAWKV_SERVER_SLOWLOG_FACTOR = "tikv.rawkv.server_slowlog_factor";
+
+  public static final String TIKV_TLS_ENABLE = "tikv.tls_enable";
+  public static final String TIKV_TLS_RELOAD_INTERVAL = "tikv.tls.reload_interval";
+  public static final String TIKV_TRUST_CERT_COLLECTION = "tikv.trust_cert_collection";
+  public static final String TIKV_KEY_CERT_CHAIN = "tikv.key_cert_chain";
+  public static final String TIKV_KEY_FILE = "tikv.key_file";
+
+  public static final String TIKV_USE_JKS = "tikv.use_jks";
+  public static final String TIKV_JKS_KEY_PATH = "tikv.jks.key_path";
+  public static final String TIKV_JKS_KEY_PASSWORD = "tikv.jks.key_password";
+  public static final String TIKV_JKS_TRUST_PATH = "tikv.jks.trust_path";
+  public static final String TIKV_JKS_TRUST_PASSWORD = "tikv.jks.trust_password";
+
+  public static final String TiKV_CIRCUIT_BREAK_ENABLE = "tikv.circuit_break.enable";
+  public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS =
+      "tikv.circuit_break.trigger.availability.window_in_seconds";
+  public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE =
+      "tikv.circuit_break.trigger.availability.error_threshold_percentage";
+  public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD =
+      "tikv.circuit_break.trigger.availability.request_volumn_threshold";
+  public static final String TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS =
+      "tikv.circuit_break.trigger.sleep_window_in_seconds";
+  public static final String TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT =
+      "tikv.circuit_break.trigger.attempt_request_count";
+
+  public static final String TIKV_SCAN_REGIONS_LIMIT = "tikv.scan_regions_limit";
+
+  public static final String TIFLASH_ENABLE = "tiflash.enable";
+  public static final String TIKV_WARM_UP_ENABLE = "tikv.warm_up.enable";
+
+  public static final String TIKV_API_VERSION = "tikv.api_version";
+
+  public static final String DEF_PD_ADDRESSES = "127.0.0.1:2379";
+  public static final String DEF_TIMEOUT = "200ms";
+  public static final String DEF_TIKV_GRPC_INGEST_TIMEOUT = "200s";
+  public static final String DEF_FORWARD_TIMEOUT = "300ms";
+  public static final String DEF_TIKV_GRPC_WARM_UP_TIMEOUT = "5000ms";
+  public static final String DEF_TIKV_PD_FIRST_GET_MEMBER_TIMEOUT = "10000ms";
+  public static final String DEF_SCAN_TIMEOUT = "20s";
+  public static final int DEF_CHECK_HEALTH_TIMEOUT = 100;
+  public static final int DEF_HEALTH_CHECK_PERIOD_DURATION = 300;
+  public static final int DEF_SCAN_BATCH_SIZE = 10240;
+  public static final int DEF_MAX_FRAME_SIZE = 268435456 * 2; // 256 * 2 MB
+  public static final String DEF_TIKV_CONN_RECYCLE_TIME = "60s";
+  public static final String DEF_TIKV_TLS_RELOAD_INTERVAL = "10s";
+  public static final int DEF_INDEX_SCAN_BATCH_SIZE = 20000;
+  public static final int DEF_REGION_SCAN_DOWNGRADE_THRESHOLD = 10000000;
+  // if keyRange size per request exceeds this limit, the request might be too large to be accepted
+  // by TiKV(maximum request size accepted by TiKV is around 1MB)
+  public static final int MAX_REQUEST_KEY_RANGE_SIZE = 20000;
+  public static final int DEF_INDEX_SCAN_CONCURRENCY = 5;
+  public static final int DEF_TABLE_SCAN_CONCURRENCY = 512;
+  public static final int DEF_BATCH_GET_CONCURRENCY = 20;
+  public static final int DEF_BATCH_PUT_CONCURRENCY = 20;
+  public static final int DEF_BATCH_DELETE_CONCURRENCY = 20;
+  public static final int DEF_BATCH_SCAN_CONCURRENCY = 5;
+  public static final int DEF_DELETE_RANGE_CONCURRENCY = 20;
+  public static final Kvrpcpb.CommandPri DEF_COMMAND_PRIORITY = Kvrpcpb.CommandPri.Low;
+  public static final Kvrpcpb.IsolationLevel DEF_ISOLATION_LEVEL = Kvrpcpb.IsolationLevel.SI;
+  public static final boolean DEF_SHOW_ROWID = false;
+  public static final String DEF_DB_PREFIX = "";
+  public static final int DEF_KV_CLIENT_CONCURRENCY = 10;
+  public static final TiConfiguration.KVMode DEF_KV_MODE = TiConfiguration.KVMode.TXN;
+  public static final String DEF_REPLICA_READ = "LEADER";
+  public static final boolean DEF_METRICS_ENABLE = false;
+  public static final int DEF_METRICS_PORT = 3140;
+  public static final String DEF_TIKV_NETWORK_MAPPING_NAME = "";
+  public static final boolean DEF_GRPC_FORWARD_ENABLE = true;
+  public static final boolean DEF_TIKV_ENABLE_ATOMIC_FOR_CAS = false;
+
+  public static final int DEF_TIKV_IMPORTER_MAX_KV_BATCH_BYTES = 1024 * 1024;
+  public static final int DEF_TIKV_IMPORTER_MAX_KV_BATCH_SIZE = 1024 * 32;
+  public static final int DEF_TIKV_SCATTER_WAIT_SECONDS = 300;
+  public static final int DEF_TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS = BackOffer.RAWKV_MAX_BACKOFF;
+
+  public static final int DEF_TIKV_RAWKV_READ_TIMEOUT_IN_MS = 2000;
+  public static final int DEF_TIKV_RAWKV_WRITE_TIMEOUT_IN_MS = 2000;
+  public static final int DEF_TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS = 2000;
+  public static final int DEF_TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS = 2000;
+  public static final int DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS = 10000;
+  public static final int DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS = 600000;
+
+  public static final int DEF_TIKV_BO_REGION_MISS_BASE_IN_MS = 20;
+  public static final String DEF_TIKV_RAWKV_SCAN_SLOWLOG_IN_MS = "5000";
+
+  public static final String NORMAL_COMMAND_PRIORITY = "NORMAL";
+  public static final String LOW_COMMAND_PRIORITY = "LOW";
+  public static final String HIGH_COMMAND_PRIORITY = "HIGH";
+
+  public static final String SNAPSHOT_ISOLATION_LEVEL = "SI";
+  public static final String READ_COMMITTED_ISOLATION_LEVEL = "RC";
+
+  public static final String RAW_KV_MODE = "RAW";
+  public static final String TXN_KV_MODE = "TXN";
+
+  public static final String LEADER = "LEADER";
+  public static final String FOLLOWER = "FOLLOWER";
+  public static final String LEADER_AND_FOLLOWER = "LEADER_AND_FOLLOWER";
+
+  public static final int DEF_TIKV_GRPC_KEEPALIVE_TIME = 10;
+  public static final int DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT = 3;
+  public static final int DEF_TIKV_GRPC_IDLE_TIMEOUT = 60;
+  public static final boolean DEF_TIKV_TLS_ENABLE = false;
+  public static final boolean DEF_TIKV_USE_JKS = false;
+  public static final boolean DEF_TIFLASH_ENABLE = false;
+  public static final boolean DEF_TIKV_WARM_UP_ENABLE = true;
+
+  public static final boolean DEF_TiKV_CIRCUIT_BREAK_ENABLE = false;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS = 60;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE = 100;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUST_VOLUMN_THRESHOLD = 10;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS = 20;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT = 10;
+
+  public static final int DEF_TIKV_SCAN_REGIONS_LIMIT = 1000;
+
+  public static final int DEF_TIKV_API_VERSION = 1;
+}

src/main/java/org/tikv/common/DefaultHostMapping.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+import static org.tikv.common.pd.PDUtils.addrToUri;
+
+import com.google.common.annotations.Beta;
+import io.etcd.jetcd.ByteSequence;
+import io.etcd.jetcd.Client;
+import io.etcd.jetcd.KeyValue;
+import io.etcd.jetcd.kv.GetResponse;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultHostMapping implements HostMapping {
+  private static final String NETWORK_MAPPING_PATH = "/client/url-mapping";
+  private final Client etcdClient;
+  private final String networkMappingName;
+  private final ConcurrentMap<String, String> hostMapping;
+  private final Logger logger = LoggerFactory.getLogger(DefaultHostMapping.class);
+
+  public DefaultHostMapping(Client etcdClient, String networkMappingName) {
+    this.etcdClient = etcdClient;
+    this.networkMappingName = networkMappingName;
+    this.hostMapping = new ConcurrentHashMap<>();
+  }
+
+  private ByteSequence hostToNetworkMappingKey(String host) {
+    String path = NETWORK_MAPPING_PATH + "/" + networkMappingName + "/" + host;
+    return ByteSequence.from(path, StandardCharsets.UTF_8);
+  }
+
+  @Beta
+  private String getMappedHostFromPD(String host) {
+    ByteSequence hostKey = hostToNetworkMappingKey(host);
+    for (int i = 0; i < 5; i++) {
+      CompletableFuture<GetResponse> future = etcdClient.getKVClient().get(hostKey);
+      try {
+        GetResponse resp = future.get();
+        List<KeyValue> kvs = resp.getKvs();
+        if (kvs.size() != 1) {
+          break;
+        }
+        return kvs.get(0).getValue().toString(StandardCharsets.UTF_8);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      } catch (ExecutionException e) {
+        logger.info("failed to get mapped Host from PD: " + host, e);
+        break;
+      } catch (Exception ignore) {
+        // ignore
+        break;
+      }
+    }
+    return host;
+  }
+
+  public URI getMappedURI(URI uri) {
+    if (networkMappingName.isEmpty()) {
+      return uri;
+    }
+    return addrToUri(
+        hostMapping.computeIfAbsent(uri.getHost(), this::getMappedHostFromPD)
+            + ":"
+            + uri.getPort());
+  }
+}

src/main/java/org/tikv/common/ExtendedDateTime.java

@@ -1,15 +1,15 @@
 /*
- *
+ * Copyright 2021 TiKV Project Authors.
- * Copyright 2019 PingCAP, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *

src/main/java/org/tikv/common/HostMapping.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+import java.io.Serializable;
+import java.net.URI;
+
+public interface HostMapping extends Serializable {
+  URI getMappedURI(URI uri);
+}

src/main/java/org/tikv/common/KVClient.java

@@ -1,15 +1,15 @@
 /*
- *
+ * Copyright 2021 TiKV Project Authors.
- * Copyright 2019 PingCAP, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
@@ -17,18 +17,15 @@
 
 package org.tikv.common;
 
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import static org.tikv.common.util.ClientUtils.*;
+
 import com.google.protobuf.ByteString;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.tikv.common.exception.GrpcException;
@@ -39,33 +36,27 @@ import org.tikv.common.region.RegionStoreClient.RegionStoreClientBuilder;
 import org.tikv.common.region.TiRegion;
 import org.tikv.common.util.BackOffFunction;
 import org.tikv.common.util.BackOffer;
+import org.tikv.common.util.Batch;
 import org.tikv.common.util.ConcreteBackOffer;
 import org.tikv.kvproto.Kvrpcpb.KvPair;
 
 public class KVClient implements AutoCloseable {
   private static final Logger logger = LoggerFactory.getLogger(KVClient.class);
+  private static final int MAX_BATCH_LIMIT = 1024;
   private static final int BATCH_GET_SIZE = 16 * 1024;
   private final RegionStoreClientBuilder clientBuilder;
   private final TiConfiguration conf;
-  private final ExecutorService executorService;
+  private final ExecutorService batchGetThreadPool;
 
-  public KVClient(TiConfiguration conf, RegionStoreClientBuilder clientBuilder) {
+  public KVClient(TiSession session, RegionStoreClientBuilder clientBuilder) {
-    Objects.requireNonNull(conf, "conf is null");
     Objects.requireNonNull(clientBuilder, "clientBuilder is null");
-    this.conf = conf;
+    this.conf = session.getConf();
     this.clientBuilder = clientBuilder;
-    executorService =
+    this.batchGetThreadPool = session.getThreadPoolForBatchGet();
-        Executors.newFixedThreadPool(
-            conf.getKvClientConcurrency(),
-            new ThreadFactoryBuilder().setNameFormat("kvclient-pool-%d").setDaemon(true).build());
   }
 
   @Override
-  public void close() {
+  public void close() {}
-    if (executorService != null) {
-      executorService.shutdownNow();
-    }
-  }
 
   /**
    * Get a key-value pair from TiKV if key exists
@@ -74,7 +65,9 @@ public class KVClient implements AutoCloseable {
    * @return a ByteString value if key exists, ByteString.EMPTY if key does not exist
    */
   public ByteString get(ByteString key, long version) throws GrpcException {
-    BackOffer backOffer = ConcreteBackOffer.newGetBackOff();
+    BackOffer backOffer =
+        ConcreteBackOffer.newGetBackOff(
+            clientBuilder.getRegionManager().getPDClient().getClusterId());
     while (true) {
       RegionStoreClient client = clientBuilder.build(key);
       try {
@@ -134,48 +127,32 @@ public class KVClient implements AutoCloseable {
 
   private List<KvPair> doSendBatchGet(BackOffer backOffer, List<ByteString> keys, long version) {
     ExecutorCompletionService<List<KvPair>> completionService =
-        new ExecutorCompletionService<>(executorService);
+        new ExecutorCompletionService<>(batchGetThreadPool);
 
-    Map<TiRegion, List<ByteString>> groupKeys = groupKeysByRegion(keys);
+    List<Batch> batches =
-    List<Batch> batches = new ArrayList<>();
+        getBatches(backOffer, keys, BATCH_GET_SIZE, MAX_BATCH_LIMIT, this.clientBuilder);
-
-    for (Map.Entry<TiRegion, List<ByteString>> entry : groupKeys.entrySet()) {
-      appendBatches(batches, entry.getKey(), entry.getValue(), BATCH_GET_SIZE);
-    }
 
     for (Batch batch : batches) {
-      BackOffer singleBatchBackOffer = ConcreteBackOffer.create(backOffer);
       completionService.submit(
-          () -> doSendBatchGetInBatchesWithRetry(singleBatchBackOffer, batch, version));
+          () -> doSendBatchGetInBatchesWithRetry(batch.getBackOffer(), batch, version));
     }
 
-    try {
+    return getKvPairs(completionService, batches, BackOffer.BATCH_GET_MAX_BACKOFF);
-      List<KvPair> result = new ArrayList<>();
-      for (int i = 0; i < batches.size(); i++) {
-        result.addAll(completionService.take().get());
-      }
-      return result;
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      throw new TiKVException("Current thread interrupted.", e);
-    } catch (ExecutionException e) {
-      throw new TiKVException("Execution exception met.", e);
-    }
   }
 
   private List<KvPair> doSendBatchGetInBatchesWithRetry(
       BackOffer backOffer, Batch batch, long version) {
-    TiRegion oldRegion = batch.region;
+    TiRegion oldRegion = batch.getRegion();
     TiRegion currentRegion =
         clientBuilder.getRegionManager().getRegionByKey(oldRegion.getStartKey());
 
     if (oldRegion.equals(currentRegion)) {
-      RegionStoreClient client = clientBuilder.build(batch.region);
+      RegionStoreClient client = clientBuilder.build(batch.getRegion());
       try {
-        return client.batchGet(backOffer, batch.keys, version);
+        return client.batchGet(backOffer, batch.getKeys(), version);
       } catch (final TiKVException e) {
         backOffer.doBackOff(BackOffFunction.BackOffFuncType.BoRegionMiss, e);
-        clientBuilder.getRegionManager().invalidateRegion(batch.region.getId());
+        clientBuilder.getRegionManager().invalidateRegion(batch.getRegion());
         logger.warn("ReSplitting ranges for BatchGetRequest", e);
 
         // retry
@@ -188,59 +165,19 @@ public class KVClient implements AutoCloseable {
 
   private List<KvPair> doSendBatchGetWithRefetchRegion(
       BackOffer backOffer, Batch batch, long version) {
-    Map<TiRegion, List<ByteString>> groupKeys = groupKeysByRegion(batch.keys);
+    List<Batch> retryBatches =
-    List<Batch> retryBatches = new ArrayList<>();
+        getBatches(backOffer, batch.getKeys(), BATCH_GET_SIZE, MAX_BATCH_LIMIT, this.clientBuilder);
-
-    for (Map.Entry<TiRegion, List<ByteString>> entry : groupKeys.entrySet()) {
-      appendBatches(retryBatches, entry.getKey(), entry.getValue(), BATCH_GET_SIZE);
-    }
 
     ArrayList<KvPair> results = new ArrayList<>();
     for (Batch retryBatch : retryBatches) {
       // recursive calls
-      List<KvPair> batchResult = doSendBatchGetInBatchesWithRetry(backOffer, retryBatch, version);
+      List<KvPair> batchResult =
+          doSendBatchGetInBatchesWithRetry(retryBatch.getBackOffer(), retryBatch, version);
       results.addAll(batchResult);
     }
     return results;
   }
 
-  /**
-   * Append batch to list and split them according to batch limit
-   *
-   * @param batches a grouped batch
-   * @param region region
-   * @param keys keys
-   * @param batchGetMaxSizeInByte batch max limit
-   */
-  private void appendBatches(
-      List<Batch> batches, TiRegion region, List<ByteString> keys, int batchGetMaxSizeInByte) {
-    int start;
-    int end;
-    if (keys == null) {
-      return;
-    }
-    int len = keys.size();
-    for (start = 0; start < len; start = end) {
-      int size = 0;
-      for (end = start; end < len && size < batchGetMaxSizeInByte; end++) {
-        size += keys.get(end).size();
-      }
-      Batch batch = new Batch(region, keys.subList(start, end));
-      batches.add(batch);
-    }
-  }
-
-  /**
-   * Group by list of keys according to its region
-   *
-   * @param keys keys
-   * @return a mapping of keys and their region
-   */
-  private Map<TiRegion, List<ByteString>> groupKeysByRegion(List<ByteString> keys) {
-    return keys.stream()
-        .collect(Collectors.groupingBy(clientBuilder.getRegionManager()::getRegionByKey));
-  }
-
   private Iterator<KvPair> scanIterator(
       TiConfiguration conf,
       RegionStoreClientBuilder builder,
@@ -258,15 +195,4 @@ public class KVClient implements AutoCloseable {
       int limit) {
     return new ConcreteScanIterator(conf, builder, startKey, version, limit);
   }
-
-  /** A Batch containing the region and a list of keys to send */
-  private static final class Batch {
-    private final TiRegion region;
-    private final List<ByteString> keys;
-
-    Batch(TiRegion region, List<ByteString> keys) {
-      this.region = region;
-      this.keys = keys;
-    }
-  }
 }

src/main/java/org/tikv/common/Main.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/MetricsServer.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+import io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.client.hotspot.DefaultExports;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MetricsServer {
+  private static final Logger logger = LoggerFactory.getLogger(MetricsServer.class);
+
+  private static MetricsServer METRICS_SERVER_INSTANCE = null;
+  private static int metricsServerRefCount = 0;
+
+  private final int port;
+  private final HTTPServer server;
+
+  public static MetricsServer getInstance(TiConfiguration conf) {
+    if (!conf.isMetricsEnable()) {
+      return null;
+    }
+
+    synchronized (MetricsServer.class) {
+      int port = conf.getMetricsPort();
+      if (METRICS_SERVER_INSTANCE != null) {
+        if (port != METRICS_SERVER_INSTANCE.port) {
+          throw new IllegalArgumentException(
+              String.format(
+                  "Do dot support multiple tikv.metrics.port, which are %d and %d",
+                  port, METRICS_SERVER_INSTANCE.port));
+        }
+      } else {
+        METRICS_SERVER_INSTANCE = new MetricsServer(port);
+      }
+      metricsServerRefCount += 1;
+      return METRICS_SERVER_INSTANCE;
+    }
+  }
+
+  private MetricsServer(int port) {
+    try {
+      this.port = port;
+      DefaultExports.initialize();
+      this.server = new HTTPServer(port, true);
+      logger.info("http server is up " + this.server.getPort());
+    } catch (Exception e) {
+      logger.error("http server not up");
+      throw new RuntimeException(e);
+    }
+  }
+
+  public void close() {
+    synchronized (MetricsServer.class) {
+      if (metricsServerRefCount == 1) {
+        if (server != null) {
+          server.stop();
+          logger.info("Metrics server on " + server.getPort() + " is stopped");
+        }
+        METRICS_SERVER_INSTANCE = null;
+      }
+
+      if (metricsServerRefCount >= 1) {
+        metricsServerRefCount -= 1;
+      }
+    }
+  }
+}

src/main/java/org/tikv/common/PDChecker.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+public enum PDChecker {
+  Learner,
+  Replica,
+  Rule,
+  Split,
+  Merge,
+  JointState,
+  Priority;
+
+  public String apiName() {
+    switch (this) {
+      case Learner:
+        return "learner";
+      case Replica:
+        return "replica";
+      case Rule:
+        return "rule";
+      case Split:
+        return "split";
+      case Merge:
+        return "merge";
+      case JointState:
+        return "joint-state";
+      case Priority:
+        return "priority";
+    }
+    throw new IllegalArgumentException();
+  }
+}

src/main/java/org/tikv/common/PDClient.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -18,7 +20,12 @@ package org.tikv.common;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.tikv.common.operation.PDErrorHandler.getRegionResponseErrorExtractor;
 import static org.tikv.common.pd.PDError.buildFromPdpbError;
+import static org.tikv.common.pd.PDUtils.addrToUri;
+import static org.tikv.common.pd.PDUtils.uriToAddr;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.protobuf.ByteString;
@@ -28,39 +35,55 @@ import io.etcd.jetcd.KeyValue;
 import io.etcd.jetcd.kv.GetResponse;
 import io.etcd.jetcd.options.GetOption;
 import io.grpc.ManagedChannel;
+import io.grpc.Metadata;
+import io.grpc.stub.MetadataUtils;
+import io.prometheus.client.Histogram;
 import java.net.URI;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.tikv.common.codec.Codec.BytesCodec;
+import org.tikv.common.apiversion.RequestKeyCodec;
-import org.tikv.common.codec.CodecDataOutput;
 import org.tikv.common.codec.KeyUtils;
 import org.tikv.common.exception.GrpcException;
 import org.tikv.common.exception.TiClientInternalException;
 import org.tikv.common.meta.TiTimestamp;
 import org.tikv.common.operation.NoopHandler;
 import org.tikv.common.operation.PDErrorHandler;
-import org.tikv.common.pd.PDUtils;
-import org.tikv.common.region.TiRegion;
 import org.tikv.common.util.BackOffFunction.BackOffFuncType;
 import org.tikv.common.util.BackOffer;
 import org.tikv.common.util.ChannelFactory;
 import org.tikv.common.util.ConcreteBackOffer;
-import org.tikv.common.util.FutureObserver;
+import org.tikv.common.util.HistogramUtils;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Metapb;
 import org.tikv.kvproto.Metapb.Store;
 import org.tikv.kvproto.PDGrpc;
 import org.tikv.kvproto.PDGrpc.PDBlockingStub;
-import org.tikv.kvproto.PDGrpc.PDStub;
+import org.tikv.kvproto.PDGrpc.PDFutureStub;
+import org.tikv.kvproto.Pdpb;
 import org.tikv.kvproto.Pdpb.Error;
 import org.tikv.kvproto.Pdpb.ErrorType;
 import org.tikv.kvproto.Pdpb.GetAllStoresRequest;
@@ -81,33 +104,59 @@ import org.tikv.kvproto.Pdpb.ScatterRegionResponse;
 import org.tikv.kvproto.Pdpb.Timestamp;
 import org.tikv.kvproto.Pdpb.TsoRequest;
 import org.tikv.kvproto.Pdpb.TsoResponse;
+import org.tikv.kvproto.Pdpb.UpdateServiceGCSafePointRequest;
 
-public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
+public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
     implements ReadOnlyPDClient {
   private static final String TIFLASH_TABLE_SYNC_PROGRESS_PATH = "/tiflash/table/sync";
-  private final Logger logger = LoggerFactory.getLogger(PDClient.class);
+  private static final long MIN_TRY_UPDATE_DURATION = 50;
+  private static final int PAUSE_CHECKER_TIMEOUT = 300; // in seconds
+  private static final int KEEP_CHECKER_PAUSE_PERIOD = PAUSE_CHECKER_TIMEOUT / 5; // in seconds
+  private static final Logger logger = LoggerFactory.getLogger(PDClient.class);
+  private static final ObjectMapper mapper = new ObjectMapper();
+
+  private final RequestKeyCodec codec;
   private RequestHeader header;
   private TsoRequest tsoReq;
-  private volatile LeaderWrapper leaderWrapper;
+  private volatile PDClientWrapper pdClientWrapper;
   private ScheduledExecutorService service;
   private ScheduledExecutorService tiflashReplicaService;
+  private final HashMap<PDChecker, ScheduledExecutorService> pauseCheckerService = new HashMap<>();
   private List<URI> pdAddrs;
   private Client etcdClient;
   private ConcurrentMap<Long, Double> tiflashReplicaMap;
+  private HostMapping hostMapping;
+  private long lastUpdateLeaderTime;
+  private final ExecutorService updateLeaderService = Executors.newSingleThreadExecutor();
+  private final AtomicBoolean updateLeaderNotify = new AtomicBoolean();
 
-  private PDClient(TiConfiguration conf, ChannelFactory channelFactory) {
+  public static final Histogram PD_GET_REGION_BY_KEY_REQUEST_LATENCY =
+      HistogramUtils.buildDuration()
+          .name("client_java_pd_get_region_by_requests_latency")
+          .help("pd getRegionByKey request latency.")
+          .labelNames("cluster")
+          .register();
+
+  private PDClient(TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
     super(conf, channelFactory);
     initCluster();
+    this.codec = codec;
     this.blockingStub = getBlockingStub();
     this.asyncStub = getAsyncStub();
   }
 
-  public static ReadOnlyPDClient create(TiConfiguration conf, ChannelFactory channelFactory) {
+  public static ReadOnlyPDClient create(
-    return createRaw(conf, channelFactory);
+      TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
+    return createRaw(conf, codec, channelFactory);
   }
 
-  static PDClient createRaw(TiConfiguration conf, ChannelFactory channelFactory) {
+  static PDClient createRaw(
-    return new PDClient(conf, channelFactory);
+      TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
+    return new PDClient(conf, codec, channelFactory);
+  }
+
+  public HostMapping getHostMapping() {
+    return hostMapping;
   }
 
   @Override
@@ -124,12 +173,75 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
     return new TiTimestamp(timestamp.getPhysical(), timestamp.getLogical());
   }
 
+  public synchronized void keepPauseChecker(PDChecker checker) {
+    if (!this.pauseCheckerService.containsKey(checker)) {
+      ScheduledExecutorService newService =
+          Executors.newSingleThreadScheduledExecutor(
+              new ThreadFactoryBuilder()
+                  .setNameFormat(String.format("PDClient-pause-%s-pool-%%d", checker.name()))
+                  .setDaemon(true)
+                  .build());
+      newService.scheduleAtFixedRate(
+          () -> pauseChecker(checker, PAUSE_CHECKER_TIMEOUT),
+          0,
+          KEEP_CHECKER_PAUSE_PERIOD,
+          TimeUnit.SECONDS);
+      this.pauseCheckerService.put(checker, newService);
+    }
+  }
+
+  public synchronized void stopKeepPauseChecker(PDChecker checker) {
+    if (this.pauseCheckerService.containsKey(checker)) {
+      this.pauseCheckerService.get(checker).shutdown();
+      this.pauseCheckerService.remove(checker);
+    }
+  }
+
+  public void resumeChecker(PDChecker checker) {
+    pauseChecker(checker, 0);
+  }
+
+  private void pauseChecker(PDChecker checker, int timeout) {
+    String verb = timeout == 0 ? "resume" : "pause";
+    URI url = pdAddrs.get(0);
+    String api = url.toString() + "/pd/api/v1/checker/" + checker.apiName();
+    HashMap<String, Integer> arguments = new HashMap<>();
+    arguments.put("delay", timeout);
+    try (CloseableHttpClient client = HttpClients.createDefault()) {
+      JsonMapper jsonMapper = new JsonMapper();
+      byte[] body = jsonMapper.writeValueAsBytes(arguments);
+      HttpPost post = new HttpPost(api);
+      post.setEntity(new ByteArrayEntity(body));
+      try (CloseableHttpResponse resp = client.execute(post)) {
+        if (resp.getStatusLine().getStatusCode() != 200) {
+          logger.error("failed to {} checker.", verb);
+        }
+        logger.info("checker {} {}d", checker.apiName(), verb);
+      }
+    } catch (Exception e) {
+      logger.error(String.format("failed to %s checker.", verb), e);
+    }
+  }
+
+  public Boolean isCheckerPaused(PDChecker checker) {
+    URI url = pdAddrs.get(0);
+    String api = url.toString() + "/pd/api/v1/checker/" + checker.apiName();
+    try {
+      HashMap<String, Boolean> status =
+          mapper.readValue(new URL(api), new TypeReference<HashMap<String, Boolean>>() {});
+      return status.get("paused");
+    } catch (Exception e) {
+      logger.error(String.format("failed to get %s checker status.", checker.apiName()), e);
+      return null;
+    }
+  }
+
   /**
    * Sends request to pd to scatter region.
    *
    * @param region represents a region info
    */
-  void scatterRegion(TiRegion region, BackOffer backOffer) {
+  void scatterRegion(Metapb.Region region, BackOffer backOffer) {
     Supplier<ScatterRegionRequest> request =
         () ->
             ScatterRegionRequest.newBuilder().setHeader(header).setRegionId(region.getId()).build();
@@ -153,7 +265,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
    *
    * @param region
    */
-  void waitScatterRegionFinish(TiRegion region, BackOffer backOffer) {
+  void waitScatterRegionFinish(Metapb.Region region, BackOffer backOffer) {
     for (; ; ) {
       GetOperatorResponse resp = getOperator(region.getId());
       if (resp != null) {
@@ -168,7 +280,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
                   "wait scatter region %d at key %s is %s",
                   region.getId(),
                   KeyUtils.formatBytes(resp.getDesc().toByteArray()),
-                  resp.getStatus().toString()));
+                  resp.getStatus()));
         }
       }
     }
@@ -179,7 +291,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
         () -> GetOperatorRequest.newBuilder().setHeader(header).setRegionId(regionId).build();
     // get operator no need to handle error and no need back offer.
     return callWithRetry(
-        ConcreteBackOffer.newCustomBackOff(0),
+        ConcreteBackOffer.newCustomBackOff(0, getClusterId()),
         PDGrpc.getGetOperatorMethod(),
         request,
         new NoopHandler<>());
@@ -206,52 +318,30 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
   }
 
   @Override
-  public TiRegion getRegionByKey(BackOffer backOffer, ByteString key) {
+  public Pair<Metapb.Region, Metapb.Peer> getRegionByKey(BackOffer backOffer, ByteString key) {
-    CodecDataOutput cdo = new CodecDataOutput();
+    Histogram.Timer requestTimer =
-    BytesCodec.writeBytes(cdo, key.toByteArray());
+        PD_GET_REGION_BY_KEY_REQUEST_LATENCY.labels(getClusterId().toString()).startTimer();
-    ByteString encodedKey = cdo.toByteString();
+    try {
+      Supplier<GetRegionRequest> request =
+          () ->
+              GetRegionRequest.newBuilder()
+                  .setHeader(header)
+                  .setRegionKey(codec.encodePdQuery(key))
+                  .build();
 
-    Supplier<GetRegionRequest> request =
+      PDErrorHandler<GetRegionResponse> handler =
-        () -> GetRegionRequest.newBuilder().setHeader(header).setRegionKey(encodedKey).build();
+          new PDErrorHandler<>(getRegionResponseErrorExtractor, this);
 
-    PDErrorHandler<GetRegionResponse> handler =
+      GetRegionResponse resp =
-        new PDErrorHandler<>(getRegionResponseErrorExtractor, this);
+          callWithRetry(backOffer, PDGrpc.getGetRegionMethod(), request, handler);
-
+      return new Pair<>(codec.decodeRegion(resp.getRegion()), resp.getLeader());
-    GetRegionResponse resp =
+    } finally {
-        callWithRetry(backOffer, PDGrpc.getGetRegionMethod(), request, handler);
+      requestTimer.observeDuration();
-    return new TiRegion(
+    }
-        resp.getRegion(),
-        resp.getLeader(),
-        conf.getIsolationLevel(),
-        conf.getCommandPriority(),
-        conf.getKvMode(),
-        conf.isReplicaRead());
   }
 
   @Override
-  public Future<TiRegion> getRegionByKeyAsync(BackOffer backOffer, ByteString key) {
+  public Pair<Metapb.Region, Metapb.Peer> getRegionByID(BackOffer backOffer, long id) {
-    FutureObserver<TiRegion, GetRegionResponse> responseObserver =
-        new FutureObserver<>(
-            resp ->
-                new TiRegion(
-                    resp.getRegion(),
-                    resp.getLeader(),
-                    conf.getIsolationLevel(),
-                    conf.getCommandPriority(),
-                    conf.getKvMode(),
-                    conf.isReplicaRead()));
-    Supplier<GetRegionRequest> request =
-        () -> GetRegionRequest.newBuilder().setHeader(header).setRegionKey(key).build();
-
-    PDErrorHandler<GetRegionResponse> handler =
-        new PDErrorHandler<>(getRegionResponseErrorExtractor, this);
-
-    callAsyncWithRetry(backOffer, PDGrpc.getGetRegionMethod(), request, responseObserver, handler);
-    return responseObserver.getFuture();
-  }
-
-  @Override
-  public TiRegion getRegionByID(BackOffer backOffer, long id) {
     Supplier<GetRegionByIDRequest> request =
         () -> GetRegionByIDRequest.newBuilder().setHeader(header).setRegionId(id).build();
     PDErrorHandler<GetRegionResponse> handler =
@@ -259,37 +349,31 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
 
     GetRegionResponse resp =
         callWithRetry(backOffer, PDGrpc.getGetRegionByIDMethod(), request, handler);
-    // Instead of using default leader instance, explicitly set no leader to null
+    return new Pair<Metapb.Region, Metapb.Peer>(
-    return new TiRegion(
+        codec.decodeRegion(resp.getRegion()), resp.getLeader());
-        resp.getRegion(),
-        resp.getLeader(),
-        conf.getIsolationLevel(),
-        conf.getCommandPriority(),
-        conf.getKvMode(),
-        conf.isReplicaRead());
   }
 
   @Override
-  public Future<TiRegion> getRegionByIDAsync(BackOffer backOffer, long id) {
+  public List<Pdpb.Region> scanRegions(
-    FutureObserver<TiRegion, GetRegionResponse> responseObserver =
+      BackOffer backOffer, ByteString startKey, ByteString endKey, int limit) {
-        new FutureObserver<>(
+    // no need to backoff because ScanRegions is just for optimization
-            resp ->
+    // introduce a warm-up timeout for ScanRegions requests
-                new TiRegion(
+    PDGrpc.PDBlockingStub stub =
-                    resp.getRegion(),
+        getBlockingStub().withDeadlineAfter(conf.getWarmUpTimeout(), TimeUnit.MILLISECONDS);
-                    resp.getLeader(),
+    Pair<ByteString, ByteString> range = codec.encodePdQueryRange(startKey, endKey);
-                    conf.getIsolationLevel(),
+    Pdpb.ScanRegionsRequest request =
-                    conf.getCommandPriority(),
+        Pdpb.ScanRegionsRequest.newBuilder()
-                    conf.getKvMode(),
+            .setHeader(header)
-                    conf.isReplicaRead()));
+            .setStartKey(range.first)
+            .setEndKey(range.second)
+            .setLimit(limit)
+            .build();
+    Pdpb.ScanRegionsResponse resp = stub.scanRegions(request);
+    if (resp == null) {
+      return null;
+    }
 
-    Supplier<GetRegionByIDRequest> request =
+    return codec.decodePdRegions(resp.getRegionsList());
-        () -> GetRegionByIDRequest.newBuilder().setHeader(header).setRegionId(id).build();
-    PDErrorHandler<GetRegionResponse> handler =
-        new PDErrorHandler<>(getRegionResponseErrorExtractor, this);
-
-    callAsyncWithRetry(
-        backOffer, PDGrpc.getGetRegionByIDMethod(), request, responseObserver, handler);
-    return responseObserver.getFuture();
   }
 
   private Supplier<GetStoreRequest> buildGetStoreReq(long storeId) {
@@ -300,6 +384,17 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
     return () -> GetAllStoresRequest.newBuilder().setHeader(header).build();
   }
 
+  private Supplier<UpdateServiceGCSafePointRequest> buildUpdateServiceGCSafePointRequest(
+      ByteString serviceId, long ttl, long safePoint) {
+    return () ->
+        UpdateServiceGCSafePointRequest.newBuilder()
+            .setHeader(header)
+            .setSafePoint(safePoint)
+            .setServiceId(serviceId)
+            .setTTL(ttl)
+            .build();
+  }
+
   private <T> PDErrorHandler<GetStoreResponse> buildPDErrorHandler() {
     return new PDErrorHandler<>(
         r -> r.getHeader().hasError() ? buildFromPdpbError(r.getHeader().getError()) : null, this);
@@ -307,23 +402,16 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
 
   @Override
   public Store getStore(BackOffer backOffer, long storeId) {
-    return callWithRetry(
+    GetStoreResponse resp =
-            backOffer, PDGrpc.getGetStoreMethod(), buildGetStoreReq(storeId), buildPDErrorHandler())
+        callWithRetry(
-        .getStore();
+            backOffer,
-  }
+            PDGrpc.getGetStoreMethod(),
-
+            buildGetStoreReq(storeId),
-  @Override
+            buildPDErrorHandler());
-  public Future<Store> getStoreAsync(BackOffer backOffer, long storeId) {
+    if (resp != null) {
-    FutureObserver<Store, GetStoreResponse> responseObserver =
+      return resp.getStore();
-        new FutureObserver<>(GetStoreResponse::getStore);
+    }
-
+    return null;
-    callAsyncWithRetry(
-        backOffer,
-        PDGrpc.getGetStoreMethod(),
-        buildGetStoreReq(storeId),
-        responseObserver,
-        buildPDErrorHandler());
-    return responseObserver.getFuture();
   }
 
   @Override
@@ -339,8 +427,22 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
   }
 
   @Override
-  public boolean isReplicaRead() {
+  public TiConfiguration.ReplicaRead getReplicaRead() {
-    return conf.isReplicaRead();
+    return conf.getReplicaRead();
+  }
+
+  @Override
+  public Long updateServiceGCSafePoint(
+      String serviceId, long ttl, long safePoint, BackOffer backOffer) {
+    return callWithRetry(
+            backOffer,
+            PDGrpc.getUpdateServiceGCSafePointMethod(),
+            buildUpdateServiceGCSafePointRequest(
+                ByteString.copyFromUtf8(serviceId), ttl, safePoint),
+            new PDErrorHandler<>(
+                r -> r.getHeader().hasError() ? buildFromPdpbError(r.getHeader().getError()) : null,
+                this))
+        .getMinSafePoint();
   }
 
   @Override
@@ -355,6 +457,8 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
     if (channelFactory != null) {
       channelFactory.close();
     }
+
+    updateLeaderService.shutdownNow();
   }
 
   @VisibleForTesting
@@ -363,77 +467,210 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
   }
 
   @VisibleForTesting
-  LeaderWrapper getLeaderWrapper() {
+  PDClientWrapper getPdClientWrapper() {
-    return leaderWrapper;
+    return pdClientWrapper;
   }
 
-  private GetMembersResponse getMembers(URI url) {
+  private GetMembersResponse doGetMembers(BackOffer backOffer, URI uri) {
-    try {
+    while (true) {
-      ManagedChannel probChan = channelFactory.getChannel(url.getHost() + ":" + url.getPort());
+      backOffer.checkTimeout();
-      PDGrpc.PDBlockingStub stub = PDGrpc.newBlockingStub(probChan);
+
-      GetMembersRequest request =
+      try {
-          GetMembersRequest.newBuilder().setHeader(RequestHeader.getDefaultInstance()).build();
+        ManagedChannel probChan = channelFactory.getChannel(uriToAddr(uri), hostMapping);
-      GetMembersResponse resp = stub.getMembers(request);
+        PDGrpc.PDBlockingStub stub =
-      // check if the response contains a valid leader
+            PDGrpc.newBlockingStub(probChan).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
-      if (resp != null && resp.getLeader().getMemberId() == 0) {
+        GetMembersRequest request =
-        return null;
+            GetMembersRequest.newBuilder().setHeader(RequestHeader.getDefaultInstance()).build();
+        GetMembersResponse resp = stub.getMembers(request);
+        // check if the response contains a valid leader
+        if (resp != null && resp.getLeader().getMemberId() == 0) {
+          return null;
+        }
+        return resp;
+      } catch (Exception e) {
+        logger.warn(
+            "failed to get member from pd server from {}, caused by: {}", uri, e.getMessage());
+        backOffer.doBackOff(BackOffFuncType.BoPDRPC, e);
       }
-      return resp;
-    } catch (Exception e) {
-      logger.warn("failed to get member from pd server.", e);
     }
-    return null;
   }
 
-  synchronized boolean switchLeader(List<String> leaderURLs) {
+  private GetMembersResponse getMembers(BackOffer backOffer, URI uri) {
-    if (leaderURLs.isEmpty()) return false;
+    try {
-    String leaderUrlStr = leaderURLs.get(0);
+      return doGetMembers(backOffer, uri);
-    // TODO: Why not strip protocol info on server side since grpc does not need it
+    } catch (Exception e) {
-    if (leaderWrapper != null && leaderUrlStr.equals(leaderWrapper.getLeaderInfo())) {
+      return null;
-      return true;
+    }
+  }
+
+  // return whether the leader has changed to target address `leaderUrlStr`.
+  synchronized boolean trySwitchLeader(String leaderUrlStr) {
+    if (pdClientWrapper != null) {
+      if (leaderUrlStr.equals(pdClientWrapper.getLeaderInfo())) {
+        // The message to leader is not forwarded by follower.
+        if (leaderUrlStr.equals(pdClientWrapper.getStoreAddress())) {
+          return true;
+        }
+      }
+      // If leader has transferred to another member, we can create another leaderWrapper.
     }
     // switch leader
-    return createLeaderWrapper(leaderUrlStr);
+    return createLeaderClientWrapper(leaderUrlStr);
   }
 
-  private boolean createLeaderWrapper(String leaderUrlStr) {
+  private synchronized boolean createLeaderClientWrapper(String leaderUrlStr) {
     try {
-      URI newLeader = PDUtils.addrToUrl(leaderUrlStr);
-      leaderUrlStr = newLeader.getHost() + ":" + newLeader.getPort();
-      if (leaderWrapper != null && leaderUrlStr.equals(leaderWrapper.getLeaderInfo())) {
-        return true;
-      }
-
       // create new Leader
-      ManagedChannel clientChannel = channelFactory.getChannel(leaderUrlStr);
+      ManagedChannel clientChannel = channelFactory.getChannel(leaderUrlStr, hostMapping);
-      leaderWrapper =
+      pdClientWrapper =
-          new LeaderWrapper(
+          new PDClientWrapper(leaderUrlStr, leaderUrlStr, clientChannel, System.nanoTime());
-              leaderUrlStr,
+      timeout = conf.getTimeout();
-              PDGrpc.newBlockingStub(clientChannel),
-              PDGrpc.newStub(clientChannel),
-              System.nanoTime());
     } catch (IllegalArgumentException e) {
-      logger.error("Error updating leader. " + leaderUrlStr, e);
       return false;
     }
-    logger.info(String.format("Switched to new leader: %s", leaderWrapper));
+    logger.info(String.format("Switched to new leader: %s", pdClientWrapper));
     return true;
   }
 
-  public void updateLeader() {
+  synchronized boolean createFollowerClientWrapper(
+      BackOffer backOffer, String followerUrlStr, String leaderUrls) {
+    // TODO: Why not strip protocol info on server side since grpc does not need it
+
+    try {
+      if (!checkHealth(backOffer, followerUrlStr, hostMapping)) {
+        return false;
+      }
+
+      // create new Leader
+      ManagedChannel channel = channelFactory.getChannel(followerUrlStr, hostMapping);
+      pdClientWrapper = new PDClientWrapper(leaderUrls, followerUrlStr, channel, System.nanoTime());
+      timeout = conf.getForwardTimeout();
+    } catch (IllegalArgumentException e) {
+      return false;
+    }
+    logger.info(String.format("Switched to new leader by follower forward: %s", pdClientWrapper));
+    return true;
+  }
+
+  public void tryUpdateLeaderOrForwardFollower() {
+    if (updateLeaderNotify.compareAndSet(false, true)) {
+      try {
+        updateLeaderService.submit(
+            () -> {
+              try {
+                updateLeaderOrForwardFollower();
+              } catch (Exception e) {
+                logger.info("update leader or forward follower failed", e);
+                throw e;
+              } finally {
+                updateLeaderNotify.set(false);
+                logger.info("updating leader finish");
+              }
+            });
+      } catch (RejectedExecutionException e) {
+        logger.error("PDClient is shutdown", e);
+        updateLeaderNotify.set(false);
+      }
+    }
+  }
+
+  private synchronized void updateLeaderOrForwardFollower() {
+    logger.warn("updating leader or forward follower");
+    if (System.currentTimeMillis() - lastUpdateLeaderTime < MIN_TRY_UPDATE_DURATION) {
+      return;
+    }
     for (URI url : this.pdAddrs) {
+      BackOffer backOffer = this.probeBackOffer();
       // since resp is null, we need update leader's address by walking through all pd server.
-      GetMembersResponse resp = getMembers(url);
+      GetMembersResponse resp = getMembers(backOffer, url);
       if (resp == null) {
         continue;
       }
+      if (resp.getLeader().getClientUrlsList().isEmpty()) {
+        continue;
+      }
+
+      String leaderUrlStr = resp.getLeader().getClientUrlsList().get(0);
+      leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
+
       // if leader is switched, just return.
-      if (switchLeader(resp.getLeader().getClientUrlsList())) {
+      if (checkHealth(backOffer, leaderUrlStr, hostMapping)
+          && createLeaderClientWrapper(leaderUrlStr)) {
+        lastUpdateLeaderTime = System.currentTimeMillis();
+        return;
+      }
+
+      if (!conf.getEnableGrpcForward()) {
+        continue;
+      }
+
+      logger.info(String.format("can not switch to new leader, try follower forward"));
+      List<Pdpb.Member> members = resp.getMembersList();
+
+      // If we have not used follower forward, try the first follower.
+      boolean hasReachNextMember =
+          pdClientWrapper != null && pdClientWrapper.getStoreAddress().equals(leaderUrlStr);
+
+      for (int i = 0; i < members.size() * 2; i++) {
+        Pdpb.Member member = members.get(i % members.size());
+        if (member.getMemberId() == resp.getLeader().getMemberId()) {
+          continue;
+        }
+        String followerUrlStr = member.getClientUrlsList().get(0);
+        followerUrlStr = uriToAddr(addrToUri(followerUrlStr));
+        if (pdClientWrapper != null && pdClientWrapper.getStoreAddress().equals(followerUrlStr)) {
+          hasReachNextMember = true;
+          continue;
+        }
+        if (hasReachNextMember
+            && createFollowerClientWrapper(backOffer, followerUrlStr, leaderUrlStr)) {
+          logger.warn(
+              String.format("forward request to pd [%s] by pd [%s]", leaderUrlStr, followerUrlStr));
+          return;
+        }
+      }
+    }
+    lastUpdateLeaderTime = System.currentTimeMillis();
+    if (pdClientWrapper == null) {
+      throw new TiClientInternalException(
+          "already tried all address on file, but not leader found yet.");
+    }
+  }
+
+  public void tryUpdateLeader() {
+    logger.info("try update leader");
+    for (URI url : this.pdAddrs) {
+      BackOffer backOffer = this.probeBackOffer();
+      // since resp is null, we need update leader's address by walking through all pd server.
+      GetMembersResponse resp = getMembers(backOffer, url);
+      if (resp == null) {
+        continue;
+      }
+      List<URI> urls =
+          resp.getMembersList()
+              .stream()
+              .map(mem -> addrToUri(mem.getClientUrls(0)))
+              .collect(Collectors.toList());
+      String leaderUrlStr = resp.getLeader().getClientUrlsList().get(0);
+      leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
+
+      // If leader is not change but becomes available, we can cancel follower forward.
+      if (checkHealth(backOffer, leaderUrlStr, hostMapping) && trySwitchLeader(leaderUrlStr)) {
+        if (!urls.equals(this.pdAddrs)) {
+          tryUpdateMembers(urls);
+        }
         return;
       }
     }
-    throw new TiClientInternalException(
+    lastUpdateLeaderTime = System.currentTimeMillis();
-        "already tried all address on file, but not leader found yet.");
+    if (pdClientWrapper == null) {
+      throw new TiClientInternalException(
+          "already tried all address on file, but not leader found yet.");
+    }
+  }
+
+  private synchronized void tryUpdateMembers(List<URI> members) {
+    this.pdAddrs = members;
   }
 
   public void updateTiFlashReplicaStatus() {
@@ -491,77 +728,130 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
 
   @Override
   protected PDBlockingStub getBlockingStub() {
-    if (leaderWrapper == null) {
+    if (pdClientWrapper == null) {
       throw new GrpcException("PDClient may not be initialized");
     }
-    return leaderWrapper
+    return pdClientWrapper.getBlockingStub().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
-        .getBlockingStub()
-        .withDeadlineAfter(getConf().getTimeout(), getConf().getTimeoutUnit());
   }
 
   @Override
-  protected PDStub getAsyncStub() {
+  protected PDFutureStub getAsyncStub() {
-    if (leaderWrapper == null) {
+    if (pdClientWrapper == null) {
       throw new GrpcException("PDClient may not be initialized");
     }
-    return leaderWrapper
+    return pdClientWrapper.getAsyncStub().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
-        .getAsyncStub()
-        .withDeadlineAfter(getConf().getTimeout(), getConf().getTimeoutUnit());
   }
 
   private void initCluster() {
+    logger.info("init cluster: start");
     GetMembersResponse resp = null;
-    List<URI> pdAddrs = getConf().getPdAddrs();
+    List<URI> pdAddrs = new ArrayList<>(getConf().getPdAddrs());
+    // shuffle PD addresses so that clients call getMembers from different PD
+    Collections.shuffle(pdAddrs);
+    this.pdAddrs = pdAddrs;
+    this.etcdClient =
+        Client.builder()
+            .endpoints(pdAddrs)
+            .executorService(
+                Executors.newCachedThreadPool(
+                    new ThreadFactoryBuilder()
+                        .setNameFormat("etcd-conn-manager-pool-%d")
+                        .setDaemon(true)
+                        .build()))
+            .build();
+    logger.info("init host mapping: start");
+    this.hostMapping =
+        Optional.ofNullable(getConf().getHostMapping())
+            .orElseGet(() -> new DefaultHostMapping(this.etcdClient, conf.getNetworkMappingName()));
+    logger.info("init host mapping: end");
+    // The first request may cost too much latency
+    long originTimeout = this.timeout;
+    this.timeout = conf.getPdFirstGetMemberTimeout();
     for (URI u : pdAddrs) {
-      resp = getMembers(u);
+      logger.info("get members with pd " + u + ": start");
+      resp = getMembers(defaultBackOffer(), u);
+      logger.info("get members with pd " + u + ": end");
       if (resp != null) {
         break;
       }
     }
+    if (resp == null) {
+      logger.error("Could not get leader member with: " + pdAddrs);
+    }
+
+    this.timeout = originTimeout;
     checkNotNull(resp, "Failed to init client for PD cluster.");
     long clusterId = resp.getHeader().getClusterId();
     header = RequestHeader.newBuilder().setClusterId(clusterId).build();
     tsoReq = TsoRequest.newBuilder().setHeader(header).setCount(1).build();
-    this.pdAddrs = pdAddrs;
-    this.etcdClient = Client.builder().endpoints(pdAddrs).build();
     this.tiflashReplicaMap = new ConcurrentHashMap<>();
-    createLeaderWrapper(resp.getLeader().getClientUrls(0));
+    this.pdAddrs =
+        resp.getMembersList()
+            .stream()
+            .map(mem -> addrToUri(mem.getClientUrls(0)))
+            .collect(Collectors.toList());
+    logger.info("init cluster with address: " + this.pdAddrs);
+
+    String leaderUrlStr = resp.getLeader().getClientUrls(0);
+    leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
+    logger.info("createLeaderClientWrapper with leader " + leaderUrlStr + ": start");
+    createLeaderClientWrapper(leaderUrlStr);
+    logger.info("createLeaderClientWrapper with leader " + leaderUrlStr + ": end");
     service =
         Executors.newSingleThreadScheduledExecutor(
-            new ThreadFactoryBuilder().setDaemon(true).build());
+            new ThreadFactoryBuilder()
+                .setNameFormat("PDClient-update-leader-pool-%d")
+                .setDaemon(true)
+                .build());
     service.scheduleAtFixedRate(
         () -> {
           // Wrap this with a try catch block in case schedule update fails
           try {
-            updateLeader();
+            tryUpdateLeader();
           } catch (Exception e) {
             logger.warn("Update leader failed", e);
           }
         },
-        1,
+        10,
-        1,
+        10,
-        TimeUnit.MINUTES);
+        TimeUnit.SECONDS);
-    tiflashReplicaService =
+    if (conf.isTiFlashEnabled()) {
-        Executors.newSingleThreadScheduledExecutor(
+      tiflashReplicaService =
-            new ThreadFactoryBuilder().setDaemon(true).build());
+          Executors.newSingleThreadScheduledExecutor(
-    tiflashReplicaService.scheduleAtFixedRate(
+              new ThreadFactoryBuilder()
-        this::updateTiFlashReplicaStatus, 10, 10, TimeUnit.SECONDS);
+                  .setNameFormat("PDClient-tiflash-replica-pool-%d")
+                  .setDaemon(true)
+                  .build());
+      tiflashReplicaService.scheduleAtFixedRate(
+          this::updateTiFlashReplicaStatus, 10, 10, TimeUnit.SECONDS);
+    }
+    logger.info("init cluster: finish");
   }
 
-  static class LeaderWrapper {
+  static class PDClientWrapper {
     private final String leaderInfo;
     private final PDBlockingStub blockingStub;
-    private final PDStub asyncStub;
+    private final PDFutureStub asyncStub;
     private final long createTime;
+    private final String storeAddress;
 
-    LeaderWrapper(
+    PDClientWrapper(
-        String leaderInfo,
+        String leaderInfo, String storeAddress, ManagedChannel clientChannel, long createTime) {
-        PDGrpc.PDBlockingStub blockingStub,
+      if (!storeAddress.equals(leaderInfo)) {
-        PDGrpc.PDStub asyncStub,
+        Metadata header = new Metadata();
-        long createTime) {
+        header.put(TiConfiguration.PD_FORWARD_META_DATA_KEY, addrToUri(leaderInfo).toString());
+        this.blockingStub =
+            PDGrpc.newBlockingStub(clientChannel)
+                .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header));
+        this.asyncStub =
+            PDGrpc.newFutureStub(clientChannel)
+                .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header));
+      } else {
+        this.blockingStub = PDGrpc.newBlockingStub(clientChannel);
+        this.asyncStub = PDGrpc.newFutureStub(clientChannel);
+      }
       this.leaderInfo = leaderInfo;
-      this.blockingStub = blockingStub;
+      this.storeAddress = storeAddress;
-      this.asyncStub = asyncStub;
       this.createTime = createTime;
     }
 
@@ -569,11 +859,15 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
       return leaderInfo;
     }
 
+    String getStoreAddress() {
+      return storeAddress;
+    }
+
     PDBlockingStub getBlockingStub() {
       return blockingStub;
     }
 
-    PDStub getAsyncStub() {
+    PDFutureStub getAsyncStub() {
       return asyncStub;
     }
 
@@ -583,7 +877,28 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
 
     @Override
     public String toString() {
-      return "[leaderInfo: " + leaderInfo + "]";
+      return "[leaderInfo: " + leaderInfo + ", storeAddress: " + storeAddress + "]";
     }
   }
+
+  public Long getClusterId() {
+    return header.getClusterId();
+  }
+
+  public List<URI> getPdAddrs() {
+    return pdAddrs;
+  }
+
+  public RequestKeyCodec getCodec() {
+    return codec;
+  }
+
+  private static BackOffer defaultBackOffer() {
+    return ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF);
+  }
+
+  private BackOffer probeBackOffer() {
+    int maxSleep = (int) getTimeout() * 2;
+    return ConcreteBackOffer.newCustomBackOff(maxSleep);
+  }
 }

src/main/java/org/tikv/common/ReadOnlyPDClient.java

@@ -1,27 +1,31 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
 
 import com.google.protobuf.ByteString;
 import java.util.List;
-import java.util.concurrent.Future;
+import org.tikv.common.apiversion.RequestKeyCodec;
 import org.tikv.common.meta.TiTimestamp;
-import org.tikv.common.region.TiRegion;
 import org.tikv.common.util.BackOffer;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Metapb;
 import org.tikv.kvproto.Metapb.Store;
+import org.tikv.kvproto.Pdpb;
 
 /** Readonly PD client including only reading related interface Supposed for TiDB-like use cases */
 public interface ReadOnlyPDClient {
@@ -38,9 +42,7 @@ public interface ReadOnlyPDClient {
    * @param key key in bytes for locating a region
    * @return the region whose startKey and endKey range covers the given key
    */
-  TiRegion getRegionByKey(BackOffer backOffer, ByteString key);
+  Pair<Metapb.Region, Metapb.Peer> getRegionByKey(BackOffer backOffer, ByteString key);
-
-  Future<TiRegion> getRegionByKeyAsync(BackOffer backOffer, ByteString key);
 
   /**
    * Get Region by Region Id
@@ -48,9 +50,12 @@ public interface ReadOnlyPDClient {
    * @param id Region Id
    * @return the region corresponding to the given Id
    */
-  TiRegion getRegionByID(BackOffer backOffer, long id);
+  Pair<Metapb.Region, Metapb.Peer> getRegionByID(BackOffer backOffer, long id);
 
-  Future<TiRegion> getRegionByIDAsync(BackOffer backOffer, long id);
+  List<Pdpb.Region> scanRegions(
+      BackOffer backOffer, ByteString startKey, ByteString endKey, int limit);
+
+  HostMapping getHostMapping();
 
   /**
    * Get Store by StoreId
@@ -60,9 +65,23 @@ public interface ReadOnlyPDClient {
    */
   Store getStore(BackOffer backOffer, long storeId);
 
-  Future<Store> getStoreAsync(BackOffer backOffer, long storeId);
-
   List<Store> getAllStores(BackOffer backOffer);
 
-  boolean isReplicaRead();
+  TiConfiguration.ReplicaRead getReplicaRead();
+
+  Long getClusterId();
+
+  RequestKeyCodec getCodec();
+
+  /**
+   * Update ServiceGCSafePoint
+   *
+   * @param serviceId ServiceId
+   * @param ttl TTL in seconds
+   * @param safePoint The TiTimestamp you want to set. Set to start_ts.getPrevious() is a good
+   *     practice
+   * @return the MinSafePoint of all services. If this value is greater than safePoint, it means
+   *     update failed
+   */
+  Long updateServiceGCSafePoint(String serviceId, long ttl, long safePoint, BackOffer backOffer);
 }

src/main/java/org/tikv/common/Snapshot.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -40,12 +42,10 @@ import org.tikv.kvproto.Kvrpcpb.KvPair;
 public class Snapshot {
   private final TiTimestamp timestamp;
   private final TiSession session;
-  private final TiConfiguration conf;
 
-  public Snapshot(@Nonnull TiTimestamp timestamp, TiConfiguration conf) {
+  public Snapshot(@Nonnull TiTimestamp timestamp, TiSession session) {
     this.timestamp = timestamp;
-    this.conf = conf;
+    this.session = session;
-    this.session = TiSession.getInstance(conf);
   }
 
   public TiSession getSession() {
@@ -67,7 +67,7 @@ public class Snapshot {
   }
 
   public ByteString get(ByteString key) {
-    try (KVClient client = new KVClient(session.getConf(), session.getRegionStoreClientBuilder())) {
+    try (KVClient client = new KVClient(session, session.getRegionStoreClientBuilder())) {
       return client.get(key, timestamp.getVersion());
     }
   }
@@ -77,10 +77,12 @@ public class Snapshot {
     for (byte[] key : keys) {
       list.add(ByteString.copyFrom(key));
     }
-    try (KVClient client = new KVClient(session.getConf(), session.getRegionStoreClientBuilder())) {
+    try (KVClient client = new KVClient(session, session.getRegionStoreClientBuilder())) {
       List<KvPair> kvPairList =
           client.batchGet(
-              ConcreteBackOffer.newCustomBackOff(backOffer), list, timestamp.getVersion());
+              ConcreteBackOffer.newCustomBackOff(backOffer, session.getPDClient().getClusterId()),
+              list,
+              timestamp.getVersion());
       return kvPairList
           .stream()
           .map(
@@ -175,6 +177,6 @@ public class Snapshot {
   }
 
   public TiConfiguration getConf() {
-    return conf;
+    return this.session.getConf();
   }
 }

src/main/java/org/tikv/common/StoreVersion.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -25,7 +27,6 @@ import org.tikv.common.util.ConcreteBackOffer;
 import org.tikv.kvproto.Metapb;
 
 public class StoreVersion {
-
   private static final int SCALE = 10000;
   private final Logger logger = LoggerFactory.getLogger(this.getClass());
   private int v0 = 9999;
@@ -60,7 +61,8 @@ public class StoreVersion {
   public static boolean minTiKVVersion(String version, PDClient pdClient) {
     StoreVersion storeVersion = new StoreVersion(version);
 
-    BackOffer bo = ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF);
+    BackOffer bo =
+        ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF, pdClient.getClusterId());
     List<Metapb.Store> storeList =
         pdClient
             .getAllStores(bo)

src/main/java/org/tikv/common/TiBatchWriteUtils.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2019 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/TiDBJDBCClient.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2019 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -39,6 +41,7 @@ public class TiDBJDBCClient implements AutoCloseable {
   private static final int DELAY_CLEAN_TABLE_LOCK_DEFAULT = 0;
   private static final String TIDB_ROW_FORMAT_VERSION_SQL = "select @@tidb_row_format_version";
   private static final int TIDB_ROW_FORMAT_VERSION_DEFAULT = 1;
+  private static final ObjectMapper objectMapper = new ObjectMapper();
   private final Logger logger = LoggerFactory.getLogger(getClass().getName());
   private final Connection connection;
 
@@ -118,7 +121,6 @@ public class TiDBJDBCClient implements AutoCloseable {
 
   private Map<String, Object> readConfMapFromTiDB() throws SQLException, IOException {
     String configJSON = (String) queryTiDBViaJDBC(SELECT_TIDB_CONFIG_SQL).get(0).get(0);
-    ObjectMapper objectMapper = new ObjectMapper();
     TypeReference<HashMap<String, Object>> typeRef =
         new TypeReference<HashMap<String, Object>>() {};
     return objectMapper.readValue(configJSON, typeRef);

src/main/java/org/tikv/common/TiSession.java

@@ -1,20 +1,24 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
 
+import static org.tikv.common.util.ClientUtils.groupKeysByRegion;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.protobuf.ByteString;
@@ -22,57 +26,254 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.tikv.common.apiversion.RequestKeyCodec;
+import org.tikv.common.apiversion.RequestKeyV1RawCodec;
+import org.tikv.common.apiversion.RequestKeyV1TxnCodec;
+import org.tikv.common.apiversion.RequestKeyV2RawCodec;
+import org.tikv.common.apiversion.RequestKeyV2TxnCodec;
 import org.tikv.common.catalog.Catalog;
-import org.tikv.common.event.CacheInvalidateEvent;
 import org.tikv.common.exception.TiKVException;
+import org.tikv.common.importer.ImporterStoreClient;
+import org.tikv.common.importer.SwitchTiKVModeClient;
 import org.tikv.common.key.Key;
 import org.tikv.common.meta.TiTimestamp;
 import org.tikv.common.region.RegionManager;
 import org.tikv.common.region.RegionStoreClient;
-import org.tikv.common.region.RegionStoreClient.RegionStoreClientBuilder;
 import org.tikv.common.region.TiRegion;
-import org.tikv.common.util.*;
+import org.tikv.common.region.TiStore;
+import org.tikv.common.util.BackOffFunction;
+import org.tikv.common.util.BackOffer;
+import org.tikv.common.util.ChannelFactory;
+import org.tikv.common.util.ConcreteBackOffer;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Errorpb;
+import org.tikv.kvproto.ImportSstpb;
 import org.tikv.kvproto.Metapb;
+import org.tikv.kvproto.Pdpb;
 import org.tikv.raw.RawKVClient;
+import org.tikv.raw.SmartRawKVClient;
+import org.tikv.service.failsafe.CircuitBreaker;
+import org.tikv.service.failsafe.CircuitBreakerImpl;
 import org.tikv.txn.KVClient;
 import org.tikv.txn.TxnKVClient;
 
 /**
  * TiSession is the holder for PD Client, Store pdClient and PD Cache All sessions share common
- * region store connection pool but separated PD conn and cache for better concurrency TiSession is
+ * region store connection pool but separated PD conn and cache for better concurrency
- * thread-safe but it's also recommended to have multiple session avoiding lock contention
+ *
+ * <p>TiSession is thread-safe but it's also recommended to have multiple session avoiding lock
+ * contention
  */
 public class TiSession implements AutoCloseable {
   private static final Logger logger = LoggerFactory.getLogger(TiSession.class);
   private static final Map<String, TiSession> sessionCachedMap = new HashMap<>();
   private final TiConfiguration conf;
+  private final RequestKeyCodec keyCodec;
   private final ChannelFactory channelFactory;
-  private Function<CacheInvalidateEvent, Void> cacheInvalidateCallback;
   // below object creation is either heavy or making connection (pd), pending for lazy loading
   private volatile PDClient client;
   private volatile Catalog catalog;
   private volatile ExecutorService indexScanThreadPool;
   private volatile ExecutorService tableScanThreadPool;
+  private volatile ExecutorService batchGetThreadPool;
+  private volatile ExecutorService batchPutThreadPool;
+  private volatile ExecutorService batchDeleteThreadPool;
+  private volatile ExecutorService batchScanThreadPool;
+  private volatile ExecutorService deleteRangeThreadPool;
   private volatile RegionManager regionManager;
+  private final boolean enableGrpcForward;
   private volatile RegionStoreClient.RegionStoreClientBuilder clientBuilder;
-  private boolean isClosed = false;
+  private volatile ImporterStoreClient.ImporterStoreClientBuilder importerClientBuilder;
+  private volatile boolean isClosed = false;
+  private volatile SwitchTiKVModeClient switchTiKVModeClient;
+  private final MetricsServer metricsServer;
+  private final CircuitBreaker circuitBreaker;
+  private static final int MAX_SPLIT_REGION_STACK_DEPTH = 6;
 
-  public TiSession(TiConfiguration conf) {
+  static {
-    this.conf = conf;
+    logger.info("Welcome to TiKV Java Client {}", getVersionInfo());
-    this.channelFactory = new ChannelFactory(conf.getMaxFrameSize());
-    this.client = PDClient.createRaw(conf, channelFactory);
   }
 
+  private static class VersionInfo {
+
+    private final String buildVersion;
+    private final String commitHash;
+
+    public VersionInfo(String buildVersion, String commitHash) {
+      this.buildVersion = buildVersion;
+      this.commitHash = commitHash;
+    }
+
+    @Override
+    public String toString() {
+      return buildVersion + "@" + commitHash;
+    }
+  }
+
+  public TiSession(TiConfiguration conf) {
+    // may throw org.tikv.common.MetricsServer  - http server not up
+    // put it at the beginning of this function to avoid unclosed Thread
+    this.metricsServer = MetricsServer.getInstance(conf);
+
+    this.conf = conf;
+
+    if (conf.getApiVersion().isV1()) {
+      if (conf.isRawKVMode()) {
+        keyCodec = new RequestKeyV1RawCodec();
+      } else {
+        keyCodec = new RequestKeyV1TxnCodec();
+      }
+    } else {
+      if (conf.isRawKVMode()) {
+        keyCodec = new RequestKeyV2RawCodec();
+      } else {
+        keyCodec = new RequestKeyV2TxnCodec();
+      }
+    }
+
+    if (conf.isTlsEnable()) {
+      if (conf.isJksEnable()) {
+        this.channelFactory =
+            new ChannelFactory(
+                conf.getMaxFrameSize(),
+                conf.getKeepaliveTime(),
+                conf.getKeepaliveTimeout(),
+                conf.getIdleTimeout(),
+                conf.getConnRecycleTimeInSeconds(),
+                conf.getCertReloadIntervalInSeconds(),
+                conf.getJksKeyPath(),
+                conf.getJksKeyPassword(),
+                conf.getJksTrustPath(),
+                conf.getJksTrustPassword());
+      } else {
+        this.channelFactory =
+            new ChannelFactory(
+                conf.getMaxFrameSize(),
+                conf.getKeepaliveTime(),
+                conf.getKeepaliveTimeout(),
+                conf.getIdleTimeout(),
+                conf.getConnRecycleTimeInSeconds(),
+                conf.getCertReloadIntervalInSeconds(),
+                conf.getTrustCertCollectionFile(),
+                conf.getKeyCertChainFile(),
+                conf.getKeyFile());
+      }
+    } else {
+      this.channelFactory =
+          new ChannelFactory(
+              conf.getMaxFrameSize(),
+              conf.getKeepaliveTime(),
+              conf.getKeepaliveTimeout(),
+              conf.getIdleTimeout());
+    }
+
+    this.client = PDClient.createRaw(conf, keyCodec, channelFactory);
+    if (conf.getApiVersion().isV2() && !StoreVersion.minTiKVVersion(Version.API_V2, client)) {
+      throw new IllegalStateException(
+          "With API v2, store versions should not older than " + Version.API_V2);
+    }
+
+    this.enableGrpcForward = conf.getEnableGrpcForward();
+    if (this.enableGrpcForward) {
+      logger.info("enable grpc forward for high available");
+    }
+    if (conf.isWarmUpEnable() && conf.isRawKVMode()) {
+      warmUp();
+    }
+    this.circuitBreaker = new CircuitBreakerImpl(conf, client.getClusterId());
+    logger.info(
+        "TiSession initialized in "
+            + conf.getKvMode()
+            + " mode in API version: "
+            + conf.getApiVersion());
+  }
+
+  private static VersionInfo getVersionInfo() {
+    VersionInfo info;
+    try {
+      final Properties properties = new Properties();
+      properties.load(TiSession.class.getClassLoader().getResourceAsStream("git.properties"));
+      String version = properties.getProperty("git.build.version");
+      String commitHash = properties.getProperty("git.commit.id.full");
+      info = new VersionInfo(version, commitHash);
+    } catch (Exception e) {
+      logger.info("Fail to read package info: " + e.getMessage());
+      info = new VersionInfo("unknown", "unknown");
+    }
+    return info;
+  }
+
+  @VisibleForTesting
+  public synchronized void warmUp() {
+    long warmUpStartTime = System.nanoTime();
+    BackOffer backOffer = ConcreteBackOffer.newRawKVBackOff(getPDClient().getClusterId());
+    try {
+      // let JVM ClassLoader load gRPC error related classes
+      // this operation may cost 100ms
+      Errorpb.Error.newBuilder().setNotLeader(Errorpb.NotLeader.newBuilder().build()).build();
+
+      this.client = getPDClient();
+      this.regionManager = getRegionManager();
+      List<Metapb.Store> stores = this.client.getAllStores(backOffer);
+      // warm up store cache
+      for (Metapb.Store store : stores) {
+        this.regionManager.updateStore(
+            null, new TiStore(this.client.getStore(backOffer, store.getId())));
+      }
+
+      // use scan region to load region cache with limit
+      ByteString startKey = ByteString.EMPTY;
+      do {
+        List<Pdpb.Region> regions =
+            regionManager.scanRegions(
+                backOffer, startKey, ByteString.EMPTY, conf.getScanRegionsLimit());
+        if (regions == null || regions.isEmpty()) {
+          // something went wrong, but the warm-up process could continue
+          break;
+        }
+        for (Pdpb.Region region : regions) {
+          regionManager.insertRegionToCache(
+              regionManager.createRegion(region.getRegion(), backOffer));
+        }
+        startKey = regions.get(regions.size() - 1).getRegion().getEndKey();
+      } while (!startKey.isEmpty());
+
+      try (RawKVClient rawKVClient = createRawClient()) {
+        ByteString exampleKey = ByteString.EMPTY;
+        Optional<ByteString> prev = rawKVClient.get(exampleKey);
+        if (prev.isPresent()) {
+          rawKVClient.delete(exampleKey);
+          rawKVClient.putIfAbsent(exampleKey, prev.get());
+          rawKVClient.put(exampleKey, prev.get());
+        } else {
+          rawKVClient.putIfAbsent(exampleKey, ByteString.EMPTY);
+          rawKVClient.put(exampleKey, ByteString.EMPTY);
+          rawKVClient.delete(exampleKey);
+        }
+      }
+    } catch (Exception e) {
+      // ignore error
+      logger.info("warm up fails, ignored ", e);
+    } finally {
+      logger.info(
+          String.format(
+              "warm up duration %d ms", (System.nanoTime() - warmUpStartTime) / 1_000_000));
+    }
+  }
+
+  @VisibleForTesting
   public static TiSession create(TiConfiguration conf) {
     return new TiSession(conf);
   }
 
+  @Deprecated
   public static TiSession getInstance(TiConfiguration conf) {
     synchronized (sessionCachedMap) {
       String key = conf.getPdAddrsString();
@@ -87,35 +288,66 @@ public class TiSession implements AutoCloseable {
   }
 
   public RawKVClient createRawClient() {
-    // Create new Region Manager avoiding thread contentions
+    checkIsClosed();
-    RegionManager regionMgr = new RegionManager(client);
+
-    RegionStoreClientBuilder builder =
+    return new RawKVClient(this, this.getRegionStoreClientBuilder());
-        new RegionStoreClientBuilder(conf, channelFactory, regionMgr, client);
+  }
-    return new RawKVClient(conf, builder);
+
+  public SmartRawKVClient createSmartRawClient() {
+    RawKVClient rawKVClient = createRawClient();
+    return new SmartRawKVClient(rawKVClient, circuitBreaker);
   }
 
   public KVClient createKVClient() {
-    // Create new Region Manager avoiding thread contentions
+    checkIsClosed();
-    RegionManager regionMgr = new RegionManager(client);
+
-    RegionStoreClientBuilder builder =
+    return new KVClient(this.conf, this.getRegionStoreClientBuilder(), this);
-        new RegionStoreClientBuilder(conf, channelFactory, regionMgr, client);
-    return new KVClient(conf, builder);
   }
 
   public TxnKVClient createTxnClient() {
+    checkIsClosed();
+
     return new TxnKVClient(conf, this.getRegionStoreClientBuilder(), this.getPDClient());
   }
 
   public RegionStoreClient.RegionStoreClientBuilder getRegionStoreClientBuilder() {
-    RegionStoreClient.RegionStoreClientBuilder res = clientBuilder;
+    checkIsClosed();
+
+    if (this.clientBuilder != null) {
+      return this.clientBuilder;
+    }
+
+    // lazily create the clientBuilder for the current TiSession
+    synchronized (this) {
+      if (this.clientBuilder == null) {
+        this.clientBuilder =
+            new RegionStoreClient.RegionStoreClientBuilder(
+                this.conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
+      }
+    }
+    return this.clientBuilder;
+  }
+
+  public ImporterStoreClient.ImporterStoreClientBuilder getImporterRegionStoreClientBuilder() {
+    checkIsClosed();
+
+    ImporterStoreClient.ImporterStoreClientBuilder res = importerClientBuilder;
     if (res == null) {
       synchronized (this) {
-        if (clientBuilder == null) {
+        if (importerClientBuilder == null) {
-          clientBuilder =
+          if (conf.isTxnKVMode()) {
-              new RegionStoreClient.RegionStoreClientBuilder(
+            importerClientBuilder =
-                  conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
+                new ImporterStoreClient.ImporterStoreClientBuilder<
+                    ImportSstpb.WriteRequest, ImportSstpb.WriteRequest>(
+                    conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
+          } else {
+            importerClientBuilder =
+                new ImporterStoreClient.ImporterStoreClientBuilder<
+                    ImportSstpb.RawWriteRequest, ImportSstpb.RawWriteResponse>(
+                    conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
+          }
         }
-        res = clientBuilder;
+        res = importerClientBuilder;
       }
     }
     return res;
@@ -126,23 +358,32 @@ public class TiSession implements AutoCloseable {
   }
 
   public TiTimestamp getTimestamp() {
-    return getPDClient().getTimestamp(ConcreteBackOffer.newTsoBackOff());
+    checkIsClosed();
+
+    return getPDClient()
+        .getTimestamp(ConcreteBackOffer.newTsoBackOff(getPDClient().getClusterId()));
   }
 
   public Snapshot createSnapshot() {
-    return new Snapshot(getTimestamp(), this.conf);
+    checkIsClosed();
+
+    return new Snapshot(getTimestamp(), this);
   }
 
   public Snapshot createSnapshot(TiTimestamp ts) {
-    return new Snapshot(ts, conf);
+    checkIsClosed();
+
+    return new Snapshot(ts, this);
   }
 
   public PDClient getPDClient() {
+    checkIsClosed();
+
     PDClient res = client;
     if (res == null) {
       synchronized (this) {
         if (client == null) {
-          client = PDClient.createRaw(this.getConf(), channelFactory);
+          client = PDClient.createRaw(this.getConf(), keyCodec, channelFactory);
         }
         res = client;
       }
@@ -151,6 +392,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public Catalog getCatalog() {
+    checkIsClosed();
+
     Catalog res = catalog;
     if (res == null) {
       synchronized (this) {
@@ -163,12 +406,14 @@ public class TiSession implements AutoCloseable {
     return res;
   }
 
-  public synchronized RegionManager getRegionManager() {
+  public RegionManager getRegionManager() {
+    checkIsClosed();
+
     RegionManager res = regionManager;
     if (res == null) {
       synchronized (this) {
         if (regionManager == null) {
-          regionManager = new RegionManager(getPDClient(), this.cacheInvalidateCallback);
+          regionManager = new RegionManager(getConf(), getPDClient(), this.channelFactory);
         }
         res = regionManager;
       }
@@ -177,6 +422,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForIndexScan() {
+    checkIsClosed();
+
     ExecutorService res = indexScanThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -196,6 +443,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForTableScan() {
+    checkIsClosed();
+
     ExecutorService res = tableScanThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -211,47 +460,174 @@ public class TiSession implements AutoCloseable {
     return res;
   }
 
+  public ExecutorService getThreadPoolForBatchPut() {
+    checkIsClosed();
+
+    ExecutorService res = batchPutThreadPool;
+    if (res == null) {
+      synchronized (this) {
+        if (batchPutThreadPool == null) {
+          batchPutThreadPool =
+              Executors.newFixedThreadPool(
+                  conf.getBatchPutConcurrency(),
+                  new ThreadFactoryBuilder()
+                      .setNameFormat("batchPut-thread-%d")
+                      .setDaemon(true)
+                      .build());
+        }
+        res = batchPutThreadPool;
+      }
+    }
+    return res;
+  }
+
+  public ExecutorService getThreadPoolForBatchGet() {
+    checkIsClosed();
+
+    ExecutorService res = batchGetThreadPool;
+    if (res == null) {
+      synchronized (this) {
+        if (batchGetThreadPool == null) {
+          batchGetThreadPool =
+              Executors.newFixedThreadPool(
+                  conf.getBatchGetConcurrency(),
+                  new ThreadFactoryBuilder()
+                      .setNameFormat("batchGet-thread-%d")
+                      .setDaemon(true)
+                      .build());
+        }
+        res = batchGetThreadPool;
+      }
+    }
+    return res;
+  }
+
+  public ExecutorService getThreadPoolForBatchDelete() {
+    checkIsClosed();
+
+    ExecutorService res = batchDeleteThreadPool;
+    if (res == null) {
+      synchronized (this) {
+        if (batchDeleteThreadPool == null) {
+          batchDeleteThreadPool =
+              Executors.newFixedThreadPool(
+                  conf.getBatchDeleteConcurrency(),
+                  new ThreadFactoryBuilder()
+                      .setNameFormat("batchDelete-thread-%d")
+                      .setDaemon(true)
+                      .build());
+        }
+        res = batchDeleteThreadPool;
+      }
+    }
+    return res;
+  }
+
+  public ExecutorService getThreadPoolForBatchScan() {
+    checkIsClosed();
+
+    ExecutorService res = batchScanThreadPool;
+    if (res == null) {
+      synchronized (this) {
+        if (batchScanThreadPool == null) {
+          batchScanThreadPool =
+              Executors.newFixedThreadPool(
+                  conf.getBatchScanConcurrency(),
+                  new ThreadFactoryBuilder()
+                      .setNameFormat("batchScan-thread-%d")
+                      .setDaemon(true)
+                      .build());
+        }
+        res = batchScanThreadPool;
+      }
+    }
+    return res;
+  }
+
+  public ExecutorService getThreadPoolForDeleteRange() {
+    checkIsClosed();
+
+    ExecutorService res = deleteRangeThreadPool;
+    if (res == null) {
+      synchronized (this) {
+        if (deleteRangeThreadPool == null) {
+          deleteRangeThreadPool =
+              Executors.newFixedThreadPool(
+                  conf.getDeleteRangeConcurrency(),
+                  new ThreadFactoryBuilder()
+                      .setNameFormat("deleteRange-thread-%d")
+                      .setDaemon(true)
+                      .build());
+        }
+        res = deleteRangeThreadPool;
+      }
+    }
+    return res;
+  }
+
   @VisibleForTesting
   public ChannelFactory getChannelFactory() {
+    checkIsClosed();
+
     return channelFactory;
   }
 
   /**
-   * This is used for setting call back function to invalidate cache information
+   * SwitchTiKVModeClient is used for SST Ingest.
    *
-   * @param callBackFunc callback function
+   * @return a SwitchTiKVModeClient
    */
-  public void injectCallBackFunc(Function<CacheInvalidateEvent, Void> callBackFunc) {
+  public SwitchTiKVModeClient getSwitchTiKVModeClient() {
-    this.cacheInvalidateCallback = callBackFunc;
+    checkIsClosed();
+
+    SwitchTiKVModeClient res = switchTiKVModeClient;
+    if (res == null) {
+      synchronized (this) {
+        if (switchTiKVModeClient == null) {
+          switchTiKVModeClient =
+              new SwitchTiKVModeClient(getPDClient(), getImporterRegionStoreClientBuilder());
+        }
+        res = switchTiKVModeClient;
+      }
+    }
+    return res;
   }
 
   /**
    * split region and scatter
    *
    * @param splitKeys
+   * @param splitRegionBackoffMS
+   * @param scatterRegionBackoffMS
+   * @param scatterWaitMS
    */
   public void splitRegionAndScatter(
       List<byte[]> splitKeys,
       int splitRegionBackoffMS,
       int scatterRegionBackoffMS,
       int scatterWaitMS) {
+    checkIsClosed();
+
     logger.info(String.format("split key's size is %d", splitKeys.size()));
     long startMS = System.currentTimeMillis();
 
     // split region
-    List<TiRegion> newRegions =
+    List<Metapb.Region> newRegions =
         splitRegion(
             splitKeys
                 .stream()
-                .map(k -> Key.toRawKey(k).next().toByteString())
+                .map(k -> Key.toRawKey(k).toByteString())
                 .collect(Collectors.toList()),
-            ConcreteBackOffer.newCustomBackOff(splitRegionBackoffMS));
+            ConcreteBackOffer.newCustomBackOff(splitRegionBackoffMS, getPDClient().getClusterId()));
 
     // scatter region
-    for (TiRegion newRegion : newRegions) {
+    for (Metapb.Region newRegion : newRegions) {
       try {
         getPDClient()
-            .scatterRegion(newRegion, ConcreteBackOffer.newCustomBackOff(scatterRegionBackoffMS));
+            .scatterRegion(
+                newRegion,
+                ConcreteBackOffer.newCustomBackOff(
+                    scatterRegionBackoffMS, getPDClient().getClusterId()));
       } catch (Exception e) {
         logger.warn(String.format("failed to scatter region: %d", newRegion.getId()), e);
       }
@@ -261,14 +637,16 @@ public class TiSession implements AutoCloseable {
     if (scatterWaitMS > 0) {
       logger.info("start to wait scatter region finish");
       long scatterRegionStartMS = System.currentTimeMillis();
-      for (TiRegion newRegion : newRegions) {
+      for (Metapb.Region newRegion : newRegions) {
         long remainMS = (scatterRegionStartMS + scatterWaitMS) - System.currentTimeMillis();
         if (remainMS <= 0) {
           logger.warn("wait scatter region timeout");
           return;
         }
         getPDClient()
-            .waitScatterRegionFinish(newRegion, ConcreteBackOffer.newCustomBackOff((int) remainMS));
+            .waitScatterRegionFinish(
+                newRegion,
+                ConcreteBackOffer.newCustomBackOff((int) remainMS, getPDClient().getClusterId()));
       }
     } else {
       logger.info("skip to wait scatter region finish");
@@ -278,16 +656,36 @@ public class TiSession implements AutoCloseable {
     logger.info("splitRegionAndScatter cost {} seconds", (endMS - startMS) / 1000);
   }
 
-  private List<TiRegion> splitRegion(List<ByteString> splitKeys, BackOffer backOffer) {
+  /**
-    List<TiRegion> regions = new ArrayList<>();
+   * split region and scatter
+   *
+   * @param splitKeys
+   */
+  public void splitRegionAndScatter(List<byte[]> splitKeys) {
+    checkIsClosed();
 
-    Map<TiRegion, List<ByteString>> groupKeys = groupKeysByRegion(splitKeys);
+    int splitRegionBackoffMS = BackOffer.SPLIT_REGION_BACKOFF;
+    int scatterRegionBackoffMS = BackOffer.SCATTER_REGION_BACKOFF;
+    int scatterWaitMS = conf.getScatterWaitSeconds() * 1000;
+    splitRegionAndScatter(splitKeys, splitRegionBackoffMS, scatterRegionBackoffMS, scatterWaitMS);
+  }
+
+  private List<Metapb.Region> splitRegion(List<ByteString> splitKeys, BackOffer backOffer) {
+    return splitRegion(splitKeys, backOffer, 1);
+  }
+
+  private List<Metapb.Region> splitRegion(
+      List<ByteString> splitKeys, BackOffer backOffer, int depth) {
+    List<Metapb.Region> regions = new ArrayList<>();
+
+    Map<TiRegion, List<ByteString>> groupKeys =
+        groupKeysByRegion(getRegionManager(), splitKeys, backOffer);
     for (Map.Entry<TiRegion, List<ByteString>> entry : groupKeys.entrySet()) {
 
-      Pair<TiRegion, Metapb.Store> pair =
+      Pair<TiRegion, TiStore> pair =
           getRegionManager().getRegionStorePairByKey(entry.getKey().getStartKey());
       TiRegion region = pair.first;
-      Metapb.Store store = pair.second;
+      TiStore store = pair.second;
       List<ByteString> splits =
           entry
               .getValue()
@@ -300,15 +698,25 @@ public class TiSession implements AutoCloseable {
             "split key equal to region start key or end key. Region splitting is not needed.");
       } else {
         logger.info("start to split region id={}, split size={}", region.getId(), splits.size());
-        List<TiRegion> newRegions;
+        List<Metapb.Region> newRegions;
         try {
           newRegions = getRegionStoreClientBuilder().build(region, store).splitRegion(splits);
+          // invalidate old region
+          getRegionManager().invalidateRegion(region);
         } catch (final TiKVException e) {
           // retry
           logger.warn("ReSplitting ranges for splitRegion", e);
-          clientBuilder.getRegionManager().invalidateRegion(region.getId());
+          getRegionManager().invalidateRegion(region);
           backOffer.doBackOff(BackOffFunction.BackOffFuncType.BoRegionMiss, e);
-          newRegions = splitRegion(splits, backOffer);
+          if (depth >= MAX_SPLIT_REGION_STACK_DEPTH) {
+            logger.warn(
+                String.format(
+                    "Skip split region because MAX_SPLIT_REGION_STACK_DEPTH(%d) reached!",
+                    MAX_SPLIT_REGION_STACK_DEPTH));
+            newRegions = new ArrayList<>();
+          } else {
+            newRegions = splitRegion(splits, backOffer, depth + 1);
+          }
         }
         logger.info("region id={}, new region size={}", region.getId(), newRegions.size());
         regions.addAll(newRegions);
@@ -319,34 +727,119 @@ public class TiSession implements AutoCloseable {
     return regions;
   }
 
-  private Map<TiRegion, List<ByteString>> groupKeysByRegion(List<ByteString> keys) {
+  private void checkIsClosed() {
-    return keys.stream()
+    if (isClosed) {
-        .collect(Collectors.groupingBy(clientBuilder.getRegionManager()::getRegionByKey));
+      throw new RuntimeException("this TiSession is closed!");
+    }
+  }
+
+  public synchronized void closeAwaitTermination(long timeoutMS) throws Exception {
+    shutdown(false);
+
+    long startMS = System.currentTimeMillis();
+    while (true) {
+      if (isTerminatedExecutorServices()) {
+        cleanAfterTerminated();
+        return;
+      }
+
+      if (System.currentTimeMillis() - startMS > timeoutMS) {
+        shutdown(true);
+        return;
+      }
+      Thread.sleep(500);
+    }
   }
 
   @Override
   public synchronized void close() throws Exception {
-    if (isClosed) {
+    shutdown(true);
-      logger.warn("this TiSession is already closed!");
+  }
-      return;
+
+  private synchronized void shutdown(boolean now) throws Exception {
+    if (!isClosed) {
+      isClosed = true;
+      synchronized (sessionCachedMap) {
+        sessionCachedMap.remove(conf.getPdAddrsString());
+      }
+
+      if (metricsServer != null) {
+        metricsServer.close();
+      }
+
+      if (circuitBreaker != null) {
+        circuitBreaker.close();
+      }
     }
 
-    isClosed = true;
+    if (now) {
-    synchronized (sessionCachedMap) {
+      shutdownNowExecutorServices();
-      sessionCachedMap.remove(conf.getPdAddrsString());
+      cleanAfterTerminated();
-    }
+    } else {
-
+      shutdownExecutorServices();
-    if (tableScanThreadPool != null) {
-      tableScanThreadPool.shutdownNow();
-    }
-    if (indexScanThreadPool != null) {
-      indexScanThreadPool.shutdownNow();
-    }
-    if (client != null) {
-      getPDClient().close();
-    }
-    if (catalog != null) {
-      getCatalog().close();
     }
   }
+
+  private synchronized void cleanAfterTerminated() throws InterruptedException {
+    if (regionManager != null) {
+      regionManager.close();
+    }
+    if (client != null) {
+      client.close();
+    }
+    if (catalog != null) {
+      catalog.close();
+    }
+
+    if (switchTiKVModeClient != null) {
+      switchTiKVModeClient.stopKeepTiKVToImportMode();
+    }
+  }
+
+  private List<ExecutorService> getExecutorServices() {
+    List<ExecutorService> executorServiceList = new ArrayList<>();
+    if (tableScanThreadPool != null) {
+      executorServiceList.add(tableScanThreadPool);
+    }
+    if (indexScanThreadPool != null) {
+      executorServiceList.add(indexScanThreadPool);
+    }
+    if (batchGetThreadPool != null) {
+      executorServiceList.add(batchGetThreadPool);
+    }
+    if (batchPutThreadPool != null) {
+      executorServiceList.add(batchPutThreadPool);
+    }
+    if (batchDeleteThreadPool != null) {
+      executorServiceList.add(batchDeleteThreadPool);
+    }
+    if (batchScanThreadPool != null) {
+      executorServiceList.add(batchScanThreadPool);
+    }
+    if (deleteRangeThreadPool != null) {
+      executorServiceList.add(deleteRangeThreadPool);
+    }
+    return executorServiceList;
+  }
+
+  private void shutdownExecutorServices() {
+    for (ExecutorService executorService : getExecutorServices()) {
+      executorService.shutdown();
+    }
+  }
+
+  private void shutdownNowExecutorServices() {
+    for (ExecutorService executorService : getExecutorServices()) {
+      executorService.shutdownNow();
+    }
+  }
+
+  private boolean isTerminatedExecutorServices() {
+    for (ExecutorService executorService : getExecutorServices()) {
+      if (!executorService.isTerminated()) {
+        return false;
+      }
+    }
+    return true;
+  }
 }

src/main/java/org/tikv/common/Utils.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Locale;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Utils {
+
+  // org.apache.spark.network.util.JavaUtils
+  private static final ImmutableMap<String, TimeUnit> timeSuffixes =
+      ImmutableMap.<String, TimeUnit>builder()
+          .put("us", TimeUnit.MICROSECONDS)
+          .put("ms", TimeUnit.MILLISECONDS)
+          .put("s", TimeUnit.SECONDS)
+          .put("m", TimeUnit.MINUTES)
+          .put("min", TimeUnit.MINUTES)
+          .put("h", TimeUnit.HOURS)
+          .put("d", TimeUnit.DAYS)
+          .build();
+
+  public static ConcurrentHashMap<String, String> getSystemProperties() {
+    ConcurrentHashMap<String, String> map = new ConcurrentHashMap<>();
+    System.getProperties()
+        .stringPropertyNames()
+        .forEach(key -> map.put(key, System.getProperty(key)));
+    return map;
+  }
+
+  public static void setSystemProperties(ConcurrentHashMap<String, String> settings) {
+    Properties prop = new Properties();
+    settings.forEach(prop::setProperty);
+    System.setProperties(prop);
+  }
+
+  // org.apache.spark.network.util.JavaUtils
+  public static long timeStringAs(String str, TimeUnit unit) {
+    String lower = str.toLowerCase(Locale.ROOT).trim();
+
+    try {
+      Matcher m = Pattern.compile("(-?[0-9]+)([a-z]+)?").matcher(lower);
+      if (!m.matches()) {
+        throw new NumberFormatException("Failed to parse time string: " + str);
+      }
+
+      long val = Long.parseLong(m.group(1));
+      String suffix = m.group(2);
+
+      // Check for invalid suffixes
+      if (suffix != null && !timeSuffixes.containsKey(suffix)) {
+        throw new NumberFormatException("Invalid suffix: \"" + suffix + "\"");
+      }
+
+      // If suffix is valid use that, otherwise none was provided and use the default passed
+      return unit.convert(val, suffix != null ? timeSuffixes.get(suffix) : unit);
+    } catch (NumberFormatException e) {
+      String timeError =
+          "Time must be specified as seconds (s), "
+              + "milliseconds (ms), microseconds (us), minutes (m or min), hour (h), or day (d). "
+              + "E.g. 50s, 100ms, or 250us.";
+
+      throw new NumberFormatException(timeError + "\n" + e.getMessage());
+    }
+  }
+
+  public static long timeStringAsMs(String str) {
+    return timeStringAs(str, TimeUnit.MILLISECONDS);
+  }
+
+  public static long timeStringAsSec(String str) {
+    return timeStringAs(str, TimeUnit.SECONDS);
+  }
+}

src/main/java/org/tikv/common/Version.java

@@ -1,15 +1,15 @@
 /*
- *
+ * Copyright 2021 TiKV Project Authors.
- * Copyright 2020 PingCAP, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
@@ -25,4 +25,6 @@ public class Version {
   public static final String RESOLVE_LOCK_V4 = "4.0.0";
 
   public static final String BATCH_WRITE = "3.0.14";
+
+  public static final String API_V2 = "6.1.0";
 }

src/main/java/org/tikv/common/allocator/RowIDAllocator.java

@@ -1,265 +0,0 @@
-/*
- * Copyright 2019 PingCAP, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.tikv.common.allocator;
-
-import com.google.common.primitives.UnsignedLongs;
-import com.google.protobuf.ByteString;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.function.Function;
-import org.tikv.common.Snapshot;
-import org.tikv.common.TiConfiguration;
-import org.tikv.common.TiSession;
-import org.tikv.common.codec.CodecDataInput;
-import org.tikv.common.codec.CodecDataOutput;
-import org.tikv.common.codec.MetaCodec;
-import org.tikv.common.exception.AllocateRowIDOverflowException;
-import org.tikv.common.exception.TiBatchWriteException;
-import org.tikv.common.meta.TiTableInfo;
-import org.tikv.common.util.BackOffer;
-import org.tikv.common.util.ConcreteBackOffer;
-import org.tikv.txn.TwoPhaseCommitter;
-
-/**
- * RowIDAllocator read current start from TiKV and write back 'start+step' back to TiKV. It designs
- * to allocate all id for data to be written at once, hence it does not need run inside a txn.
- *
- * <p>(start, end] is allocated
- */
-public final class RowIDAllocator implements Serializable {
-  private final long maxShardRowIDBits;
-  private final long dbId;
-  private final TiConfiguration conf;
-  private final long step;
-  private long end;
-
-  private RowIDAllocator(long maxShardRowIDBits, long dbId, long step, TiConfiguration conf) {
-    this.maxShardRowIDBits = maxShardRowIDBits;
-    this.dbId = dbId;
-    this.step = step;
-    this.conf = conf;
-  }
-
-  /**
-   * @param index should >= 1
-   * @return
-   */
-  public long getShardRowId(long index) {
-    return getShardRowId(maxShardRowIDBits, index, index + getStart());
-  }
-
-  static long getShardRowId(long maxShardRowIDBits, long partitionIndex, long rowID) {
-    if (maxShardRowIDBits <= 0 || maxShardRowIDBits >= 16) {
-      return rowID;
-    }
-
-    // assert rowID < Math.pow(2, 64 - maxShardRowIDBits)
-
-    long partition = partitionIndex & ((1L << maxShardRowIDBits) - 1);
-    return rowID | (partition << (64 - maxShardRowIDBits - 1));
-  }
-
-  public static RowIDAllocator create(
-      long dbId, TiTableInfo table, TiConfiguration conf, boolean unsigned, long step) {
-    RowIDAllocator allocator = new RowIDAllocator(table.getMaxShardRowIDBits(), dbId, step, conf);
-    if (unsigned) {
-      allocator.initUnsigned(
-          TiSession.getInstance(conf).createSnapshot(),
-          table.getId(),
-          table.getMaxShardRowIDBits());
-    } else {
-      allocator.initSigned(
-          TiSession.getInstance(conf).createSnapshot(),
-          table.getId(),
-          table.getMaxShardRowIDBits());
-    }
-
-    return allocator;
-  }
-
-  public long getStart() {
-    return end - step;
-  }
-
-  public long getEnd() {
-    return end;
-  }
-
-  // set key value pair to tikv via two phase committer protocol.
-  private void set(ByteString key, byte[] value) {
-    TiSession session = TiSession.getInstance(conf);
-    TwoPhaseCommitter twoPhaseCommitter =
-        new TwoPhaseCommitter(conf, session.getTimestamp().getVersion());
-
-    twoPhaseCommitter.prewritePrimaryKey(
-        ConcreteBackOffer.newCustomBackOff(BackOffer.PREWRITE_MAX_BACKOFF),
-        key.toByteArray(),
-        value);
-
-    twoPhaseCommitter.commitPrimaryKey(
-        ConcreteBackOffer.newCustomBackOff(BackOffer.BATCH_COMMIT_BACKOFF),
-        key.toByteArray(),
-        session.getTimestamp().getVersion());
-
-    try {
-      twoPhaseCommitter.close();
-    } catch (Throwable ignored) {
-    }
-  }
-
-  private void updateMeta(ByteString key, byte[] oldVal, Snapshot snapshot) {
-    // 1. encode hash meta key
-    // 2. load meta via hash meta key from TiKV
-    // 3. update meta's filed count and set it back to TiKV
-    CodecDataOutput cdo = new CodecDataOutput();
-    ByteString metaKey = MetaCodec.encodeHashMetaKey(cdo, key.toByteArray());
-    long fieldCount;
-    ByteString metaVal = snapshot.get(metaKey);
-
-    // decode long from bytes
-    // big endian the 8 bytes
-    fieldCount = new CodecDataInput(metaVal.toByteArray()).readLong();
-
-    // update meta field count only oldVal is null
-    if (oldVal == null || oldVal.length == 0) {
-      fieldCount++;
-      cdo.reset();
-      cdo.writeLong(fieldCount);
-
-      set(metaKey, cdo.toBytes());
-    }
-  }
-
-  private long updateHash(
-      ByteString key,
-      ByteString field,
-      Function<byte[], byte[]> calculateNewVal,
-      Snapshot snapshot) {
-    // 1. encode hash data key
-    // 2. get value in byte from get operation
-    // 3. calculate new value via calculateNewVal
-    // 4. check old value equals to new value or not
-    // 5. set the new value back to TiKV via 2pc
-    // 6. encode a hash meta key
-    // 7. update a hash meta field count if needed
-
-    CodecDataOutput cdo = new CodecDataOutput();
-    MetaCodec.encodeHashDataKey(cdo, key.toByteArray(), field.toByteArray());
-    ByteString dataKey = cdo.toByteString();
-    byte[] oldVal = snapshot.get(dataKey.toByteArray());
-
-    byte[] newVal = calculateNewVal.apply(oldVal);
-    if (Arrays.equals(newVal, oldVal)) {
-      // not need to update
-      return 0L;
-    }
-
-    set(dataKey, newVal);
-    updateMeta(key, oldVal, snapshot);
-    return Long.parseLong(new String(newVal));
-  }
-
-  private static boolean isDBExisted(long dbId, Snapshot snapshot) {
-    ByteString dbKey = MetaCodec.encodeDatabaseID(dbId);
-    ByteString json = MetaCodec.hashGet(MetaCodec.KEY_DBs, dbKey, snapshot);
-    return json != null && !json.isEmpty();
-  }
-
-  private static boolean isTableExisted(long dbId, long tableId, Snapshot snapshot) {
-    ByteString dbKey = MetaCodec.encodeDatabaseID(dbId);
-    ByteString tableKey = MetaCodec.tableKey(tableId);
-    return !MetaCodec.hashGet(dbKey, tableKey, snapshot).isEmpty();
-  }
-
-  public static boolean shardRowBitsOverflow(
-      long base, long step, long shardRowBits, boolean reservedSignBit) {
-    long signBit = reservedSignBit ? 1 : 0;
-    long mask = ((1L << shardRowBits) - 1) << (64 - shardRowBits - signBit);
-    if (reservedSignBit) {
-      return ((base + step) & mask) > 0;
-    } else {
-      return Long.compareUnsigned((base + step) & mask, 0) > 0;
-    }
-  }
-
-  /**
-   * read current row id from TiKV and write the calculated value back to TiKV. The calculation rule
-   * is start(read from TiKV) + step.
-   */
-  public long udpateAllocateId(
-      long dbId, long tableId, long step, Snapshot snapshot, long shard, boolean hasSignedBit) {
-    if (isDBExisted(dbId, snapshot) && isTableExisted(dbId, tableId, snapshot)) {
-      return updateHash(
-          MetaCodec.encodeDatabaseID(dbId),
-          MetaCodec.autoTableIDKey(tableId),
-          (oldVal) -> {
-            long base = 0;
-            if (oldVal != null && oldVal.length != 0) {
-              base = Long.parseLong(new String(oldVal));
-            }
-            if (shard >= 1 && shardRowBitsOverflow(base, step, shard, hasSignedBit)) {
-              throw new AllocateRowIDOverflowException(base, step, shard);
-            }
-            base += step;
-            return String.valueOf(base).getBytes();
-          },
-          snapshot);
-    }
-
-    throw new IllegalArgumentException("table or database is not existed");
-  }
-
-  /** read current row id from TiKV according to database id and table id. */
-  public static long getAllocateId(long dbId, long tableId, Snapshot snapshot) {
-    if (isDBExisted(dbId, snapshot) && isTableExisted(dbId, tableId, snapshot)) {
-      ByteString dbKey = MetaCodec.encodeDatabaseID(dbId);
-      ByteString tblKey = MetaCodec.autoTableIDKey(tableId);
-      ByteString val = MetaCodec.hashGet(dbKey, tblKey, snapshot);
-      if (val.isEmpty()) return 0L;
-      return Long.parseLong(val.toStringUtf8());
-    }
-
-    throw new IllegalArgumentException("table or database is not existed");
-  }
-
-  private void initSigned(Snapshot snapshot, long tableId, long shard) {
-    // get new start from TiKV, and calculate new end and set it back to TiKV.
-    long newStart = getAllocateId(dbId, tableId, snapshot);
-    long tmpStep = Math.min(Long.MAX_VALUE - newStart, step);
-    if (tmpStep != step) {
-      throw new TiBatchWriteException("cannot allocate ids for this write");
-    }
-    if (newStart == Long.MAX_VALUE) {
-      throw new TiBatchWriteException("cannot allocate more ids since it ");
-    }
-    end = udpateAllocateId(dbId, tableId, tmpStep, snapshot, shard, true);
-  }
-
-  private void initUnsigned(Snapshot snapshot, long tableId, long shard) {
-    // get new start from TiKV, and calculate new end and set it back to TiKV.
-    long newStart = getAllocateId(dbId, tableId, snapshot);
-    // for unsigned long, -1L is max value.
-    long tmpStep = UnsignedLongs.min(-1L - newStart, step);
-    if (tmpStep != step) {
-      throw new TiBatchWriteException("cannot allocate ids for this write");
-    }
-    // when compare unsigned long, the min value is largest value.
-    if (UnsignedLongs.compare(newStart, -1L) == 0) {
-      throw new TiBatchWriteException(
-          "cannot allocate more ids since the start reaches " + "unsigned long's max value ");
-    }
-    end = udpateAllocateId(dbId, tableId, tmpStep, snapshot, shard, false);
-  }
-}

src/main/java/org/tikv/common/apiversion/CodecUtils.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import org.tikv.common.codec.Codec.BytesCodec;
+import org.tikv.common.codec.CodecDataInput;
+import org.tikv.common.codec.CodecDataOutput;
+
+// TODO(iosmanthus): use ByteString.wrap to avoid once more copying.
+public class CodecUtils {
+  public static ByteString encode(ByteString key) {
+    CodecDataOutput cdo = new CodecDataOutput();
+    BytesCodec.writeBytes(cdo, key.toByteArray());
+    return cdo.toByteString();
+  }
+
+  public static ByteString decode(ByteString key) {
+    return ByteString.copyFrom(BytesCodec.readBytes(new CodecDataInput(key)));
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyCodec.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Kvrpcpb.KvPair;
+import org.tikv.kvproto.Kvrpcpb.Mutation;
+import org.tikv.kvproto.Metapb;
+import org.tikv.kvproto.Pdpb;
+
+public interface RequestKeyCodec {
+  ByteString encodeKey(ByteString key);
+
+  default List<ByteString> encodeKeys(List<ByteString> keys) {
+    return keys.stream().map(this::encodeKey).collect(Collectors.toList());
+  }
+
+  default List<Mutation> encodeMutations(List<Mutation> mutations) {
+    return mutations
+        .stream()
+        .map(mut -> Mutation.newBuilder().mergeFrom(mut).setKey(encodeKey(mut.getKey())).build())
+        .collect(Collectors.toList());
+  }
+
+  ByteString decodeKey(ByteString key);
+
+  default KvPair decodeKvPair(KvPair pair) {
+    return KvPair.newBuilder().mergeFrom(pair).setKey(decodeKey(pair.getKey())).build();
+  }
+
+  default List<KvPair> decodeKvPairs(List<KvPair> pairs) {
+    return pairs.stream().map(this::decodeKvPair).collect(Collectors.toList());
+  }
+
+  Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end);
+
+  ByteString encodePdQuery(ByteString key);
+
+  Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end);
+
+  Metapb.Region decodeRegion(Metapb.Region region);
+
+  default List<Pdpb.Region> decodePdRegions(List<Pdpb.Region> regions) {
+    return regions
+        .stream()
+        .map(
+            r ->
+                Pdpb.Region.newBuilder()
+                    .mergeFrom(r)
+                    .setRegion(this.decodeRegion(r.getRegion()))
+                    .build())
+        .collect(Collectors.toList());
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV1Codec.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import java.util.List;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Kvrpcpb.KvPair;
+import org.tikv.kvproto.Kvrpcpb.Mutation;
+import org.tikv.kvproto.Metapb.Region;
+import org.tikv.kvproto.Pdpb;
+
+public class RequestKeyV1Codec implements RequestKeyCodec {
+  @Override
+  public ByteString encodeKey(ByteString key) {
+    return key;
+  }
+
+  @Override
+  public List<ByteString> encodeKeys(List<ByteString> keys) {
+    return keys;
+  }
+
+  @Override
+  public List<Mutation> encodeMutations(List<Mutation> mutations) {
+    return mutations;
+  }
+
+  @Override
+  public ByteString decodeKey(ByteString key) {
+    return key;
+  }
+
+  @Override
+  public KvPair decodeKvPair(KvPair pair) {
+    return pair;
+  }
+
+  @Override
+  public List<KvPair> decodeKvPairs(List<KvPair> pairs) {
+    return pairs;
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end) {
+    return Pair.create(start, end);
+  }
+
+  @Override
+  public ByteString encodePdQuery(ByteString key) {
+    return key;
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
+    return Pair.create(start, end);
+  }
+
+  @Override
+  public Region decodeRegion(Region region) {
+    return region;
+  }
+
+  @Override
+  public List<Pdpb.Region> decodePdRegions(List<Pdpb.Region> regions) {
+    return regions;
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV1RawCodec.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+public class RequestKeyV1RawCodec extends RequestKeyV1Codec implements RequestKeyCodec {
+  public RequestKeyV1RawCodec() {}
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV1TxnCodec.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Metapb;
+
+public class RequestKeyV1TxnCodec extends RequestKeyV1Codec implements RequestKeyCodec {
+  public RequestKeyV1TxnCodec() {}
+
+  @Override
+  public ByteString encodePdQuery(ByteString key) {
+    return CodecUtils.encode(key);
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
+    if (!start.isEmpty()) {
+      start = CodecUtils.encode(start);
+    }
+
+    if (!end.isEmpty()) {
+      end = CodecUtils.encode(end);
+    }
+
+    return Pair.create(start, end);
+  }
+
+  @Override
+  public Metapb.Region decodeRegion(Metapb.Region region) {
+    Metapb.Region.Builder builder = Metapb.Region.newBuilder().mergeFrom(region);
+    ByteString start = region.getStartKey();
+    ByteString end = region.getEndKey();
+
+    if (!start.isEmpty()) {
+      start = CodecUtils.decode(start);
+    }
+
+    if (!end.isEmpty()) {
+      end = CodecUtils.decode(end);
+    }
+
+    return builder.setStartKey(start).setEndKey(end).build();
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV2Codec.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Metapb;
+import org.tikv.kvproto.Metapb.Region;
+
+public class RequestKeyV2Codec implements RequestKeyCodec {
+  protected static final ByteString RAW_DEFAULT_PREFIX =
+      ByteString.copyFrom(new byte[] {'r', 0, 0, 0});
+  protected static final ByteString RAW_DEFAULT_END =
+      ByteString.copyFrom(new byte[] {'r', 0, 0, 1});
+  protected static final ByteString TXN_DEFAULT_PREFIX =
+      ByteString.copyFrom(new byte[] {'x', 0, 0, 0});
+  protected static final ByteString TXN_DEFAULT_END =
+      ByteString.copyFrom(new byte[] {'x', 0, 0, 1});
+  protected ByteString keyPrefix;
+  protected ByteString infiniteEndKey;
+
+  @Override
+  public ByteString encodeKey(ByteString key) {
+    return keyPrefix.concat(key);
+  }
+
+  @Override
+  public ByteString decodeKey(ByteString key) {
+    if (key.isEmpty()) {
+      return key;
+    }
+
+    if (!key.startsWith(keyPrefix)) {
+      throw new IllegalArgumentException("key corrupted, wrong prefix");
+    }
+
+    return key.substring(keyPrefix.size());
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end) {
+    start = encodeKey(start);
+
+    end = end.isEmpty() ? infiniteEndKey : encodeKey(end);
+
+    return Pair.create(start, end);
+  }
+
+  @Override
+  public ByteString encodePdQuery(ByteString key) {
+    return CodecUtils.encode(encodeKey(key));
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
+    Pair<ByteString, ByteString> range = encodeRange(start, end);
+    return Pair.create(CodecUtils.encode(range.first), CodecUtils.encode(range.second));
+  }
+
+  @Override
+  public Region decodeRegion(Region region) {
+    Metapb.Region.Builder builder = Metapb.Region.newBuilder().mergeFrom(region);
+
+    ByteString start = region.getStartKey();
+    ByteString end = region.getEndKey();
+
+    if (!start.isEmpty()) {
+      start = CodecUtils.decode(start);
+    }
+
+    if (!end.isEmpty()) {
+      end = CodecUtils.decode(end);
+    }
+
+    if (ByteString.unsignedLexicographicalComparator().compare(start, infiniteEndKey) >= 0
+        || (!end.isEmpty()
+            && ByteString.unsignedLexicographicalComparator().compare(end, keyPrefix) <= 0)) {
+      throw new IllegalArgumentException("region out of keyspace" + region.toString());
+    }
+
+    start = start.startsWith(keyPrefix) ? start.substring(keyPrefix.size()) : ByteString.EMPTY;
+    end = end.startsWith(keyPrefix) ? end.substring(keyPrefix.size()) : ByteString.EMPTY;
+
+    return builder.setStartKey(start).setEndKey(end).build();
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV2RawCodec.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+public class RequestKeyV2RawCodec extends RequestKeyV2Codec {
+  public RequestKeyV2RawCodec() {
+    super();
+
+    this.keyPrefix = RAW_DEFAULT_PREFIX;
+    this.infiniteEndKey = RAW_DEFAULT_END;
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV2TxnCodec.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+public class RequestKeyV2TxnCodec extends RequestKeyV2Codec {
+  public RequestKeyV2TxnCodec() {
+    super();
+
+    this.keyPrefix = TXN_DEFAULT_PREFIX;
+    this.infiniteEndKey = TXN_DEFAULT_END;
+  }
+}

src/main/java/org/tikv/common/catalog/Catalog.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.catalog;

src/main/java/org/tikv/common/catalog/CatalogTransaction.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.catalog;
@@ -38,6 +40,7 @@ import org.tikv.common.util.Pair;
 
 public class CatalogTransaction {
   protected static final Logger logger = LoggerFactory.getLogger(CatalogTransaction.class);
+  private static final ObjectMapper mapper = new ObjectMapper();
   private final Snapshot snapshot;
 
   CatalogTransaction(Snapshot snapshot) {
@@ -49,7 +52,6 @@ public class CatalogTransaction {
     Objects.requireNonNull(cls, "cls is null");
 
     logger.debug(String.format("Parse Json %s : %s", cls.getSimpleName(), json.toStringUtf8()));
-    ObjectMapper mapper = new ObjectMapper();
     try {
       return mapper.readValue(json.toStringUtf8(), cls);
     } catch (JsonParseException | JsonMappingException e) {

src/main/java/org/tikv/common/codec/Codec.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;
@@ -20,6 +22,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import gnu.trove.list.array.TIntArrayList;
 import java.math.BigDecimal;
 import java.sql.Date;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -32,6 +35,7 @@ import org.tikv.common.ExtendedDateTime;
 import org.tikv.common.exception.ConvertOverflowException;
 import org.tikv.common.exception.InvalidCodecFormatException;
 import org.tikv.common.exception.TypeException;
+import org.tikv.common.exception.UnsupportedSyntaxException;
 
 public class Codec {
 
@@ -522,6 +526,37 @@ public class Codec {
         int minute,
         int second,
         int microsec) {
+      boolean zeroDate = false, zeroTime = false;
+      boolean zeroInDate = false;
+      if (year == 0 && month == 0 && day == 0) {
+        zeroDate = true;
+      }
+      if (hour == 0 && minute == 0 && microsec == 0) {
+        zeroTime = true;
+      }
+      if (month == 0 || day == 0) {
+        zeroInDate = true;
+      }
+      // This behavior can be modified using the zeroDateTimeBehavior configuration property.
+      // The allowable values are:
+      //    * exception (the default), which throws an SQLException with an SQLState of S1009.
+      //    * convertToNull, which returns NULL instead of the date.
+      //    * round, which rounds the date to the nearest closest value which is 0001-01-01.
+      if (zeroDate && zeroTime) {
+        year = 1;
+        month = 1;
+        day = 1;
+      } else if (!zeroDate && zeroInDate) {
+        String dateString = String.format("%04d-%02d-%02d", year, month, day);
+        try {
+          java.util.Date d = new SimpleDateFormat("yyyy-MM-dd").parse(dateString);
+          year = d.getYear() + 1900;
+          month = d.getMonth() + 1;
+          day = d.getDate();
+        } catch (Exception e) {
+          throw new UnsupportedSyntaxException("illegal date value: " + dateString);
+        }
+      }
       try {
         DateTime dateTime =
             new DateTime(year, month, day, hour, minute, second, microsec / 1000, tz);
@@ -634,6 +669,35 @@ public class Codec {
       int month = (int) (ym % 13);
       int year = (int) (ym / 13);
 
+      boolean zeroDate = false;
+      boolean zeroInDate = false;
+      if (year == 0 && month == 0 && day == 0) {
+        zeroDate = true;
+      }
+      if (month == 0 || day == 0) {
+        zeroInDate = true;
+      }
+      // This behavior can be modified using the zeroDateTimeBehavior configuration property.
+      // The allowable values are:
+      //    * exception (the default), which throws an SQLException with an SQLState of S1009.
+      //    * convertToNull, which returns NULL instead of the date.
+      //    * round, which rounds the date to the nearest closest value which is 0001-01-01.
+      if (zeroDate) {
+        year = 1;
+        month = 1;
+        day = 1;
+      } else if (zeroInDate) {
+        String dateString = String.format("%04d-%02d-%02d", year, month, day);
+        try {
+          java.util.Date d = new SimpleDateFormat("yyyy-MM-dd").parse(dateString);
+          year = d.getYear() + 1900;
+          month = d.getMonth() + 1;
+          day = d.getDate();
+        } catch (Exception e) {
+          throw new UnsupportedSyntaxException("illegal date value: " + dateString);
+        }
+      }
+
       return new LocalDate(year, month, day, null);
     }
 
@@ -723,6 +787,9 @@ public class Codec {
     }
 
     public static String readEnumFromIndex(int idx, List<String> elems) {
+      if (idx == -1) {
+        return "";
+      }
       if (idx < 0 || idx >= elems.size()) throw new TypeException("Index is out of range");
       return elems.get(idx);
     }

src/main/java/org/tikv/common/codec/CodecDataInput.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/CodecDataInputLittleEndian.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/CodecDataOutput.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/CodecDataOutputLittleEndian.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/KeyUtils.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/MetaCodec.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/MyDecimal.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/RowDecoderV2.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/RowEncoderV2.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;