[to #788] Reduce the number of ObjectMapper creations to improve performance (#787)

Signed-off-by: Jia Fan <fanjiaeminem@qq.com>

.ci/build.groovy

@@ -1,63 +0,0 @@
-def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) {
-    
-    catchError {
-        node ('build') {
-            container("java") {
-                stage('Prepare') {
-                    dir("/home/jenkins/agent/git/client-java") {
-                        sh """
-                        rm -rf /maven/.m2/repository/*
-                        rm -rf /maven/.m2/settings.xml
-                        rm -rf ~/.m2/settings.xml
-                        archive_url=http://fileserver.pingcap.net/download/builds/pingcap/client-java/cache/tikv-client-java-m2-cache-latest.tar.gz
-                        if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -sL \$archive_url | tar -zx -C /maven || true; fi
-                        """
-                        if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) {
-                            deleteDir()
-                        }
-                        checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: '+refs/pull/*:refs/remotes/origin/pr/*', url: 'git@github.com:tikv/client-java.git']]]
-                        sh "git checkout -f ${ghprbActualCommit}"
-                    }
-                }
-
-                stage('Format') {
-                    dir("/home/jenkins/agent/git/client-java") {
-                        sh """
-                        mvn com.coveo:fmt-maven-plugin:format
-                        git diff --quiet
-                        formatted="\$?"
-                        if [[ "\${formatted}" -eq 1 ]]
-                        then
-                           echo "code format error, please run the following commands:"
-                           echo "   mvn com.coveo:fmt-maven-plugin:format"
-                           exit 1
-                        fi
-                        """
-                    }
-                }
-
-                stage('Build') {
-                    dir("/home/jenkins/agent/git/client-java") {
-                        timeout(30) {
-                            sh ".ci/build.sh"
-                        }
-                    }
-                }
-            }
-        }
-        currentBuild.result = "SUCCESS"
-    }
-    
-    stage('Summary') {
-        def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
-        def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" +
-        "${ghprbPullLink}" + "\n" +
-        "${ghprbPullDescription}" + "\n" +
-        "Build Result: `${currentBuild.result}`" + "\n" +
-        "Elapsed Time: `${duration} mins` " + "\n" +
-        "${env.RUN_DISPLAY_URL}"
-        print msg
-    }
-}
-
-return this

.ci/build.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -x
-set -euo pipefail
-
-mvn clean compile

.ci/integration_test.groovy

@@ -1,112 +0,0 @@
-def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) {
-
-    def TIDB_BRANCH = "release-4.0"
-    def TIKV_BRANCH = "release-4.0"
-    def PD_BRANCH = "release-4.0"
-
-    // parse tidb branch
-    def m1 = ghprbCommentBody =~ /tidb\s*=\s*([^\s\\]+)(\s|\\|$)/
-    if (m1) {
-        TIDB_BRANCH = "${m1[0][1]}"
-    }
-    println "TIDB_BRANCH=${TIDB_BRANCH}"
-
-    // parse pd branch
-    def m2 = ghprbCommentBody =~ /pd\s*=\s*([^\s\\]+)(\s|\\|$)/
-    if (m2) {
-        PD_BRANCH = "${m2[0][1]}"
-    }
-    println "PD_BRANCH=${PD_BRANCH}"
-
-    // parse tikv branch
-    def m3 = ghprbCommentBody =~ /tikv\s*=\s*([^\s\\]+)(\s|\\|$)/
-    if (m3) {
-        TIKV_BRANCH = "${m3[0][1]}"
-    }
-    println "TIKV_BRANCH=${TIKV_BRANCH}"
-
-    catchError {
-        node ('build') {
-            container("java") {
-                stage('Prepare') {
-                    dir("/home/jenkins/agent/git/client-java") {
-                        sh """
-                        rm -rf /maven/.m2/repository/*
-                        rm -rf /maven/.m2/settings.xml
-                        rm -rf ~/.m2/settings.xml
-                        archive_url=http://fileserver.pingcap.net/download/builds/pingcap/client-java/cache/tikv-client-java-m2-cache-latest.tar.gz
-                        if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -sL \$archive_url | tar -zx -C /maven || true; fi
-                        """
-                        if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) {
-                            deleteDir()
-                        }
-                        checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: '+refs/pull/*:refs/remotes/origin/pr/*', url: 'git@github.com:tikv/client-java.git']]]
-                        sh "git checkout -f ${ghprbActualCommit}"
-                    }
-
-                    dir("/home/jenkins/agent/git/client-java/_run") {
-                        // tidb
-                        def tidb_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tidb/${TIDB_BRANCH}/sha1").trim()
-                        sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz | tar xz"
-                        // tikv
-                        def tikv_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tikv/${TIKV_BRANCH}/sha1").trim()
-                        sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz | tar xz"
-                        // pd
-                        def pd_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/pd/${PD_BRANCH}/sha1").trim()
-                        sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz | tar xz"
-
-                        sh """
-                        killall -9 tidb-server || true
-                        killall -9 tikv-server || true
-                        killall -9 pd-server || true
-                        killall -9 java || true
-                        sleep 10
-                        bin/pd-server --name=pd --data-dir=pd --config=../.ci/config/pd.toml &>pd.log &
-                        sleep 10
-                        bin/tikv-server --pd=127.0.0.1:2379 -s tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 --config=../.ci/config/tikv.toml &>tikv.log &
-                        sleep 10
-                        ps aux | grep '-server' || true
-                        curl -s 127.0.0.1:2379/pd/api/v1/status || true
-                        bin/tidb-server --store=tikv --path="127.0.0.1:2379" --config=../.ci/config/tidb.toml &>tidb.log &
-                        sleep 60
-                        """
-                    }
-                }
-
-                stage('Test') {
-                    dir("/home/jenkins/agent/git/client-java") {
-                        try {
-                            timeout(30) {
-                                sh ".ci/test.sh"
-                            }
-                        } catch (err) {
-                            sh """
-                            ps aux | grep '-server' || true
-                            curl -s 127.0.0.1:2379/pd/api/v1/status || true
-                            """
-                            sh "cat _run/pd.log"
-                            sh "cat _run/tikv.log"
-                            sh "cat _run/tidb.log"
-                            throw err
-                        }
-                    }
-                }
-            }
-        }
-        currentBuild.result = "SUCCESS"
-    }
-
-    stage('Summary') {
-        def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
-        def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" +
-        "${ghprbPullLink}" + "\n" +
-        "${ghprbPullDescription}" + "\n" +
-        "Integration Common Test Result: `${currentBuild.result}`" + "\n" +
-        "Elapsed Time: `${duration} mins` " + "\n" +
-        "${env.RUN_DISPLAY_URL}"
-
-        print msg
-    }
-}
-
-return this

.ci/test.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -x
-set -euo pipefail
-
-mvn clean test

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: "\U0001F41B Bug Report"
+about: As a User, I want to report a Bug.
+labels: type/bug
+---
+
+## Bug Report
+
+### 1. Describe the bug
+
+<!-- A clear and concise description of what the bug is. -->
+
+### 2. Minimal reproduce step (Required)
+
+<!-- a step by step guide for reproducing the bug. -->
+
+### 3. What did you see instead (Required)
+
+### 4. What did you expect to see? (Required)
+
+### 5. What are your Java Client and TiKV versions? (Required)
+
+- Client Java:
+- TiKV:

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,23 @@
+---
+name: "\U0001F680 Feature Request"
+about: As a user, I want to request a New Feature on the product.
+labels: type/feature-request
+---
+
+## Feature Request
+
+**Is your feature request related to a problem? Please describe:**
+<!-- A clear and concise description of what the problem is. Ex. I'm always
+frustrated when [...] -->
+
+**Describe the feature you'd like:**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered:**
+<!-- A clear and concise description of any alternative solutions or features
+you've considered. -->
+
+**Teachability, Documentation, Adoption, Migration Strategy:**
+<!-- If you can, explain some scenarios of how users might use this, situations it
+would be helpful in. Any API designs, mockups, or diagrams are also helpful.
+-->

.github/config/pd.toml

@@ -0,0 +1,4 @@
+# PD Configuration.
+[replication]
+enable-placement-rules = true
+max-replicas = 1

.github/config/tikv_rawkv.toml

@@ -0,0 +1,21 @@
+# TiKV Configuration.
+
+[raftstore]
+# set store capacity, if no set, use disk capacity.
+capacity = "6G"
+pd-heartbeat-tick-interval = "2s"
+pd-store-heartbeat-tick-interval = "5s"
+split-region-check-tick-interval = "1s"
+
+[rocksdb]
+max-open-files = 10000
+
+[raftdb]
+max-open-files = 10000
+
+[storage.block-cache]
+capacity = "128MB"
+
+[storage]
+reserve-space = "0MB"
+enable-ttl = true

.github/config/tikv_txnkv.toml

@@ -0,0 +1,20 @@
+# TiKV Configuration.
+
+[raftstore]
+# set store capacity, if no set, use disk capacity.
+capacity = "6G"
+pd-heartbeat-tick-interval = "2s"
+pd-store-heartbeat-tick-interval = "5s"
+split-region-check-tick-interval = "1s"
+
+[rocksdb]
+max-open-files = 10000
+
+[raftdb]
+max-open-files = 10000
+
+[storage.block-cache]
+capacity = "128MB"
+
+[storage]
+reserve-space = "0MB"

.github/config/tikv_v2.toml

@@ -0,0 +1,17 @@
+# TiKV Configuration.
+
+[raftstore]
+pd-heartbeat-tick-interval = "2s"
+pd-store-heartbeat-tick-interval = "5s"
+split-region-check-tick-interval = "1s"
+
+[rocksdb]
+max-open-files = 10000
+
+[raftdb]
+max-open-files = 10000
+
+[storage]
+reserve-space = "0MB"
+api-version = 2
+enable-ttl = true

.github/labeler.yml

@@ -0,0 +1,17 @@
+component/config:
+  - config/*
+
+component/br:
+  - src/main/java/org/tikv/br/*
+
+component/cdc:
+  - src/main/java/org/tikv/cdc/*
+
+component/common:
+  - src/main/java/org/tikv/common/*
+
+component/rawkv:
+  - src/main/java/org/tikv/raw/*
+
+component/txnkv:
+  - src/main/java/org/tikv/txn/*

.github/license-checker.yml

@@ -0,0 +1,26 @@
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: TiKV Project Authors.
+  paths-ignore:
+    - 'config/'
+    - 'dev/'
+    - 'docs/'
+    - 'metrics/'
+    - 'LICENSE'
+    - 'Makefile'
+    - 'pom.xml'
+    - 'shell.nix'
+    - '.ci/'
+    - '.gitignore'
+    - '.gitattributes'
+    - '.github/'
+    - '**/*.md'
+    - '**/*.properties'
+    - '**/*.json'
+    - '**/*.pem'
+    - '**/*.crt'
+    - '**/*.g4'
+    - 'src/main/java/io/grpc/'
+    - 'src/main/java/io/netty/'
+  comment: on-failure

.github/pull_request_template.md

@@ -0,0 +1,46 @@
+<!-- Thank you for contributing to TiKV Java Client!
+
+PR Title Format: "[close/to/fix #issue_number] summary" -->
+
+### What problem does this PR solve?
+
+Issue Number: close #issue_number
+
+Problem Description: **TBD**
+
+### What is changed and how does it work?
+
+
+
+### Code changes
+
+<!-- REMOVE the items that are not applicable -->
+- Has exported function/method change
+- Has exported variable/fields change
+- Has methods of interface change
+- Has persistent data change
+- No code
+
+### Check List for Tests
+
+This PR has been tested by at least one of the following methods:
+- Unit test
+- Integration test
+- Manual test (add detailed scripts or steps below)
+- No code
+
+### Side effects
+
+<!-- REMOVE the items that are not applicable -->
+- Possible performance regression, WHY: **TBD**
+- Increased code complexity, WHY: **TBD**
+- Breaking backward compatibility, WHY: **TBD**
+- NO side effects
+
+### Related changes
+
+<!-- REMOVE the items that are not applicable -->
+- Need to cherry-pick to the release branch
+- Need to update the documentation
+- Need to be included in the release note
+- NO related changes

.github/workflows/ci.yml

@@ -0,0 +1,84 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  format:
+    name: Check Format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8.0'
+          distribution: 'adopt'
+      - run: |
+          ./dev/javafmt
+          if [[ $(git diff) != "" ]]
+          then
+              echo "code format error, please run the following commands:"
+              echo "   ./dev/javafmt"
+              exit 1
+          fi
+
+  integration-test:
+    name: Integration Test - ${{ matrix.tikv_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tikv_version: [v5.0.6, v5.3.4, v5.4.3]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8.0'
+          distribution: 'adopt'
+      - name: Install TiUP
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+          /home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
+      - name: Start TiUP Playground
+        run: |
+          # Start TiKV in APIV1TTL
+          touch tiup-v1ttl.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag rawkv --mode tikv-slim --kv 1 --without-monitor --kv.port 20160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup-v1ttl.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1ttl.log)
+          cat tiup-v1ttl.log
+          echo "Wait for bootstrap"
+          sleep 10s
+
+          # Start TiKV in APIV1
+          touch tiup-v1.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag txnkv --mode tikv-slim --kv 1 --without-monitor --kv.port 30160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2381 2>&1 >> tiup-v1.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1.log)
+          cat tiup-v1.log
+          echo "Wait for bootstrap"
+          sleep 10s
+
+          # Get PD address
+          echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+          echo "TXNKV_PD_ADDRESSES=127.0.0.1:2381" >> $GITHUB_ENV
+
+      - name: Run Integration Test
+        run: mvn clean test
+      - name: Print TiKV logs
+        if: failure()
+        run: |
+          echo "RawKV TiKV logs"
+          cat /home/runner/.tiup/data/rawkv/tikv-0/tikv.log
+          
+          echo "TxnKV TiKV logs"
+          cat /home/runner/.tiup/data/txnkv/tikv-0/tikv.log
+      - name: Upload coverage
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/ci_v2.yml

@@ -0,0 +1,52 @@
+name: CI (APIv2)
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  integration-test:
+    name: Integration Test - ${{ matrix.tikv_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tikv_version: [v6.5.3, v7.1.1, nightly]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8.0'
+          distribution: 'adopt'
+      - name: Install TiUP
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+          /home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
+      - name: Start TiUP Playground
+        run: |
+          # Start TiKV in APIV2
+          touch tiup.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --tag kv --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_v2.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup.log)
+          cat tiup.log
+
+          # Get PD address
+          echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+          echo "TXNKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+
+      - name: Run Integration Test
+        run: mvn clean test
+      - name: Print TiKV logs
+        if: failure()
+        run: |
+          echo "TiKV logs"
+          cat /home/runner/.tiup/data/kv/tikv-0/tikv.log
+      - name: Upload coverage
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/gh-pages.yml

@@ -0,0 +1,38 @@
+name: GitHub Pages
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  deploy:
+    runs-on: ubuntu-20.04
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup mdBook
+        uses: peaceiris/actions-mdbook@v1
+        with:
+          mdbook-version: '0.4.8'
+          # mdbook-version: 'latest'
+
+      - run: mdbook build ./docs
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
+      - name: Maven site
+        run: mvn clean compile site && mv ./target/site ./docs/book/site
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.ref == 'refs/heads/master' }}
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/book

.github/workflows/labeler.yml

@@ -0,0 +1,11 @@
+name: "PR Labeler"
+on:
+- pull_request_target
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v3
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/license-checker.yml

@@ -0,0 +1,24 @@
+name: License checker
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check-license:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check License Header
+        uses: apache/skywalking-eyes@v0.3.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          log: info
+          config: .github/license-checker.yml
+

.github/workflows/pr-title-checker.yml

@@ -0,0 +1,14 @@
+name: 'PR Title Checker'
+on:
+  pull_request:
+    types: [edited, opened, synchronize, reopened]
+
+jobs:
+  title-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: naveenk1223/action-pr-title@master
+        with:
+          regex: '\[to|fix|close #[0-9]+\] .+'
+          prefix_case_sensitive: true
+          max_length: -1

.github/workflows/spotbugs.yml

@@ -0,0 +1,30 @@
+name: Spotbugs
+
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - uses: actions/cache@v1
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Build with Maven
+        run: mvn -B verify spotbugs:spotbugs -Dmaven.test.skip=true
+      - uses: jwgmeligmeyling/spotbugs-github-action@v1
+        with:
+          path: "**/spotbugsXml.xml"

.gitignore

@@ -5,6 +5,7 @@ pub.sh
 
 # ignore compiled classes
 target
+.classpath
 
 # ignore version info
 src/main/java/com/pingcap/tikv/TiVersion.java
@@ -70,3 +71,9 @@ out/
 
 # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898
 # gradle/wrapper/gradle-wrapper.properties
+
+# vscode
+.settings
+
+# mdbook
+docs/book

Makefile

@@ -0,0 +1,16 @@
+.DEFAULT: build
+
+.PHONY: clean
+
+build:
+	mvn clean package -Dmaven.test.skip=true
+
+fmt:
+	./dev/javafmt
+
+test:
+	mvn clean test
+
+clean:
+	mvn clean
+	rm -rf target tipb raft-rs kvproto

README.md

@@ -1,182 +1,89 @@
+[![Maven Central](https://img.shields.io/maven-central/v/org.tikv/tikv-client-java.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22org.tikv%22%20AND%20a:%22tikv-client-java%22)
+[![Slack](https://img.shields.io/badge/chat-on%20Slack-brightgreen.svg)](https://slack.tidb.io/invite?team=tikv-wg&channel=client)
+[![codecov](https://codecov.io/gh/tikv/client-java/branch/master/graph/badge.svg?token=nSAjGaN0EH)](https://codecov.io/gh/tikv/client-java)
+
 ## TiKV JAVA Client
 
-A Java client for [TiDB](https://github.com/pingcap/tidb)/[TiKV](https://github.com/tikv/tikv).
+A Java client for [TiKV](https://github.com/tikv/tikv).
 It is supposed to:
 + Communicate via [gRPC](http://www.grpc.io/)
 + Talk to Placement Driver searching for a region
-+ Talk to TiKV for reading/writing data and the resulted data is encoded/decoded just like what we do in TiDB.
-+ Talk to Coprocessor for calculation pushdown
++ Talk to TiKV for reading/writing data
 
-## How to build
+## Quick Start
 
-### Maven
+> TiKV Java Client is designed to communicate with [PD](https://github.com/tikv/pd) and [TiKV](https://github.com/tikv/tikv), please run PD and TiKV in advance.
 
-The alternative way to build a usable jar for testing will be
+Build Java client from source file:
 
-```
+```sh
 mvn clean install -Dmaven.test.skip=true
 ```
 
-The following command can install dependencies for you.
-
-```
-mvn package
-```
-
-The jar can be found in `./target/`
-
-## Usage
-
-This project is designed to hook with `pd` and `tikv` which you can find in `PingCAP` github page.
-
-When you work with this project, you have to communicate with `pd` and `tikv`. Please run TiKV and PD in advance.
-
-## Component: Raw Ti-Client in Java
-
-Java Implementation of Raw TiKV-Client to support RawKVClient commands.
-
-Demo is avaliable in [KVRawClientTest](https://github.com/birdstorm/KVRawClientTest/)
-
-### Build
-```
-mvn clean install -Dmaven.test.skip=true
-```
-
-### Add to dependency
-
-#### Use jar for binary
-
-Add your jar built with all dependencies into you project's library to use `tikv-client-java` as dependency
-
-#### Use as maven dependency
-
-After building, add following lines into your `pom.xml` if you are using Maven
+Add maven dependency to `pom.xml`:
 
 ```xml
 <dependency>
 	<groupId>org.tikv</groupId>
 	<artifactId>tikv-client-java</artifactId>
-	<version>3.0.0</version>
+	<version>3.3.0</version>
 </dependency>
 ```
 
-### Entrance
-`org.tikv.raw.RawKVClient`
-
-### Create a RawKVClient
+Create a transactional `KVClient` and communicates with TiKV:
 
 ```java
+import org.tikv.common.TiConfiguration;
 import org.tikv.common.TiSession;
-import org.tikv.raw.RawKVClient;
+import org.tikv.txn.KVClient;
 
 public class Main {
-	public static void main() {
-		// You MUST create a raw configuration if you are using RawKVClient.
-		TiConfiguration conf = TiConfiguration.createRawDefault(YOUR_PD_ADDRESSES);
+	public static void main(String[] args) throws Exception {
+		TiConfiguration conf = TiConfiguration.createDefault(YOUR_PD_ADDRESSES);
 		TiSession session = TiSession.create(conf);
-		RawKVClient = session.createRawKVClient();
+		KVClient client = session.createKVClient();
 	}
 }
 ```
 
-### API
+Or create a `RawKVClient` if you don't need the transaction semantic:
 
 ```java
-/**
- * Put a raw key-value pair to TiKV
- *
- * @param key raw key
- * @param value raw value
- */
-void put(ByteString key, ByteString value)
+import org.tikv.common.TiConfiguration;
+import org.tikv.common.TiSession;
+import org.tikv.raw.RawKVClient;
+
+public class Main {
+	public static void main(String[] args) throws Exception {
+		TiConfiguration conf = TiConfiguration.createRawDefault(YOUR_PD_ADDRESSES);
+		TiSession session = TiSession.create(conf);
+		RawKVClient client = session.createRawClient();
+	}
+}
 ```
 
-```java
-/**
- * Get a raw key-value pair from TiKV if key exists
- *
- * @param key raw key
- * @return a ByteString value if key exists, ByteString.EMPTY if key does not exist
- */
-ByteString get(ByteString key)
-```
+Find more demo in [TiKV Java Client User Documents](https://tikv.github.io/client-java/examples/introduction.html)
 
-```java
-/**
- * Scan raw key-value pairs from TiKV in range [startKey, endKey)
- *
- * @param startKey raw start key, inclusive
- * @param endKey raw end key, exclusive
- * @param limit limit of key-value pairs scanned, should be less than {@link #MAX_RAW_SCAN_LIMIT}
- * @return list of key-value pairs in range
- */
-List<Kvrpcpb.KvPair> scan(ByteString startKey, ByteString endKey, int limit)
-```
+## Documentation
 
-```java
-/**
- * Scan raw key-value pairs from TiKV in range [startKey, endKey)
- *
- * @param startKey raw start key, inclusive
- * @param limit limit of key-value pairs scanned, should be less than {@link #MAX_RAW_SCAN_LIMIT}
- * @return list of key-value pairs in range
- */
-List<Kvrpcpb.KvPair> scan(ByteString startKey, int limit)
-```
+See [Java Client Documents](/docs/README.md) for references about how to config and monitor Java Client.
 
-```java
-/**
- * Delete a raw key-value pair from TiKV if key exists
- *
- * @param key raw key to be deleted
- */
-void delete(ByteString key)
-```
+A [Maven site](https://tikv.github.io/client-java/site) is also available. It includes:
+1. [API reference](https://tikv.github.io/client-java/site/apidocs/index.html)
+2. [Spotbugs Reports](https://tikv.github.io/client-java/site/spotbugs.html)
+3. [Source Code Xref](https://tikv.github.io/client-java/site/xref/index.html)
 
-## Java Client 配置参数
+## Community
 
-本文介绍了与部署使用 Java Client 相关的配置参数。
+### Forum
 
-### 常用配置 JVM 参数
+- User forum: [AskTUG](https://asktug.com/)
+- Contributor forum: [https://internals.tidb.io/](https://internals.tidb.io/)
 
-以下包括常用配置的 JVM 相关参数。
-
-####tikv.pd.addresses
-- pd 集群的地址，逗号分隔
-- 默认值 127.0.0.1:2379
-
-####tikv.grpc.timeout_in_ms
-- grpc 请求的 timeout 时间
-- 默认值 600ms
-
-####tikv.grpc.scan_timeout_in_ms
-- scan/delete range grpc 请求的 timeout 时间
-- 默认值 20s
-
-### ThreadPool 配置 JVM 参数
-
-以下包括 ThreadPool 相关的参数及其默认配置，可通过 JVM 参数传入。
-
-####tikv.batch_get_concurrency
-- Client 端 batchGet 请求的线程池大小
-- 默认值 20
-
-####tikv.batch_put_concurrency
-- Client 端 batchPut 请求的线程池大小
-- 默认值 20
-
-####tikv.batch_delete_concurrency
-- Client 端 batchDelete 请求的线程池大小
-- 默认值 20
-
-####tikv.batch_scan_concurrency
-- Client 端 batchScan 请求的线程池大小
-- 默认值 5
-
-####tikv.delete_range_concurrency
-- Client 端 deleteRange 请求的线程池大小
-- 默认值 20
+### Contribute to TiKV Java Client
 
+See [Contribution Guide](https://tikv.github.io/client-java/contribution/introduction.html) for references about how to contribute to this project.
 
 ## License
+
 Apache 2.0 license. See the [LICENSE](./LICENSE) file for details.

dev/README.md

@@ -1,8 +1,6 @@
-# TiSpark Dev Tools Guide
+# TiKV Java Client Dev Tools
 
-## Formatting
-
-### Java Format
+## Code Formatting
 
 TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https://github.com/coveooss/fmt-maven-plugin) which follows Google's code styleguide. It is also checked on CI before build.
 
@@ -18,4 +16,4 @@ TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https:
 
     ```shell script
    ./dev/javafmt
-    ```
+    ```

dev/intellij-java-google-style.xml

@@ -2,222 +2,222 @@
 <code_scheme name="GoogleStyle">
     <option name="OTHER_INDENT_OPTIONS">
         <value>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
-            <option name="TAB_SIZE" value="2" />
-            <option name="USE_TAB_CHARACTER" value="false" />
-            <option name="SMART_TABS" value="false" />
-            <option name="LABEL_INDENT_SIZE" value="0" />
-            <option name="LABEL_INDENT_ABSOLUTE" value="false" />
-            <option name="USE_RELATIVE_INDENTS" value="false" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
+            <option name="TAB_SIZE" value="2"/>
+            <option name="USE_TAB_CHARACTER" value="false"/>
+            <option name="SMART_TABS" value="false"/>
+            <option name="LABEL_INDENT_SIZE" value="0"/>
+            <option name="LABEL_INDENT_ABSOLUTE" value="false"/>
+            <option name="USE_RELATIVE_INDENTS" value="false"/>
         </value>
     </option>
-    <option name="INSERT_INNER_CLASS_IMPORTS" value="true" />
-    <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
-    <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
+    <option name="INSERT_INNER_CLASS_IMPORTS" value="true"/>
+    <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999"/>
+    <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999"/>
     <option name="PACKAGES_TO_USE_IMPORT_ON_DEMAND">
-        <value />
+        <value/>
     </option>
     <option name="IMPORT_LAYOUT_TABLE">
         <value>
-            <package name="" withSubpackages="true" static="true" />
-            <emptyLine />
-            <package name="" withSubpackages="true" static="false" />
+            <package name="" withSubpackages="true" static="true"/>
+            <emptyLine/>
+            <package name="" withSubpackages="true" static="false"/>
         </value>
     </option>
-    <option name="RIGHT_MARGIN" value="100" />
-    <option name="JD_ALIGN_PARAM_COMMENTS" value="false" />
-    <option name="JD_ALIGN_EXCEPTION_COMMENTS" value="false" />
-    <option name="JD_P_AT_EMPTY_LINES" value="false" />
-    <option name="JD_KEEP_EMPTY_PARAMETER" value="false" />
-    <option name="JD_KEEP_EMPTY_EXCEPTION" value="false" />
-    <option name="JD_KEEP_EMPTY_RETURN" value="false" />
-    <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
-    <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="0" />
-    <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
-    <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="0" />
-    <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
-    <option name="ALIGN_MULTILINE_FOR" value="false" />
-    <option name="CALL_PARAMETERS_WRAP" value="1" />
-    <option name="METHOD_PARAMETERS_WRAP" value="1" />
-    <option name="EXTENDS_LIST_WRAP" value="1" />
-    <option name="THROWS_KEYWORD_WRAP" value="1" />
-    <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
-    <option name="BINARY_OPERATION_WRAP" value="1" />
-    <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
-    <option name="TERNARY_OPERATION_WRAP" value="1" />
-    <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
-    <option name="FOR_STATEMENT_WRAP" value="1" />
-    <option name="ARRAY_INITIALIZER_WRAP" value="1" />
-    <option name="WRAP_COMMENTS" value="true" />
-    <option name="IF_BRACE_FORCE" value="3" />
-    <option name="DOWHILE_BRACE_FORCE" value="3" />
-    <option name="WHILE_BRACE_FORCE" value="3" />
-    <option name="FOR_BRACE_FORCE" value="3" />
-    <option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true" />
+    <option name="RIGHT_MARGIN" value="100"/>
+    <option name="JD_ALIGN_PARAM_COMMENTS" value="false"/>
+    <option name="JD_ALIGN_EXCEPTION_COMMENTS" value="false"/>
+    <option name="JD_P_AT_EMPTY_LINES" value="false"/>
+    <option name="JD_KEEP_EMPTY_PARAMETER" value="false"/>
+    <option name="JD_KEEP_EMPTY_EXCEPTION" value="false"/>
+    <option name="JD_KEEP_EMPTY_RETURN" value="false"/>
+    <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false"/>
+    <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="0"/>
+    <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
+    <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="0"/>
+    <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
+    <option name="ALIGN_MULTILINE_FOR" value="false"/>
+    <option name="CALL_PARAMETERS_WRAP" value="1"/>
+    <option name="METHOD_PARAMETERS_WRAP" value="1"/>
+    <option name="EXTENDS_LIST_WRAP" value="1"/>
+    <option name="THROWS_KEYWORD_WRAP" value="1"/>
+    <option name="METHOD_CALL_CHAIN_WRAP" value="1"/>
+    <option name="BINARY_OPERATION_WRAP" value="1"/>
+    <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
+    <option name="TERNARY_OPERATION_WRAP" value="1"/>
+    <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
+    <option name="FOR_STATEMENT_WRAP" value="1"/>
+    <option name="ARRAY_INITIALIZER_WRAP" value="1"/>
+    <option name="WRAP_COMMENTS" value="true"/>
+    <option name="IF_BRACE_FORCE" value="3"/>
+    <option name="DOWHILE_BRACE_FORCE" value="3"/>
+    <option name="WHILE_BRACE_FORCE" value="3"/>
+    <option name="FOR_BRACE_FORCE" value="3"/>
+    <option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true"/>
     <AndroidXmlCodeStyleSettings>
-        <option name="USE_CUSTOM_SETTINGS" value="true" />
+        <option name="USE_CUSTOM_SETTINGS" value="true"/>
         <option name="LAYOUT_SETTINGS">
             <value>
-                <option name="INSERT_BLANK_LINE_BEFORE_TAG" value="false" />
+                <option name="INSERT_BLANK_LINE_BEFORE_TAG" value="false"/>
             </value>
         </option>
     </AndroidXmlCodeStyleSettings>
     <JSCodeStyleSettings>
-        <option name="INDENT_CHAINED_CALLS" value="false" />
+        <option name="INDENT_CHAINED_CALLS" value="false"/>
     </JSCodeStyleSettings>
     <Python>
-        <option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true" />
+        <option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true"/>
     </Python>
     <TypeScriptCodeStyleSettings>
-        <option name="INDENT_CHAINED_CALLS" value="false" />
+        <option name="INDENT_CHAINED_CALLS" value="false"/>
     </TypeScriptCodeStyleSettings>
     <XML>
-        <option name="XML_ALIGN_ATTRIBUTES" value="false" />
-        <option name="XML_LEGACY_SETTINGS_IMPORTED" value="true" />
+        <option name="XML_ALIGN_ATTRIBUTES" value="false"/>
+        <option name="XML_LEGACY_SETTINGS_IMPORTED" value="true"/>
     </XML>
     <codeStyleSettings language="CSS">
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="ECMA Script Level 4">
-        <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
-        <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
-        <option name="ALIGN_MULTILINE_FOR" value="false" />
-        <option name="CALL_PARAMETERS_WRAP" value="1" />
-        <option name="METHOD_PARAMETERS_WRAP" value="1" />
-        <option name="EXTENDS_LIST_WRAP" value="1" />
-        <option name="BINARY_OPERATION_WRAP" value="1" />
-        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
-        <option name="TERNARY_OPERATION_WRAP" value="1" />
-        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
-        <option name="FOR_STATEMENT_WRAP" value="1" />
-        <option name="ARRAY_INITIALIZER_WRAP" value="1" />
-        <option name="IF_BRACE_FORCE" value="3" />
-        <option name="DOWHILE_BRACE_FORCE" value="3" />
-        <option name="WHILE_BRACE_FORCE" value="3" />
-        <option name="FOR_BRACE_FORCE" value="3" />
-        <option name="PARENT_SETTINGS_INSTALLED" value="true" />
+        <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
+        <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
+        <option name="ALIGN_MULTILINE_FOR" value="false"/>
+        <option name="CALL_PARAMETERS_WRAP" value="1"/>
+        <option name="METHOD_PARAMETERS_WRAP" value="1"/>
+        <option name="EXTENDS_LIST_WRAP" value="1"/>
+        <option name="BINARY_OPERATION_WRAP" value="1"/>
+        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
+        <option name="TERNARY_OPERATION_WRAP" value="1"/>
+        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
+        <option name="FOR_STATEMENT_WRAP" value="1"/>
+        <option name="ARRAY_INITIALIZER_WRAP" value="1"/>
+        <option name="IF_BRACE_FORCE" value="3"/>
+        <option name="DOWHILE_BRACE_FORCE" value="3"/>
+        <option name="WHILE_BRACE_FORCE" value="3"/>
+        <option name="FOR_BRACE_FORCE" value="3"/>
+        <option name="PARENT_SETTINGS_INSTALLED" value="true"/>
     </codeStyleSettings>
     <codeStyleSettings language="HTML">
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="JAVA">
-        <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
-        <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
-        <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
-        <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
-        <option name="ALIGN_MULTILINE_RESOURCES" value="false" />
-        <option name="ALIGN_MULTILINE_FOR" value="false" />
-        <option name="CALL_PARAMETERS_WRAP" value="1" />
-        <option name="METHOD_PARAMETERS_WRAP" value="1" />
-        <option name="EXTENDS_LIST_WRAP" value="1" />
-        <option name="THROWS_KEYWORD_WRAP" value="1" />
-        <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
-        <option name="BINARY_OPERATION_WRAP" value="1" />
-        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
-        <option name="TERNARY_OPERATION_WRAP" value="1" />
-        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
-        <option name="FOR_STATEMENT_WRAP" value="1" />
-        <option name="ARRAY_INITIALIZER_WRAP" value="1" />
-        <option name="WRAP_COMMENTS" value="true" />
-        <option name="IF_BRACE_FORCE" value="3" />
-        <option name="DOWHILE_BRACE_FORCE" value="3" />
-        <option name="WHILE_BRACE_FORCE" value="3" />
-        <option name="FOR_BRACE_FORCE" value="3" />
-        <option name="PARENT_SETTINGS_INSTALLED" value="true" />
+        <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false"/>
+        <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
+        <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1"/>
+        <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
+        <option name="ALIGN_MULTILINE_RESOURCES" value="false"/>
+        <option name="ALIGN_MULTILINE_FOR" value="false"/>
+        <option name="CALL_PARAMETERS_WRAP" value="1"/>
+        <option name="METHOD_PARAMETERS_WRAP" value="1"/>
+        <option name="EXTENDS_LIST_WRAP" value="1"/>
+        <option name="THROWS_KEYWORD_WRAP" value="1"/>
+        <option name="METHOD_CALL_CHAIN_WRAP" value="1"/>
+        <option name="BINARY_OPERATION_WRAP" value="1"/>
+        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
+        <option name="TERNARY_OPERATION_WRAP" value="1"/>
+        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
+        <option name="FOR_STATEMENT_WRAP" value="1"/>
+        <option name="ARRAY_INITIALIZER_WRAP" value="1"/>
+        <option name="WRAP_COMMENTS" value="true"/>
+        <option name="IF_BRACE_FORCE" value="3"/>
+        <option name="DOWHILE_BRACE_FORCE" value="3"/>
+        <option name="WHILE_BRACE_FORCE" value="3"/>
+        <option name="FOR_BRACE_FORCE" value="3"/>
+        <option name="PARENT_SETTINGS_INSTALLED" value="true"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="JSON">
         <indentOptions>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="JavaScript">
-        <option name="RIGHT_MARGIN" value="80" />
-        <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
-        <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
-        <option name="ALIGN_MULTILINE_FOR" value="false" />
-        <option name="CALL_PARAMETERS_WRAP" value="1" />
-        <option name="METHOD_PARAMETERS_WRAP" value="1" />
-        <option name="BINARY_OPERATION_WRAP" value="1" />
-        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
-        <option name="TERNARY_OPERATION_WRAP" value="1" />
-        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
-        <option name="FOR_STATEMENT_WRAP" value="1" />
-        <option name="ARRAY_INITIALIZER_WRAP" value="1" />
-        <option name="IF_BRACE_FORCE" value="3" />
-        <option name="DOWHILE_BRACE_FORCE" value="3" />
-        <option name="WHILE_BRACE_FORCE" value="3" />
-        <option name="FOR_BRACE_FORCE" value="3" />
-        <option name="PARENT_SETTINGS_INSTALLED" value="true" />
+        <option name="RIGHT_MARGIN" value="80"/>
+        <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
+        <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
+        <option name="ALIGN_MULTILINE_FOR" value="false"/>
+        <option name="CALL_PARAMETERS_WRAP" value="1"/>
+        <option name="METHOD_PARAMETERS_WRAP" value="1"/>
+        <option name="BINARY_OPERATION_WRAP" value="1"/>
+        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
+        <option name="TERNARY_OPERATION_WRAP" value="1"/>
+        <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
+        <option name="FOR_STATEMENT_WRAP" value="1"/>
+        <option name="ARRAY_INITIALIZER_WRAP" value="1"/>
+        <option name="IF_BRACE_FORCE" value="3"/>
+        <option name="DOWHILE_BRACE_FORCE" value="3"/>
+        <option name="WHILE_BRACE_FORCE" value="3"/>
+        <option name="FOR_BRACE_FORCE" value="3"/>
+        <option name="PARENT_SETTINGS_INSTALLED" value="true"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="PROTO">
-        <option name="RIGHT_MARGIN" value="80" />
+        <option name="RIGHT_MARGIN" value="80"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="2" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="2"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="protobuf">
-        <option name="RIGHT_MARGIN" value="80" />
+        <option name="RIGHT_MARGIN" value="80"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="2" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="2"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="Python">
-        <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
-        <option name="RIGHT_MARGIN" value="80" />
-        <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
-        <option name="PARENT_SETTINGS_INSTALLED" value="true" />
+        <option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
+        <option name="RIGHT_MARGIN" value="80"/>
+        <option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
+        <option name="PARENT_SETTINGS_INSTALLED" value="true"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="SASS">
         <indentOptions>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="SCSS">
         <indentOptions>
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="TypeScript">
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="XML">
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="2" />
-            <option name="TAB_SIZE" value="2" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="2"/>
+            <option name="TAB_SIZE" value="2"/>
         </indentOptions>
         <arrangement>
             <rules>
@@ -226,7 +226,7 @@
                         <match>
                             <AND>
                                 <NAME>xmlns:android</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>^$</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -237,7 +237,7 @@
                         <match>
                             <AND>
                                 <NAME>xmlns:.*</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>^$</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -249,7 +249,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:id</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -260,7 +260,7 @@
                         <match>
                             <AND>
                                 <NAME>style</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>^$</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -271,7 +271,7 @@
                         <match>
                             <AND>
                                 <NAME>.*</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>^$</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -283,7 +283,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:.*Style</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -295,7 +295,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_width</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -306,7 +306,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_height</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -317,7 +317,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_weight</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -328,7 +328,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_margin</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -339,7 +339,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginTop</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -350,7 +350,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginBottom</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -361,7 +361,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginStart</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -372,7 +372,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginEnd</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -383,7 +383,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginLeft</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -394,7 +394,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_marginRight</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -405,7 +405,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:layout_.*</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -417,7 +417,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:padding</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -428,7 +428,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingTop</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -439,7 +439,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingBottom</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -450,7 +450,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingStart</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -461,7 +461,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingEnd</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -472,7 +472,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingLeft</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -483,7 +483,7 @@
                         <match>
                             <AND>
                                 <NAME>.*:paddingRight</NAME>
-                                <XML_ATTRIBUTE />
+                                <XML_ATTRIBUTE/>
                                 <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
                             </AND>
                         </match>
@@ -537,62 +537,62 @@
         </arrangement>
     </codeStyleSettings>
     <Objective-C>
-        <option name="INDENT_NAMESPACE_MEMBERS" value="0" />
-        <option name="INDENT_C_STRUCT_MEMBERS" value="2" />
-        <option name="INDENT_CLASS_MEMBERS" value="2" />
-        <option name="INDENT_VISIBILITY_KEYWORDS" value="1" />
-        <option name="INDENT_INSIDE_CODE_BLOCK" value="2" />
-        <option name="KEEP_STRUCTURES_IN_ONE_LINE" value="true" />
-        <option name="FUNCTION_PARAMETERS_WRAP" value="5" />
-        <option name="FUNCTION_CALL_ARGUMENTS_WRAP" value="5" />
-        <option name="TEMPLATE_CALL_ARGUMENTS_WRAP" value="5" />
-        <option name="TEMPLATE_CALL_ARGUMENTS_ALIGN_MULTILINE" value="true" />
-        <option name="ALIGN_INIT_LIST_IN_COLUMNS" value="false" />
-        <option name="SPACE_BEFORE_SUPERCLASS_COLON" value="false" />
+        <option name="INDENT_NAMESPACE_MEMBERS" value="0"/>
+        <option name="INDENT_C_STRUCT_MEMBERS" value="2"/>
+        <option name="INDENT_CLASS_MEMBERS" value="2"/>
+        <option name="INDENT_VISIBILITY_KEYWORDS" value="1"/>
+        <option name="INDENT_INSIDE_CODE_BLOCK" value="2"/>
+        <option name="KEEP_STRUCTURES_IN_ONE_LINE" value="true"/>
+        <option name="FUNCTION_PARAMETERS_WRAP" value="5"/>
+        <option name="FUNCTION_CALL_ARGUMENTS_WRAP" value="5"/>
+        <option name="TEMPLATE_CALL_ARGUMENTS_WRAP" value="5"/>
+        <option name="TEMPLATE_CALL_ARGUMENTS_ALIGN_MULTILINE" value="true"/>
+        <option name="ALIGN_INIT_LIST_IN_COLUMNS" value="false"/>
+        <option name="SPACE_BEFORE_SUPERCLASS_COLON" value="false"/>
     </Objective-C>
     <Objective-C-extensions>
-        <option name="GENERATE_INSTANCE_VARIABLES_FOR_PROPERTIES" value="ASK" />
-        <option name="RELEASE_STYLE" value="IVAR" />
-        <option name="TYPE_QUALIFIERS_PLACEMENT" value="BEFORE" />
+        <option name="GENERATE_INSTANCE_VARIABLES_FOR_PROPERTIES" value="ASK"/>
+        <option name="RELEASE_STYLE" value="IVAR"/>
+        <option name="TYPE_QUALIFIERS_PLACEMENT" value="BEFORE"/>
         <file>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Import" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Macro" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Typedef" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Enum" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Constant" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Global" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Struct" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="FunctionPredecl" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Function" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Import"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Macro"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Typedef"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Enum"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Constant"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Global"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Struct"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="FunctionPredecl"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Function"/>
         </file>
         <class>
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Property" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Synthesize" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InitMethod" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="StaticMethod" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InstanceMethod" />
-            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="DeallocMethod" />
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Property"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Synthesize"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InitMethod"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="StaticMethod"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InstanceMethod"/>
+            <option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="DeallocMethod"/>
         </class>
         <extensions>
-            <pair source="cc" header="h" />
-            <pair source="c" header="h" />
+            <pair source="cc" header="h"/>
+            <pair source="c" header="h"/>
         </extensions>
     </Objective-C-extensions>
     <codeStyleSettings language="ObjectiveC">
-        <option name="RIGHT_MARGIN" value="80" />
-        <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />
-        <option name="BLANK_LINES_BEFORE_IMPORTS" value="0" />
-        <option name="BLANK_LINES_AFTER_IMPORTS" value="0" />
-        <option name="BLANK_LINES_AROUND_CLASS" value="0" />
-        <option name="BLANK_LINES_AROUND_METHOD" value="0" />
-        <option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="0" />
-        <option name="ALIGN_MULTILINE_BINARY_OPERATION" value="false" />
-        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
-        <option name="FOR_STATEMENT_WRAP" value="1" />
-        <option name="ASSIGNMENT_WRAP" value="1" />
+        <option name="RIGHT_MARGIN" value="80"/>
+        <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1"/>
+        <option name="BLANK_LINES_BEFORE_IMPORTS" value="0"/>
+        <option name="BLANK_LINES_AFTER_IMPORTS" value="0"/>
+        <option name="BLANK_LINES_AROUND_CLASS" value="0"/>
+        <option name="BLANK_LINES_AROUND_METHOD" value="0"/>
+        <option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="0"/>
+        <option name="ALIGN_MULTILINE_BINARY_OPERATION" value="false"/>
+        <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
+        <option name="FOR_STATEMENT_WRAP" value="1"/>
+        <option name="ASSIGNMENT_WRAP" value="1"/>
         <indentOptions>
-            <option name="INDENT_SIZE" value="2" />
-            <option name="CONTINUATION_INDENT_SIZE" value="4" />
+            <option name="INDENT_SIZE" value="2"/>
+            <option name="CONTINUATION_INDENT_SIZE" value="4"/>
         </indentOptions>
     </codeStyleSettings>
 </code_scheme>

dev/javafmt

@@ -2,4 +2,5 @@
 
 echo $MAVEN_HOME
 
-mvn com.coveo:fmt-maven-plugin:format
+mvn com.coveo:fmt-maven-plugin:format
+mvn xml-format:xml-format

dev/proto.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+#   Copyright 2017 PingCAP, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+proto_dir="proto"
+
+if [ -d $proto_dir ]; then
+	rm -r $proto_dir
+fi
+
+repos=("https://github.com/pingcap/kvproto" "https://github.com/pingcap/raft-rs" "https://github.com/pingcap/tipb")
+commits=(3056ca36e6f2a71a9fc7ba7135e6b119fd977553 b9891b673573fad77ebcf9bbe0969cf945841926 c4d518eb1d60c21f05b028b36729e64610346dac)
+
+for i in "${!repos[@]}"; do 
+	repo_name=$(basename ${repos[$i]})
+	git_command="git -C $repo_name"
+
+	if [ -d "$repo_name" ]; then
+		$git_command checkout `basename $($git_command symbolic-ref --short refs/remotes/origin/HEAD)`
+		$git_command fetch --all
+		$git_command pull --all
+	else
+		git clone ${repos[$i]} $repo_name
+		$git_command fetch -p
+	fi
+
+	$git_command checkout ${commits[$i]}
+done

dev/spotbugs-include.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<FindBugsFilter>
+    <Match>
+        <Package name="~org\.tikv\.(?!kvproto).*"/>
+        <Not>
+            <Package name="~org\.tikv\.common\.parser.*"/>
+        </Not>
+        <Not>
+            <Or>
+                <Bug pattern="EI_EXPOSE_REP"/>
+                <Bug pattern="EI_EXPOSE_REP2"/>
+            </Or>
+        </Not>
+        <Rank value="1"/>
+    </Match>
+</FindBugsFilter>

docs/README.md

@@ -0,0 +1,28 @@
+# Client Java Docs
+
+NOTE: See [tikv.github.io/client-java](https://tikv.github.io/client-java/) for
+the rendered developer and contributor documents.
+
+This directory contains all the source of developer and contributor
+documentations in markdown format. The table of content can be found in
+[./src/SUMMARY.md](./src/SUMMARY.md).
+
+## How to contribute to the document
+
+The rendered document is generated by `mdbook`, to get it:
+
+```sh
+cargo install mdbook
+```
+
+Build the source in your development machine:
+
+```sh
+mdbook build
+```
+
+Get more information about how to use `mdbook`:
+
+```sh
+mdbook help
+```

docs/book.toml

@@ -0,0 +1,6 @@
+[book]
+authors = ["Jian Zhang"]
+language = "en"
+multilingual = false
+src = "src"
+title = "TiKV Java Client User Documents"

docs/src/SUMMARY.md

@@ -0,0 +1,33 @@
+# Summary
+
+<!-- Links to empty page is replaced by empty link `() `, so that we can prevent readers to click into these unfinished page. Once the corresponding page is done, we can delete the empty parenthesis and then the pages will be visible. -->
+
+- [Introduction](./introduction/introduction.md)
+
+- [Production Readiness](./production-readiness.md)
+
+- [Start With Examples](./examples/introduction.md)
+    - [Quick Start](./examples/quick-start.md)
+    - [Interact with TiKV RawKV API](./examples/rawkv.md)
+    - [Interact with TiKV TxnKV API](./examples/txnkv.md)
+    - [TiKV RawKV Bulk Load]() (./examples/bulk-load.md)
+
+- [Performance]() (./performance/introduction.md)
+    - [YCSB Benchmarks]() (./performance/ycsb.md)
+
+- [Administration](./administration/introduction.md)
+    - [Configuration](./administration/configuration.md)
+    - [Monitoring](./administration/monitoring.md)
+
+- [Troubleshooting](./troubleshooting/introduction.md)
+    - [Slow Request Diagnosis](./troubleshooting/slow-request.md)
+    - [Error Request Diagnosis]() (./troubleshooting/error-request.md)
+
+- [Architecture and Internals](./architecture/introduction.md)
+    - [The Lifecycle of A Request](./architecture/request-lifecycle.md)
+    - [Backoff and Retry Policy](./architecture/availability.md)
+    - [Slow Log and Metrics]() (./architecture/observability.md)
+    - [Region Cache]() (./architecture/region-cache.md)
+
+- [Contribution Guide](./contribution/introduction.md)
+    - [Bug Severity Guidelines](./contribution/bug-severity-guide.md)

docs/src/administration/configuration.md

@@ -0,0 +1,121 @@
+## Java Client Configuration Parameter
+
+### JVM Parameter
+
+The following includes JVM related parameters.
+
+#### tikv.pd.addresses
+- pd addresses, separated by comma
+- default: 127.0.0.1:2379
+
+#### tikv.grpc.timeout_in_ms
+- timeout of grpc request  
+- default: 600ms
+
+#### tikv.grpc.scan_timeout_in_ms
+- timeout of scan/delete range grpc request
+- default: 20s
+
+#### tikv.importer.max_kv_batch_bytes
+- Maximal package size transporting from clients to TiKV Server (ingest API)
+- default: 1048576 (1M)
+
+#### tikv.importer.max_kv_batch_size
+- Maximal batch size transporting from clients to TiKV Server (ingest API)
+- default: 32768 (32K)
+
+#### tikv.scatter_wait_seconds
+- time to wait for scattering regions
+- default: 300 (5min)
+
+#### tikv.rawkv.default_backoff_in_ms
+- RawKV default backoff in milliseconds
+- default: 20000 (20 seconds)
+
+### Metrics Parameter
+
+#### tikv.metrics.enable
+- whether to enable metrics exporting
+- default: false
+
+#### tikv.metrics.port
+- the metrics exporting http port
+- default: 3140
+
+### ThreadPool Parameter
+
+The following includes ThreadPool related parameters, which can be passed in through JVM parameters.
+
+#### tikv.batch_get_concurrency
+- the thread pool size of batchGet on client side
+- default: 20
+
+#### tikv.batch_put_concurrency
+- the thread pool size of batchPut on client side
+- default: 20
+
+#### tikv.batch_delete_concurrency
+- the thread pool size of batchDelete on client side
+- default: 20
+
+#### tikv.batch_scan_concurrency
+- the thread pool size of batchScan on client side
+- default: 5
+
+#### tikv.delete_range_concurrency
+- the thread pool size of deleteRange on client side
+- default: 20
+
+#### tikv.enable_atomic_for_cas
+- whether to enable `Compare And Set`, set true if using `RawKVClient.compareAndSet` or `RawKVClient.putIfAbsent`
+- default: false
+
+### TLS
+
+#### tikv.tls_enable
+- whether to enable TLS
+- default: false
+
+#### tikv.trust_cert_collection
+- Trusted certificates for verifying the remote endpoint's certificate, e.g. /home/tidb/ca.pem. The file should contain an X.509 certificate collection in PEM format.
+- default: null
+
+#### tikv.key_cert_chain
+- an X.509 certificate chain file in PEM format, e.g. /home/tidb/client.pem.
+- default: null
+
+#### tikv.key_file
+- a PKCS#8 private key file in PEM format. e.g. /home/tidb/client-key.pem.
+- default: null
+
+#### tikv.tls.reload_interval
+- The interval in seconds to poll the change of TLS context, if a change is detected, the TLS context will be rebuilded.
+- default: `"10s"`, `"0s"` means disable TLS context reload.
+
+#### tikv.conn.recycle_time
+- After a TLS context reloading, the old connections will be forced to shutdown after `tikv.conn.recycle_time` to prevent channel leak.
+- default: `"60s"`.
+
+#### tikv.rawkv.read_timeout_in_ms
+- RawKV read timeout in milliseconds. This parameter controls the timeout of `get` `getKeyTTL`.
+- default: 2000 (2 seconds)
+
+#### tikv.rawkv.write_timeout_in_ms
+- RawKV write timeout in milliseconds. This parameter controls the timeout of `put` `putAtomic` `putIfAbsent` `delete` `deleteAtomic`.
+- default: 2000 (2 seconds)
+
+#### tikv.rawkv.batch_read_timeout_in_ms
+- RawKV batch read timeout in milliseconds. This parameter controls the timeout of `batchGet`.
+- default: 2000 (2 seconds)
+
+#### tikv.rawkv.batch_write_timeout_in_ms
+- RawKV batch write timeout in milliseconds. This parameter controls the timeout of `batchPut` `batchDelete` `batchDeleteAtomic`.
+- default: 2000 (2 seconds)
+
+#### tikv.rawkv.scan_timeout_in_ms
+- RawKV scan timeout in milliseconds. This parameter controls the timeout of `batchScan` `scan` `scanPrefix`.
+- default: 10000 (10 seconds)
+
+#### tikv.rawkv.clean_timeout_in_ms
+- RawKV clean timeout in milliseconds. This parameter controls the timeout of `deleteRange` `deletePrefix`.
+- default: 600000 (10 minutes)

docs/src/administration/introduction.md

@@ -0,0 +1 @@
+# Administration

docs/src/administration/monitoring.md

@@ -0,0 +1,33 @@
+## Java Client Metrics
+
+Client Java supports exporting metrics to Prometheus using poll mode and viewing on Grafana. The following steps shows how to enable this function.
+
+### Step 1: Enable metrics exporting
+
+- set the config `tikv.metrics.enable` to `true`
+- call TiConfiguration.setMetricsEnable(true)
+
+### Step 2: Set the metrics port
+
+- set the config `tikv.metrics.port`
+- call TiConfiguration.setMetricsPort
+
+Default port is 3140.
+
+### Step 3: Config Prometheus
+
+Add the following config to `conf/prometheus.yml` and restart Prometheus.
+
+```yaml
+- job_name: "tikv-client"
+    honor_labels: true
+    static_configs:
+    - targets:
+        - '127.0.0.1:3140'
+        - '127.0.0.2:3140'
+        - '127.0.0.3:3140'
+```
+
+### Step 4: Config Grafana
+
+Import the [Client-Java-Summary dashboard config](/metrics/grafana/client_java_summary.json) to Grafana.

docs/src/architecture/availability.md

@@ -0,0 +1,47 @@
+# Availability: Backoff and Retry Policy
+
+## BackOffer
+
+The retry and timeout mechanism for a request is controlled by a `BackOffer` object, which is created one per `RawKVClient` method. The `BackOffer` will decide how much time the next sleep and retry should spend, and whether to timeout the request if not enough time is left for retrying the request.
+If we need a back off sleep, we call backOffer.doBackOff(funcType, exception), and the current thread will sleep for a decided time. If the current operation will timeout after sleep, the doBackOff simply throw an exception to abort the operation.
+
+## callWithRetry
+
+RegionStoreClient.callWithRetry inherits from AbstractGRPCClient.callWithRetry. The concrete logic is in RetryPolicy.callWithRetry, which implements a retry mechanism, but the specific retry strategy is determined by the ErrorHandler.
+ErrorHandler’s handler{Request, Response}Error function returns a boolean value indicating whether to retry inside callWithRetry.
+The control flow for callWithRetry is as follows:
+
+![callWithRetry](./callWithRetry.jpg)
+
+The error handler is chosen obeying the following table:
+
+| gPRC request	| the result	| handler |
+| -- | -- | -- |
+| throws exception	| -	| handleRequestError
+| no exception	| is null	| handleRequestError
+| no exception	| is error	| handleResponseError
+| no exception	| normal	| normal return
+
+The handleRequestError function copes with the following situations:
+
+| situation  | retry within callWithRetry | note   |
+|----------|---------------|------------------------------|
+| invalid store in region manager  | true  | refresh ClientStub  |
+| region has not got multiple copies  | false | |
+| successfully switched to new leader | true | |
+| seekProxyStore | true if success | only when `tikv.enable_grpc_forward` is set |
+| other | false | |
+
+The handleResponseError function copes with the following gRPC errors:
+
+| error                | retry within callWithRetry |
+|----------------------|----------------------------|
+| NotLeader            | true if leader unchanged   |
+| StoreNotMatch        | false                      |
+| EphochNotMatch       | true if region epoch in `ctx` is ahead of TiKV's |
+| ServerIsBusy         | true                       |
+| StaleCommand         | true                       |
+| RaftEntryTooLarge    | throw                      |
+| KeyNotInRegion       | throw                      |
+| Raft ProposalDropped | true                       |
+| other                | false                      |

docs/src/architecture/introduction.md

@@ -0,0 +1,3 @@
+# Architecture
+
+This section includes in-depthA description of the client architecture.

docs/src/architecture/observability.md

@@ -0,0 +1 @@
+# Observability: Slow Log and Metrics

docs/src/architecture/region-cache.md

@@ -0,0 +1 @@
+# Region Cache

docs/src/architecture/request-lifecycle.md

@@ -0,0 +1,7 @@
+# The Lifecycle of A Request
+
+![time graph](./timegraph.png)
+
+The client talks to TiKV store directly using gRPC requests, which are created in RegionStoreClient. If a request failed, the client could retry after a back off sleep. The retry logic is delegated to AbstractGRPCClient::callWithRetry method. callWithRetry may decide to retry request within the function, or, if the RegionStoreClient must be recreated (due to, for example, region split), return a failure and let outermost RawKVClient to do the retry.
+
+![request-overview](./request-overview.jpg)

docs/src/contribution/bug-severity-guide.md

@@ -0,0 +1,49 @@
+## Bug Severity Guidelines
+
+This is a **working-in-progress** guide about determining defects severity on
+TiKV Java Client according to the impact on the online service. The higher
+effect the defect has on the overall functionality or performance, the higher
+the severity is. There are 4 severity levels:
+
+1. Critical
+2. Major
+3. Moderate
+4. Minor
+
+Each severity is described with examples in the remaining contents.
+
+### Critical Defects
+
+A defect that affects critical data or functionality and leaves users
+with no workaround is classified as a critical defect. These defects are
+labeled with `type/bug` and `severity/critical`, can be found
+[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fcritical)
+
+Guideline 1. A defect that breaks the API definition is regarded as critical.
+For example:
+
+* [client-java/issues/412](https://github.com/tikv/client-java/issues/412)
+in this defect, gRPC timeout is not set for certain requests, which causes the
+requests can not be terminated as expected when the processing time is too long.
+
+### Major Defects
+
+A defect that affects critical data or functionality and forces users to employ
+a workaround is classified as a major defect. These defects are labeled with
+`type/bug` and `severity/major`, can be found
+[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fmajor)
+
+### Moderate Defects
+
+A defect that affects non-critical data or functionality and forces users to
+employ a workaround is classified as moderate defect. These defects are labeled
+with `type/bug` and `severity/moderate`, can be found
+[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fmoderate)
+
+### Minor Defects
+
+A defect that does not affect data or functionality. It does not even need a
+workaround. It does not impact productivity or efficiency. It is merely an
+inconvenience. These defects are labeled with `type/bug` and `severity/minor`,
+can be found
+[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fminor)

docs/src/contribution/introduction.md

@@ -0,0 +1,21 @@
+# Contribution Guide
+
+## Build the package
+
+```
+mvn clean package -Dmaven.test.skip=true
+```
+
+## Install the package to local maven repository
+
+```
+mvn clean install -Dmaven.test.skip=true
+```
+
+## Run tests
+
+```
+export RAWKV_PD_ADDRESSES=127.0.0.1:2379
+export TXNKV_PD_ADDRESSES=127.0.0.1:2379
+mvn clean test
+```

docs/src/examples/introduction.md

@@ -0,0 +1,3 @@
+# Start With Examples
+
+This section contains examples to demonstrate basic usages of the java client.

docs/src/examples/quick-start.md

@@ -0,0 +1,109 @@
+# Quick Start
+
+The package is hosted on maven central repository. To build from source, refer to the [Contribution Guide](../contribution/introduction.html).
+
+## Create a maven project
+
+First download [maven] and follow the [installation instructoins][install]. Then `mvn` command should be available in the `$PATH`.
+
+[maven]: https://maven.apache.org/download.html
+[install]: https://maven.apache.org/install.html
+
+create a maven project by following command:
+
+```
+mvn archetype:generate -DgroupId=com.example -DartifactId=java-client-example -DarchetypeArtifactId=maven-archetype-quickstart -DinteractiveMode=false
+cd java-client-example
+```
+
+## Add dependency
+
+Add maven dependency to `pom.xml`.
+
+```xml
+<dependency>
+  <groupId>org.tikv</groupId>
+  <artifactId>tikv-client-java</artifactId>
+  <version>3.3.0</version>
+</dependency>
+<dependency>
+  <groupId>org.slf4j</groupId>
+  <artifactId>slf4j-log4j12</artifactId>
+  <version>1.7.32</version>
+</dependency>
+```
+
+Now `pom.xml` should look like this:
+
+```xml
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.example</groupId>
+  <artifactId>java-project</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>java-project</name>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.12</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.tikv</groupId>
+      <artifactId>tikv-client-java</artifactId>
+      <version>3.1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>1.7.32</version>
+    </dependency>
+  </dependencies>
+</project>
+```
+
+## Writing code
+
+To interact with TiKV, we should first create a `TiConfiguration` with PD address, create a `TiSession` using `TiSession.create`, and then create a client.
+For example, if we want to put a `World` in `Hello` key in RawKV, write the following code in `src/main/java/com/example/App.java`.
+
+```java
+import org.tikv.common.TiConfiguration;
+import org.tikv.common.TiSession;
+import org.tikv.raw.RawKVClient;
+import org.tikv.shade.com.google.protobuf.ByteString;
+
+public class App {
+  public static void main(String[] args) throws Exception {
+    String pdAddr = "127.0.0.1:2379";
+    // You MUST create a raw configuration if you are using RawKVClient.
+    TiConfiguration conf = TiConfiguration.createRawDefault(pdAddr);
+    try (TiSession session = TiSession.create(conf)) {
+      try (RawKVClient client = session.createRawClient()) {
+        client.put(ByteString.copyFromUtf8("Hello"), ByteString.copyFromUtf8("World"));
+        ByteString value = client.get(ByteString.copyFromUtf8("Hello"));
+        System.out.println(value);
+      }
+    }
+  }
+}
+```
+
+More examples for RawKV and TxnKV are in following chapters.
+
+## Running program
+
+Run following command:
+
+```
+mvn assembly:assembly -DdescriptorId=jar-with-dependencies
+java -cp target/java-client-example-1.0-SNAPSHOT-jar-with-dependencies.jar com.example.App
+```

docs/src/examples/rawkv.md

@@ -0,0 +1,100 @@
+# RawKV
+
+Below is the basic usages of RawKV. See [API document] to see a full list of methods available.
+
+[API document]: https://tikv.github.io/client-java/apidocs/org/tikv/raw/RawKVClient
+
+```java
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import org.tikv.common.TiConfiguration;
+import org.tikv.common.TiSession;
+import org.tikv.kvproto.Kvrpcpb;
+import org.tikv.raw.RawKVClient;
+import org.tikv.shade.com.google.protobuf.ByteString;
+
+public class Main {
+  public static void main(String[] args) throws Exception {
+    // You MUST create a raw configuration if you are using RawKVClient.
+    TiConfiguration conf = TiConfiguration.createRawDefault("127.0.0.1:2379");
+    TiSession session = TiSession.create(conf);
+    RawKVClient client = session.createRawClient();
+
+    // put
+    client.put(ByteString.copyFromUtf8("k1"), ByteString.copyFromUtf8("Hello"));
+    client.put(ByteString.copyFromUtf8("k2"), ByteString.copyFromUtf8(","));
+    client.put(ByteString.copyFromUtf8("k3"), ByteString.copyFromUtf8("World"));
+    client.put(ByteString.copyFromUtf8("k4"), ByteString.copyFromUtf8("!"));
+    client.put(ByteString.copyFromUtf8("k5"), ByteString.copyFromUtf8("Raw KV"));
+
+    // get
+    Optional<ByteString> result = client.get(ByteString.copyFromUtf8("k1"));
+    System.out.println(result.get().toStringUtf8());
+
+    // batch get
+    List<Kvrpcpb.KvPair> list = client.batchGet(new ArrayList<ByteString>() {{
+        add(ByteString.copyFromUtf8("k1"));
+        add(ByteString.copyFromUtf8("k3"));
+    }});
+    System.out.println(list);
+
+    // scan
+    list = client.scan(ByteString.copyFromUtf8("k1"), ByteString.copyFromUtf8("k6"), 10);
+    System.out.println(list);
+
+    // close
+    client.close();
+    session.close();
+  }
+}
+```
+
+
+## API V2
+With TiKV version >= 6.1.0, we release a new feature called "TiKV API V2" which provides a new raw key-value storage format allowing the coexistence of RawKV and TxnKV. Please refer to [v6.10 release notes](https://docs.pingcap.com/tidb/stable/release-6.1.0#ease-of-use) for detail.
+
+To enable the API V2 mode, users need to specify the API version of the client.
+
+```java
+// import ...
+import org.tikv.common.TiConfiguration.ApiVersion;
+
+public class Main {
+  public static void main(String[] args) throws Exception {
+    TiConfiguration conf = TiConfiguration.createRawDefault("127.0.0.1:2379");
+    conf.setApiVersion(ApiVersion.V2);
+    try(TiSession session = TiSession.create(conf)) {
+      try(RawKVClient client = session.createRawClient()) {
+        // The client will read and write date in the format of API V2, which is
+        // transparent to the users.
+        client.put(ByteString.copyFromUtf8("hello"), ByteString.copyFromUtf8("world"));
+        // other client operations.
+      }
+    }
+  }
+}
+```
+
+### Compatibility
+
+The V2 Client should not access the cluster other than V2, this requires users to [enable the API V2](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#api-version-new-in-v610) for the cluster:
+
+```toml
+[storage]
+# The V2 cluster must enable ttl for RawKV explicitly
+enable-ttl = true
+api-version = 2
+```
+
+If V2 client accesses a V1 cluster or V1 cluster accesses a V2 cluster, the requests will be denied by the cluster. You can check the compatibility via the following matrix.
+
+
+|                       | V1 Server | V1TTL Server | V2 Server |
+| --------------------- | --------- | ------------ | --------- |
+| V1 RawClient          | Raw       | Raw          | Error     |
+| V1 RawClient with TTL | Error     | Raw          | Error     |
+| V1 TxnClient          | Txn       | Error        | Error     |
+| V1 TiDB               | TiDB Data | Error        | TiDB Data |
+| V2 RawClient          | Error     | Error        | Raw       |
+| V2 TxnClient          | Error     | Error        | Txn       |

docs/src/examples/txnkv.md

@@ -0,0 +1,69 @@
+# TxnKV
+
+Below is the basic usages of TxnKV.
+Data should be written into TxnKV using [`TwoPhaseCommitter`], and be read using [`org.tikv.txn.KVClient`][KVClient].
+
+[`TwoPhaseCommitter`]: https://tikv.github.io/client-java/apidocs/org/tikv/txn/TwoPhaseCommitter.html
+[KVClient]: https://tikv.github.io/client-java/apidocs/org/tikv/txn/KVClient.html
+
+```java
+import java.util.Arrays;
+import java.util.List;
+
+import org.tikv.common.BytePairWrapper;
+import org.tikv.common.ByteWrapper;
+import org.tikv.common.TiConfiguration;
+import org.tikv.common.TiSession;
+import org.tikv.common.util.BackOffer;
+import org.tikv.common.util.ConcreteBackOffer;
+import org.tikv.kvproto.Kvrpcpb.KvPair;
+import org.tikv.shade.com.google.protobuf.ByteString;
+import org.tikv.txn.KVClient;
+import org.tikv.txn.TwoPhaseCommitter;
+
+public class App {
+    public static void main(String[] args) throws Exception {
+        TiConfiguration conf = TiConfiguration.createDefault("127.0.0.1:2379");
+        try (TiSession session = TiSession.create(conf)) {
+            // two-phrase write
+            long startTS = session.getTimestamp().getVersion();
+            try (TwoPhaseCommitter twoPhaseCommitter = new TwoPhaseCommitter(session, startTS)) {
+                BackOffer backOffer = ConcreteBackOffer.newCustomBackOff(1000);
+                byte[] primaryKey = "key1".getBytes("UTF-8");
+                byte[] key2 = "key2".getBytes("UTF-8");
+
+                // first phrase: prewrite
+                twoPhaseCommitter.prewritePrimaryKey(backOffer, primaryKey, "val1".getBytes("UTF-8"));
+                List<BytePairWrapper> pairs = Arrays
+                        .asList(new BytePairWrapper(key2, "val2".getBytes("UTF-8")));
+                twoPhaseCommitter.prewriteSecondaryKeys(primaryKey, pairs.iterator(), 1000);
+
+                // second phrase: commit
+                long commitTS = session.getTimestamp().getVersion();
+                twoPhaseCommitter.commitPrimaryKey(backOffer, primaryKey, commitTS);
+                List<ByteWrapper> keys = Arrays.asList(new ByteWrapper(key2));
+                twoPhaseCommitter.commitSecondaryKeys(keys.iterator(), commitTS, 1000);
+            }
+
+            try (KVClient kvClient = session.createKVClient()) {
+                long version = session.getTimestamp().getVersion();
+                ByteString key1 = ByteString.copyFromUtf8("key1");
+                ByteString key2 = ByteString.copyFromUtf8("key2");
+
+                // get value of a single key
+                ByteString val = kvClient.get(key1, version);
+                System.out.println(val);
+
+                // get value of multiple keys
+                BackOffer backOffer = ConcreteBackOffer.newCustomBackOff(1000);
+                List<KvPair> kvPairs = kvClient.batchGet(backOffer, Arrays.asList(key1, key2), version);
+                System.out.println(kvPairs);
+
+                // get value of a range of keys
+                kvPairs = kvClient.scan(key1, ByteString.copyFromUtf8("key3"), version);
+                System.out.println(kvPairs);
+            }
+        }
+    }
+}
+```

docs/src/introduction/introduction.md

@@ -0,0 +1,15 @@
+# Introduction
+
+Welcome to the documents of TiKV Java Client. This document is oriented to:
+
+1. Application developers who needs to integrate java client to their
+   application code. There are documents about best practices, administrations,
+   troubleshootings, and internals of TiKV Java Client for developers.
+
+2. TiKV contributors who wish to contribute to TiKV Java Client, become a TiKV
+   reviewer, committer, or maintainer. There are documents about setting up
+   environment, submitting PR, triage issues, and manage releases in the
+   community.
+
+Wish you have a good journey in developing application or contributing to the
+TiKV Java Client.

docs/src/performance/introduction.md

@@ -0,0 +1 @@
+# Performance

docs/src/performance/ycsb.md

@@ -0,0 +1 @@
+# YCSB Benchmarks

docs/src/production-readiness.md

@@ -0,0 +1,18 @@
+# Production Readiness
+
+In general, the latest [release](https://github.com/tikv/client-java/releases) of TiKV Java Client is ready for production use. But it is not battle-tested as full featured client for TiKV in all use cases. This page will give you more details.
+
+## RawKV
+All RawKV APIs are covered by [CI](https://github.com/tikv/client-java/actions/workflows/ci.yml).
+
+At this time, RawKV has been used in the production environment of some commercial customers in latency sensitive systems. But they only use part of the RawKV APIs (mainly including `raw_put`, `raw_get`, `raw_compare_and_swap`, and `raw_batch_put`).
+
+## TxnKV
+All TxnKV APIs are covered by [CI](https://github.com/tikv/client-java/actions/workflows/ci.yml).
+
+In addition, TxnKV has been used in the [TiSpark](https://docs.pingcap.com/tidb/stable/tispark-overview) and [TiBigData](https://github.com/tidb-incubator/TiBigData) project to integrate data from TiDB to Big Data ecosystem. TiSpark and TiBigData are used in the production system of some commercial customers and internet companies.
+
+Similar to RawKV, only part of APIs are used in this scenario (mainly including `prewrite/commit` and `coprocessor`). And this use case doesn't care about latency but throughput and reliability.
+
+## TiDB Cloud
+Directly using TiKV is not possible on TiDB Cloud due to the fact that client has to access the whole cluster, which has security issues. And TiKV managed service is not coming soon as it's not contained in [roadmap](https://docs.pingcap.com/tidbcloud/tidb-cloud-roadmap) yet.

docs/src/troubleshooting/error-request.md

@@ -0,0 +1 @@
+# Error Request Diagnosis

docs/src/troubleshooting/introduction.md

@@ -0,0 +1 @@
+# Troubleshooting

docs/src/troubleshooting/slow-request.md

@@ -0,0 +1,34 @@
+# Slow Request Diagnosis
+
+If a request take too much time, we can collect the detailed time spend in each component in a “slow log”.
+
+<!-- wrap text in the code block -->
+<pre>
+<code class="hljs" style="white-space: pre-wrap;">
+2022-02-11 11:07:56 WARN  SlowLogImpl:88 - A request spent 55 ms. start=11:07:56.938, end=11:07:56.993, SlowLog:{"trace_id":4361090673996453790,"spans":[{"event":"put","begin":"11:07:56.938","duration_ms":55,"properties":{"region":"{Region[2] ConfVer[5] Version[60] Store[1] KeyRange[]:[]}","key":"Hello"}},{"event":"getRegionByKey","begin":"11:07:56.938","duration_ms":0},{"event":"callWithRetry","begin":"11:07:56.943","duration_ms":49,"properties":{"method":"tikvpb.Tikv/RawPut"}},{"event":"gRPC","begin":"11:07:56.943","duration_ms":49,"properties":{"method":"tikvpb.Tikv/RawPut"}}]}
+</code>
+</pre>
+
+## Slow log configurations
+
+| SlowLog settings | default value |
+| -- | -- |
+| tikv.rawkv.read_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
+| tikv.rawkv.write_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
+| tikv.rawkv.batch_read_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
+| tikv.rawkv.batch_write_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
+| tikv.rawkv.scan_slowlog_in_ms | 5s |
+
+Each settings can be set by system properties, configuration files or `set...` method of `TiConfiguration`.
+
+System properties can be set by `-D` parameter of `java` command.
+
+```
+java -cp target/java-client-example-1.0-SNAPSHOT-jar-with-dependencies.jar -Dtikv.rawkv.read_slowlog_in_ms=100 com.example.App
+```
+
+Configuration file is `src/main/resources/tikv.properties` in maven projects.
+
+## Visualize slow log
+
+TBD

pom.xml

@@ -1,16 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
-
     <groupId>org.tikv</groupId>
     <artifactId>tikv-client-java</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.3.4-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>TiKV Java Client</name>
     <description>A Java Client for TiKV</description>
     <url>http://github.com/tikv/client-java</url>
-
     <licenses>
         <license>
             <name>Apache 2.0 License</name>
@@ -18,11 +15,9 @@
             <distribution>repo</distribution>
         </license>
     </licenses>
-
     <organization>
         <name>PingCAP</name>
     </organization>
-
     <developers>
         <developer>
             <name>Xiaoyu Ma</name>
@@ -49,33 +44,54 @@
             <organizationUrl>https://www.pingcap.com</organizationUrl>
         </developer>
     </developers>
-
     <scm>
         <connection>scm:git:git://github.com/tikv/client-java.git</connection>
         <developerConnection>scm:git:ssh://github.com:tikv/client-java.git</developerConnection>
         <url>https://github.com/tikv/client-java/tree/master</url>
     </scm>
-
     <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <protobuf.version>3.5.1</protobuf.version>
+        <protobuf.version>3.18.0</protobuf.version>
         <log4j.version>1.2.17</log4j.version>
         <slf4j.version>1.7.16</slf4j.version>
-        <grpc.version>1.24.0</grpc.version>
+        <grpc.version>1.60.0</grpc.version>
+        <netty.tcnative.version>2.0.34.Final</netty.tcnative.version>
+        <gson.version>2.8.9</gson.version>
         <powermock.version>1.6.6</powermock.version>
-        <jackson.version>2.10.0</jackson.version>
+        <jackson-annotations.version>2.13.2</jackson-annotations.version>
+        <jackson.version>2.13.4.2</jackson.version>
         <trove4j.version>3.0.1</trove4j.version>
-        <jetcd.version>0.4.1</jetcd.version>
+        <jetcd.version>0.7.7</jetcd.version>
         <joda-time.version>2.9.9</joda-time.version>
         <joda-convert.version>1.9.2</joda-convert.version>
         <proto.folder>${basedir}/proto</proto.folder>
         <gpg.keyname>fake gpg key name</gpg.keyname>
         <gpg.skip>true</gpg.skip>
-        <javadoc.skip>true</javadoc.skip>
     </properties>
-
     <dependencies>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.19.6</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java-util</artifactId>
+            <version>3.19.6</version>
+        </dependency>
+        <dependency>
+            <groupId>io.perfmark</groupId>
+            <artifactId>perfmark-api</artifactId>
+            <version>0.24.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.perfmark</groupId>
+            <artifactId>perfmark-traceviewer</artifactId>
+            <version>0.24.0</version>
+        </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
@@ -120,22 +136,49 @@
             <groupId>io.grpc</groupId>
             <artifactId>grpc-protobuf</artifactId>
             <version>${grpc.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-stub</artifactId>
             <version>${grpc.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-services</artifactId>
+            <version>${grpc.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java-util</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-tcnative-boringssl-static</artifactId>
+            <version>${netty.tcnative.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-testing</artifactId>
             <version>${grpc.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>${gson.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
-            <version>${jackson.version}</version>
+            <version>${jackson-annotations.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
@@ -145,20 +188,6 @@
         <dependency>
             <groupId>io.etcd</groupId>
             <artifactId>jetcd-core</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>io.etcd</groupId>
-                    <artifactId>jetcd-resolver</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>io.etcd</groupId>
-                    <artifactId>jetcd-common</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>io.grpc</groupId>
-                    <artifactId>grpc-grpclb</artifactId>
-                </exclusion>
-            </exclusions>
             <version>${jetcd.version}</version>
         </dependency>
         <dependency>
@@ -180,7 +209,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.9</version>
+            <version>3.10</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -189,6 +218,16 @@
             <version>3.9</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.15</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.13</version>
+        </dependency>
         <dependency>
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient</artifactId>
@@ -213,6 +252,31 @@
             <version>0.10.0</version>
         </dependency>
     </dependencies>
+    <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>3.3.1</version>
+            </plugin>
+            <plugin>
+                <groupId>com.github.spotbugs</groupId>
+                <artifactId>spotbugs-maven-plugin</artifactId>
+                <version>4.5.2.0</version>
+                <configuration>
+                    <xmlOutput>true</xmlOutput>
+                    <!-- Optional directory to put spotbugs xdoc xml report -->
+                    <xmlOutputDirectory>target/site</xmlOutputDirectory>
+                    <includeFilterFile>dev/spotbugs-include.xml</includeFilterFile>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jxr-plugin</artifactId>
+                <version>3.1.1</version>
+            </plugin>
+        </plugins>
+    </reporting>
     <build>
         <resources>
             <resource>
@@ -256,7 +320,7 @@
                     <execution>
                         <id>clone proto files</id>
                         <configuration>
-                            <executable>${basedir}/scripts/proto.sh</executable>
+                            <executable>${basedir}/dev/proto.sh</executable>
                         </configuration>
                         <phase>validate</phase>
                         <goals>
@@ -307,7 +371,6 @@
                     </execution>
                 </executions>
             </plugin>
-
             <plugin>
                 <groupId>org.xolstice.maven.plugins</groupId>
                 <artifactId>protobuf-maven-plugin</artifactId>
@@ -330,7 +393,6 @@
                     </execution>
                 </executions>
             </plugin>
-
             <!-- Compiler Plug-in -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -382,26 +444,6 @@
                     </filesets>
                 </configuration>
             </plugin>
-            <!-- Javadoc Plug-in -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
-                <configuration>
-                    <skip>${javadoc.skip}</skip>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>attach-javadocs</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                        <configuration> <!-- add this to disable checking -->
-                            <additionalparam>-Xdoclint:none</additionalparam>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
             <!--- Needs to shade Protobuf 3 since other projects might use other version -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -438,6 +480,10 @@
                                     <pattern>io.prometheus</pattern>
                                     <shadedPattern>org.tikv.shade.io.prometheus</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <pattern>com.fasterxml.jackson</pattern>
+                                    <shadedPattern>org.tikv.shade.com.fasterxml.jackson</shadedPattern>
+                                </relocation>
                             </relocations>
                         </configuration>
                     </execution>
@@ -457,7 +503,7 @@
                     </execution>
                     <execution>
                         <id>jacoco-site</id>
-                        <phase>package</phase>
+                        <phase>test</phase>
                         <goals>
                             <goal>report</goal>
                         </goals>
@@ -514,9 +560,110 @@
                 <configuration>
                     <serverId>ossrh</serverId>
                     <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-                    <autoReleaseAfterClose>false</autoReleaseAfterClose>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>io.github.git-commit-id</groupId>
+                <artifactId>git-commit-id-maven-plugin</artifactId>
+                <version>4.9.9</version>
+                <executions>
+                    <execution>
+                        <id>get-the-git-infos</id>
+                        <goals>
+                            <goal>revision</goal>
+                        </goals>
+                        <phase>initialize</phase>
+                    </execution>
+                    <execution>
+                        <id>validate-the-git-infos</id>
+                        <goals>
+                            <goal>validateRevision</goal>
+                        </goals>
+                        <phase>package</phase>
+                    </execution>
+                </executions>
+                <configuration>
+                    <verbose>true</verbose>
+                    <generateGitPropertiesFile>true</generateGitPropertiesFile>
+                    <generateGitPropertiesFilename>${project.build.outputDirectory}/git.properties</generateGitPropertiesFilename>
+                    <includeOnlyProperties>
+                        <includeOnlyProperty>^git.build.(time|version)$</includeOnlyProperty>
+                        <includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
+                    </includeOnlyProperties>
+                    <commitIdGenerationMode>full</commitIdGenerationMode>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.0.0-M5</version>
+                <configuration>
+                    <forkMode>always</forkMode>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-site-plugin</artifactId>
+                <version>3.7.1</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-project-info-reports-plugin</artifactId>
+                <version>3.0.0</version>
+            </plugin>
+            <plugin>
+                <groupId>com.github.spotbugs</groupId>
+                <artifactId>spotbugs-maven-plugin</artifactId>
+                <version>4.5.2.0</version>
+                <configuration>
+                    <includeFilterFile>dev/spotbugs-include.xml</includeFilterFile>
+                    <xmlOutput>true</xmlOutput>
+                    <failOnError>false</failOnError>
+                </configuration>
+                <dependencies>
+                    <!-- overwrite dependency on spotbugs if you want to specify the version of spotbugs -->
+                    <dependency>
+                        <groupId>com.github.spotbugs</groupId>
+                        <artifactId>spotbugs</artifactId>
+                        <version>4.5.3</version>
+                    </dependency>
+                </dependencies>
+            </plugin>
+            <plugin>
+                <groupId>au.com.acegi</groupId>
+                <artifactId>xml-format-maven-plugin</artifactId>
+                <version>3.2.0</version>
+                <executions>
+                    <execution>
+                        <id>xml-format</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>xml-format</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <indentSize>4</indentSize>
                 </configuration>
             </plugin>
         </plugins>
     </build>
+    <profiles>
+        <profile>
+            <id>jdk9plus</id>
+            <activation>
+                <!-- activated when building with JDK9+ -->
+                <jdk>!1.8</jdk>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>javax.annotation</groupId>
+                    <artifactId>javax.annotation-api</artifactId>
+                    <version>1.3.2</version>
+                    <scope>provided</scope>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
 </project>

scripts/proto.sh

@@ -1,45 +0,0 @@
-#!/bin/sh
-#
-#   Copyright 2017 PingCAP, Inc.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-
-CURRENT_DIR=`pwd`
-TIKV_CLIENT_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-cd $TIKV_CLIENT_HOME
-
-kvproto_hash=6ed99a08e262d8a32d6355dcba91cf99cb92074a
-
-raft_rs_hash=b9891b673573fad77ebcf9bbe0969cf945841926
-
-tipb_hash=c4d518eb1d60c21f05b028b36729e64610346dac
-
-if [ -d "kvproto" ]; then
-	cd kvproto; git fetch -p; git checkout ${kvproto_hash}; cd ..
-else
-	git clone https://github.com/pingcap/kvproto; cd kvproto; git checkout ${kvproto_hash}; cd ..
-fi
-
-if [ -d "raft-rs" ]; then
-	cd raft-rs; git fetch -p; git checkout ${raft_rs_hash}; cd ..
-else
-	git clone https://github.com/pingcap/raft-rs; cd raft-rs; git checkout ${raft_rs_hash}; cd ..
-fi
-
-if [ -d "tipb" ]; then
-	cd tipb; git fetch -p; git checkout ${tipb_hash}; cd ..
-else
-	git clone https://github.com/pingcap/tipb; cd tipb; git checkout ${tipb_hash}; cd ..
-fi
-
-cd $CURRENT_DIR

scripts/rstats.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+#!coding:utf-8
+
+# Copyright 2022 TiKV Project Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import re
+import json
+import argparse
+
+def main():
+    pattern = r'.*SlowLog:.*'
+    slowstr = "SlowLog:"
+    grpc_pattern = "gRPC tikvpb.Tikv"
+    backoff_pattern = "backoff "
+
+    args = parse_args()
+    items = []
+    with open(args.slowlog, encoding = 'utf-8') as f:
+        for line in f.readlines():
+            matched = re.match(pattern, line, re.M|re.I)
+            if matched is not None:
+                log = json.loads(line[(line.index(slowstr) + len(slowstr)):])
+                item = {
+                    'req': log['func'],
+                    'start': log['start'],
+                    'tot_lat': latency_ms(log),
+                    'tot_grpc': 0,
+                    'tot_bo': 0,
+                }
+                items.append(item)
+                for span in log['spans']:
+                    if grpc_pattern in span['name'] and span['duration'] != 'N/A':
+                        item['tot_grpc'] += latency_ms(span)
+                    elif backoff_pattern in span['name'] and span['duration'] != 'N/A':
+                        item['tot_bo'] += latency_ms(span)
+
+        if args.order == "total":
+            items = sorted(items, key=lambda d: d['tot_lat'], reverse=True)
+        elif args.order == "grpc":
+            items = sorted(items, key=lambda d: d['tot_grpc'], reverse=True)
+        elif args.order == "backoff":
+            items = sorted(items, key=lambda d: d['tot_bo'], reverse=True)
+        else:
+            print("unsupported order option, use default value: total")
+            items = sorted(items, key=lambda d: d['tot_lat'], reverse=True)
+
+    fmtStr = "{:<12} {:<14} {:<14} {:<20} {:<20}"
+    print(fmtStr.format("Request", "Start", "Total Lat(ms)", "Total gRPC Lat(ms)", "Total Backoff Lat(ms)"))
+    for item in items:
+        print(fmtStr.format(item['req'], item['start'], item['tot_lat'], item['tot_grpc'], item['tot_bo']))
+
+def latency_ms(span):
+    return int(span['duration'][:len(span['duration'])-2])
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="rstats: A TiKV Java Client Request Stats Analyzer")
+    parser.add_argument("-o", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff")
+    parser.add_argument("slowlog", help="slow log file")
+    return parser.parse_args()
+
+if __name__ == '__main__':
+    main()
+

src/main/java/org/tikv/BUILD

@@ -1,34 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-java_library(
-    name = "tikv-java-client-lib",
-    srcs = glob(
-        ["**/*.java"],
-    ),
-    deps = [
-        "//:java",
-        "@com_fasterxml_jackson_core_jackson_annotations//jar",
-        "@com_fasterxml_jackson_core_jackson_core//jar",
-        "@com_fasterxml_jackson_core_jackson_databind//jar",
-        "@com_google_code_findbugs_jsr305//jar",
-        "@com_google_code_gson_gson//jar",
-        "@com_google_errorprone_error_prone_annotations//jar",
-        "@com_google_guava_guava//jar",
-        "@com_google_protobuf_protobuf_java//jar",
-        "@joda_time//jar",
-        # the following are defined in rules_protobuf
-        "@org_pubref_rules_protobuf//java:grpc_compiletime_deps",
-        "@org_pubref_rules_protobuf//java:netty_runtime_deps",
-
-        "@org_slf4j_slf4j_api//jar",
-        "@org_slf4j_jcl_over_slf4j//jar",
-        "@org_slf4j_jul_to_slf4j//jar",
-        "@log4j_log4j//jar",
-        "@net_sf_trove4j_trove4j//jar",
-    ],
-)
-
-filegroup(
-    name = "srcs",
-    srcs = ["BUILD"] + glob(["**/*.java"]),
-)

src/main/java/org/tikv/Main.java

@@ -1,5 +0,0 @@
-package org.tikv;
-
-public class Main {
-  public static void main(String args[]) throws Exception {}
-}

src/main/java/org/tikv/common/AbstractGRPCClient.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -18,10 +20,16 @@ package org.tikv.common;
 import static io.grpc.stub.ClientCalls.asyncBidiStreamingCall;
 import static io.grpc.stub.ClientCalls.blockingServerStreamingCall;
 
+import io.grpc.ManagedChannel;
 import io.grpc.MethodDescriptor;
+import io.grpc.health.v1.HealthCheckRequest;
+import io.grpc.health.v1.HealthCheckResponse;
+import io.grpc.health.v1.HealthGrpc;
+import io.grpc.stub.AbstractFutureStub;
 import io.grpc.stub.AbstractStub;
 import io.grpc.stub.ClientCalls;
 import io.grpc.stub.StreamObserver;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -29,18 +37,20 @@ import org.tikv.common.operation.ErrorHandler;
 import org.tikv.common.policy.RetryMaxMs.Builder;
 import org.tikv.common.policy.RetryPolicy;
 import org.tikv.common.streaming.StreamingResponse;
+import org.tikv.common.util.BackOffFunction.BackOffFuncType;
 import org.tikv.common.util.BackOffer;
 import org.tikv.common.util.ChannelFactory;
 
 public abstract class AbstractGRPCClient<
-        BlockingStubT extends AbstractStub<BlockingStubT>, StubT extends AbstractStub<StubT>>
+        BlockingStubT extends AbstractStub<BlockingStubT>,
+        FutureStubT extends AbstractFutureStub<FutureStubT>>
     implements AutoCloseable {
   protected final Logger logger = LoggerFactory.getLogger(this.getClass());
   protected final ChannelFactory channelFactory;
   protected TiConfiguration conf;
   protected long timeout;
   protected BlockingStubT blockingStub;
-  protected StubT asyncStub;
+  protected FutureStubT asyncStub;
 
   protected AbstractGRPCClient(TiConfiguration conf, ChannelFactory channelFactory) {
     this.conf = conf;
@@ -52,7 +62,7 @@ public abstract class AbstractGRPCClient<
       TiConfiguration conf,
       ChannelFactory channelFactory,
       BlockingStubT blockingStub,
-      StubT asyncStub) {
+      FutureStubT asyncStub) {
     this.conf = conf;
     this.timeout = conf.getTimeout();
     this.channelFactory = channelFactory;
@@ -73,17 +83,16 @@ public abstract class AbstractGRPCClient<
     if (logger.isTraceEnabled()) {
       logger.trace(String.format("Calling %s...", method.getFullMethodName()));
     }
-    RetryPolicy.Builder<RespT> builder = new Builder<>(backOffer);
+    RetryPolicy<RespT> policy = new Builder<RespT>(backOffer).create(handler);
     RespT resp =
-        builder
-            .create(handler)
-            .callWithRetry(
-                () -> {
-                  BlockingStubT stub = getBlockingStub();
-                  return ClientCalls.blockingUnaryCall(
-                      stub.getChannel(), method, stub.getCallOptions(), requestFactory.get());
-                },
-                method.getFullMethodName());
+        policy.callWithRetry(
+            () -> {
+              BlockingStubT stub = getBlockingStub();
+              return ClientCalls.blockingUnaryCall(
+                  stub.getChannel(), method, stub.getCallOptions(), requestFactory.get());
+            },
+            method.getFullMethodName(),
+            backOffer);
 
     if (logger.isTraceEnabled()) {
       logger.trace(String.format("leaving %s...", method.getFullMethodName()));
@@ -99,19 +108,18 @@ public abstract class AbstractGRPCClient<
       ErrorHandler<RespT> handler) {
     logger.debug(String.format("Calling %s...", method.getFullMethodName()));
 
-    RetryPolicy.Builder<RespT> builder = new Builder<>(backOffer);
-    builder
-        .create(handler)
-        .callWithRetry(
-            () -> {
-              StubT stub = getAsyncStub();
-              ClientCalls.asyncUnaryCall(
-                  stub.getChannel().newCall(method, stub.getCallOptions()),
-                  requestFactory.get(),
-                  responseObserver);
-              return null;
-            },
-            method.getFullMethodName());
+    RetryPolicy<RespT> policy = new Builder<RespT>(backOffer).create(handler);
+    policy.callWithRetry(
+        () -> {
+          FutureStubT stub = getAsyncStub();
+          ClientCalls.asyncUnaryCall(
+              stub.getChannel().newCall(method, stub.getCallOptions()),
+              requestFactory.get(),
+              responseObserver);
+          return null;
+        },
+        method.getFullMethodName(),
+        backOffer);
     logger.debug(String.format("leaving %s...", method.getFullMethodName()));
   }
 
@@ -122,17 +130,17 @@ public abstract class AbstractGRPCClient<
       ErrorHandler<StreamObserver<ReqT>> handler) {
     logger.debug(String.format("Calling %s...", method.getFullMethodName()));
 
-    RetryPolicy.Builder<StreamObserver<ReqT>> builder = new Builder<>(backOffer);
+    RetryPolicy<StreamObserver<ReqT>> policy =
+        new Builder<StreamObserver<ReqT>>(backOffer).create(handler);
     StreamObserver<ReqT> observer =
-        builder
-            .create(handler)
-            .callWithRetry(
-                () -> {
-                  StubT stub = getAsyncStub();
-                  return asyncBidiStreamingCall(
-                      stub.getChannel().newCall(method, stub.getCallOptions()), responseObserver);
-                },
-                method.getFullMethodName());
+        policy.callWithRetry(
+            () -> {
+              FutureStubT stub = getAsyncStub();
+              return asyncBidiStreamingCall(
+                  stub.getChannel().newCall(method, stub.getCallOptions()), responseObserver);
+            },
+            method.getFullMethodName(),
+            backOffer);
     logger.debug(String.format("leaving %s...", method.getFullMethodName()));
     return observer;
   }
@@ -144,18 +152,18 @@ public abstract class AbstractGRPCClient<
       ErrorHandler<StreamingResponse> handler) {
     logger.debug(String.format("Calling %s...", method.getFullMethodName()));
 
-    RetryPolicy.Builder<StreamingResponse> builder = new Builder<>(backOffer);
+    RetryPolicy<StreamingResponse> policy =
+        new Builder<StreamingResponse>(backOffer).create(handler);
     StreamingResponse response =
-        builder
-            .create(handler)
-            .callWithRetry(
-                () -> {
-                  BlockingStubT stub = getBlockingStub();
-                  return new StreamingResponse(
-                      blockingServerStreamingCall(
-                          stub.getChannel(), method, stub.getCallOptions(), requestFactory.get()));
-                },
-                method.getFullMethodName());
+        policy.callWithRetry(
+            () -> {
+              BlockingStubT stub = getBlockingStub();
+              return new StreamingResponse(
+                  blockingServerStreamingCall(
+                      stub.getChannel(), method, stub.getCallOptions(), requestFactory.get()));
+            },
+            method.getFullMethodName(),
+            backOffer);
     logger.debug(String.format("leaving %s...", method.getFullMethodName()));
     return response;
   }
@@ -170,5 +178,32 @@ public abstract class AbstractGRPCClient<
 
   protected abstract BlockingStubT getBlockingStub();
 
-  protected abstract StubT getAsyncStub();
+  protected abstract FutureStubT getAsyncStub();
+
+  private boolean doCheckHealth(BackOffer backOffer, String addressStr, HostMapping hostMapping) {
+    while (true) {
+      backOffer.checkTimeout();
+
+      try {
+        ManagedChannel channel = channelFactory.getChannel(addressStr, hostMapping);
+        HealthGrpc.HealthBlockingStub stub =
+            HealthGrpc.newBlockingStub(channel)
+                .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
+        HealthCheckRequest req = HealthCheckRequest.newBuilder().build();
+        HealthCheckResponse resp = stub.check(req);
+        return resp.getStatus() == HealthCheckResponse.ServingStatus.SERVING;
+      } catch (Exception e) {
+        logger.warn("check health failed, addr: {}, caused by: {}", addressStr, e.getMessage());
+        backOffer.doBackOff(BackOffFuncType.BoCheckHealth, e);
+      }
+    }
+  }
+
+  protected boolean checkHealth(BackOffer backOffer, String addressStr, HostMapping hostMapping) {
+    try {
+      return doCheckHealth(backOffer, addressStr, hostMapping);
+    } catch (Exception e) {
+      return false;
+    }
+  }
 }

src/main/java/org/tikv/common/BytePairWrapper.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/ByteWrapper.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/ConfigUtils.java

@@ -1,28 +1,42 @@
 /*
- * Copyright 2021 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
 
+import org.tikv.common.util.BackOffer;
 import org.tikv.kvproto.Kvrpcpb;
 
 public class ConfigUtils {
+  public static final String TIKV_CONFIGURATION_FILENAME = "tikv.properties";
+
   public static final String TIKV_PD_ADDRESSES = "tikv.pd.addresses";
   public static final String TIKV_GRPC_TIMEOUT = "tikv.grpc.timeout_in_ms";
+  public static final String TIKV_GRPC_INGEST_TIMEOUT = "tikv.grpc.ingest_timeout_in_ms";
+  public static final String TIKV_GRPC_FORWARD_TIMEOUT = "tikv.grpc.forward_timeout_in_ms";
+  public static final String TIKV_GRPC_WARM_UP_TIMEOUT = "tikv.grpc.warm_up_timeout_in_ms";
+  public static final String TIKV_PD_FIRST_GET_MEMBER_TIMEOUT =
+      "tikv.grpc.pd_first_get_member_timeout_in_ms";
   public static final String TIKV_GRPC_SCAN_TIMEOUT = "tikv.grpc.scan_timeout_in_ms";
   public static final String TIKV_GRPC_SCAN_BATCH_SIZE = "tikv.grpc.scan_batch_size";
   public static final String TIKV_GRPC_MAX_FRAME_SIZE = "tikv.grpc.max_frame_size";
+  public static final String TIKV_GRPC_KEEPALIVE_TIME = "tikv.grpc.keepalive_time";
+  public static final String TIKV_GRPC_KEEPALIVE_TIMEOUT = "tikv.grpc.keepalive_timeout";
+  public static final String TIKV_GRPC_IDLE_TIMEOUT = "tikv.grpc.idle_timeout";
+  public static final String TIKV_CONN_RECYCLE_TIME = "tikv.conn.recycle_time";
 
   public static final String TIKV_INDEX_SCAN_BATCH_SIZE = "tikv.index.scan_batch_size";
   public static final String TIKV_INDEX_SCAN_CONCURRENCY = "tikv.index.scan_concurrency";
@@ -42,18 +56,87 @@ public class ConfigUtils {
   public static final String TIKV_KV_CLIENT_CONCURRENCY = "tikv.kv_client_concurrency";
 
   public static final String TIKV_KV_MODE = "tikv.kv_mode";
-  public static final String TIKV_IS_REPLICA_READ = "tikv.is_replica_read";
+  public static final String TIKV_REPLICA_READ = "tikv.replica_read";
 
   public static final String TIKV_METRICS_ENABLE = "tikv.metrics.enable";
   public static final String TIKV_METRICS_PORT = "tikv.metrics.port";
 
   public static final String TIKV_NETWORK_MAPPING_NAME = "tikv.network.mapping";
+  public static final String TIKV_ENABLE_GRPC_FORWARD = "tikv.enable_grpc_forward";
+  public static final String TIKV_GRPC_HEALTH_CHECK_TIMEOUT = "tikv.grpc.health_check_timeout";
+  public static final String TIKV_HEALTH_CHECK_PERIOD_DURATION =
+      "tikv.health_check_period_duration";
+
+  public static final String TIKV_ENABLE_ATOMIC_FOR_CAS = "tikv.enable_atomic_for_cas";
+
+  public static final String TIKV_IMPORTER_MAX_KV_BATCH_BYTES = "tikv.importer.max_kv_batch_bytes";
+  public static final String TIKV_IMPORTER_MAX_KV_BATCH_SIZE = "tikv.importer.max_kv_batch_size";
+
+  public static final String TIKV_SCATTER_WAIT_SECONDS = "tikv.scatter_wait_seconds";
+
+  public static final String TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS = "tikv.rawkv.default_backoff_in_ms";
+  public static final String TIKV_RAWKV_READ_TIMEOUT_IN_MS = "tikv.rawkv.read_timeout_in_ms";
+  public static final String TIKV_RAWKV_WRITE_TIMEOUT_IN_MS = "tikv.rawkv.write_timeout_in_ms";
+  public static final String TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS =
+      "tikv.rawkv.batch_read_timeout_in_ms";
+  public static final String TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS =
+      "tikv.rawkv.batch_write_timeout_in_ms";
+  public static final String TIKV_RAWKV_SCAN_TIMEOUT_IN_MS = "tikv.rawkv.scan_timeout_in_ms";
+  public static final String TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS = "tikv.rawkv.clean_timeout_in_ms";
+  public static final String TIKV_BO_REGION_MISS_BASE_IN_MS = "tikv.bo_region_miss_base_in_ms";
+  public static final String TIKV_RAWKV_READ_SLOWLOG_IN_MS = "tikv.rawkv.read_slowlog_in_ms";
+  public static final String TIKV_RAWKV_WRITE_SLOWLOG_IN_MS = "tikv.rawkv.write_slowlog_in_ms";
+  public static final String TIKV_RAWKV_BATCH_READ_SLOWLOG_IN_MS =
+      "tikv.rawkv.batch_read_slowlog_in_ms";
+  public static final String TIKV_RAWKV_BATCH_WRITE_SLOWLOG_IN_MS =
+      "tikv.rawkv.batch_write_slowlog_in_ms";
+  public static final String TIKV_RAWKV_SCAN_SLOWLOG_IN_MS = "tikv.rawkv.scan_slowlog_in_ms";
+  public static final String TIKV_RAWKV_SERVER_SLOWLOG_FACTOR = "tikv.rawkv.server_slowlog_factor";
+
+  public static final String TIKV_TLS_ENABLE = "tikv.tls_enable";
+  public static final String TIKV_TLS_RELOAD_INTERVAL = "tikv.tls.reload_interval";
+  public static final String TIKV_TRUST_CERT_COLLECTION = "tikv.trust_cert_collection";
+  public static final String TIKV_KEY_CERT_CHAIN = "tikv.key_cert_chain";
+  public static final String TIKV_KEY_FILE = "tikv.key_file";
+
+  public static final String TIKV_USE_JKS = "tikv.use_jks";
+  public static final String TIKV_JKS_KEY_PATH = "tikv.jks.key_path";
+  public static final String TIKV_JKS_KEY_PASSWORD = "tikv.jks.key_password";
+  public static final String TIKV_JKS_TRUST_PATH = "tikv.jks.trust_path";
+  public static final String TIKV_JKS_TRUST_PASSWORD = "tikv.jks.trust_password";
+
+  public static final String TiKV_CIRCUIT_BREAK_ENABLE = "tikv.circuit_break.enable";
+  public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS =
+      "tikv.circuit_break.trigger.availability.window_in_seconds";
+  public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE =
+      "tikv.circuit_break.trigger.availability.error_threshold_percentage";
+  public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD =
+      "tikv.circuit_break.trigger.availability.request_volumn_threshold";
+  public static final String TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS =
+      "tikv.circuit_break.trigger.sleep_window_in_seconds";
+  public static final String TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT =
+      "tikv.circuit_break.trigger.attempt_request_count";
+
+  public static final String TIKV_SCAN_REGIONS_LIMIT = "tikv.scan_regions_limit";
+
+  public static final String TIFLASH_ENABLE = "tiflash.enable";
+  public static final String TIKV_WARM_UP_ENABLE = "tikv.warm_up.enable";
+
+  public static final String TIKV_API_VERSION = "tikv.api_version";
 
   public static final String DEF_PD_ADDRESSES = "127.0.0.1:2379";
-  public static final String DEF_TIMEOUT = "600ms";
+  public static final String DEF_TIMEOUT = "200ms";
+  public static final String DEF_TIKV_GRPC_INGEST_TIMEOUT = "200s";
+  public static final String DEF_FORWARD_TIMEOUT = "300ms";
+  public static final String DEF_TIKV_GRPC_WARM_UP_TIMEOUT = "5000ms";
+  public static final String DEF_TIKV_PD_FIRST_GET_MEMBER_TIMEOUT = "10000ms";
   public static final String DEF_SCAN_TIMEOUT = "20s";
+  public static final int DEF_CHECK_HEALTH_TIMEOUT = 100;
+  public static final int DEF_HEALTH_CHECK_PERIOD_DURATION = 300;
   public static final int DEF_SCAN_BATCH_SIZE = 10240;
   public static final int DEF_MAX_FRAME_SIZE = 268435456 * 2; // 256 * 2 MB
+  public static final String DEF_TIKV_CONN_RECYCLE_TIME = "60s";
+  public static final String DEF_TIKV_TLS_RELOAD_INTERVAL = "10s";
   public static final int DEF_INDEX_SCAN_BATCH_SIZE = 20000;
   public static final int DEF_REGION_SCAN_DOWNGRADE_THRESHOLD = 10000000;
   // if keyRange size per request exceeds this limit, the request might be too large to be accepted
@@ -72,10 +155,27 @@ public class ConfigUtils {
   public static final String DEF_DB_PREFIX = "";
   public static final int DEF_KV_CLIENT_CONCURRENCY = 10;
   public static final TiConfiguration.KVMode DEF_KV_MODE = TiConfiguration.KVMode.TXN;
-  public static final boolean DEF_IS_REPLICA_READ = false;
+  public static final String DEF_REPLICA_READ = "LEADER";
   public static final boolean DEF_METRICS_ENABLE = false;
   public static final int DEF_METRICS_PORT = 3140;
   public static final String DEF_TIKV_NETWORK_MAPPING_NAME = "";
+  public static final boolean DEF_GRPC_FORWARD_ENABLE = true;
+  public static final boolean DEF_TIKV_ENABLE_ATOMIC_FOR_CAS = false;
+
+  public static final int DEF_TIKV_IMPORTER_MAX_KV_BATCH_BYTES = 1024 * 1024;
+  public static final int DEF_TIKV_IMPORTER_MAX_KV_BATCH_SIZE = 1024 * 32;
+  public static final int DEF_TIKV_SCATTER_WAIT_SECONDS = 300;
+  public static final int DEF_TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS = BackOffer.RAWKV_MAX_BACKOFF;
+
+  public static final int DEF_TIKV_RAWKV_READ_TIMEOUT_IN_MS = 2000;
+  public static final int DEF_TIKV_RAWKV_WRITE_TIMEOUT_IN_MS = 2000;
+  public static final int DEF_TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS = 2000;
+  public static final int DEF_TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS = 2000;
+  public static final int DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS = 10000;
+  public static final int DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS = 600000;
+
+  public static final int DEF_TIKV_BO_REGION_MISS_BASE_IN_MS = 20;
+  public static final String DEF_TIKV_RAWKV_SCAN_SLOWLOG_IN_MS = "5000";
 
   public static final String NORMAL_COMMAND_PRIORITY = "NORMAL";
   public static final String LOW_COMMAND_PRIORITY = "LOW";
@@ -86,4 +186,27 @@ public class ConfigUtils {
 
   public static final String RAW_KV_MODE = "RAW";
   public static final String TXN_KV_MODE = "TXN";
+
+  public static final String LEADER = "LEADER";
+  public static final String FOLLOWER = "FOLLOWER";
+  public static final String LEADER_AND_FOLLOWER = "LEADER_AND_FOLLOWER";
+
+  public static final int DEF_TIKV_GRPC_KEEPALIVE_TIME = 10;
+  public static final int DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT = 3;
+  public static final int DEF_TIKV_GRPC_IDLE_TIMEOUT = 60;
+  public static final boolean DEF_TIKV_TLS_ENABLE = false;
+  public static final boolean DEF_TIKV_USE_JKS = false;
+  public static final boolean DEF_TIFLASH_ENABLE = false;
+  public static final boolean DEF_TIKV_WARM_UP_ENABLE = true;
+
+  public static final boolean DEF_TiKV_CIRCUIT_BREAK_ENABLE = false;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS = 60;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE = 100;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUST_VOLUMN_THRESHOLD = 10;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS = 20;
+  public static final int DEF_TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT = 10;
+
+  public static final int DEF_TIKV_SCAN_REGIONS_LIMIT = 1000;
+
+  public static final int DEF_TIKV_API_VERSION = 1;
 }

src/main/java/org/tikv/common/DefaultHostMapping.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+import static org.tikv.common.pd.PDUtils.addrToUri;
+
+import com.google.common.annotations.Beta;
+import io.etcd.jetcd.ByteSequence;
+import io.etcd.jetcd.Client;
+import io.etcd.jetcd.KeyValue;
+import io.etcd.jetcd.kv.GetResponse;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultHostMapping implements HostMapping {
+  private static final String NETWORK_MAPPING_PATH = "/client/url-mapping";
+  private final Client etcdClient;
+  private final String networkMappingName;
+  private final ConcurrentMap<String, String> hostMapping;
+  private final Logger logger = LoggerFactory.getLogger(DefaultHostMapping.class);
+
+  public DefaultHostMapping(Client etcdClient, String networkMappingName) {
+    this.etcdClient = etcdClient;
+    this.networkMappingName = networkMappingName;
+    this.hostMapping = new ConcurrentHashMap<>();
+  }
+
+  private ByteSequence hostToNetworkMappingKey(String host) {
+    String path = NETWORK_MAPPING_PATH + "/" + networkMappingName + "/" + host;
+    return ByteSequence.from(path, StandardCharsets.UTF_8);
+  }
+
+  @Beta
+  private String getMappedHostFromPD(String host) {
+    ByteSequence hostKey = hostToNetworkMappingKey(host);
+    for (int i = 0; i < 5; i++) {
+      CompletableFuture<GetResponse> future = etcdClient.getKVClient().get(hostKey);
+      try {
+        GetResponse resp = future.get();
+        List<KeyValue> kvs = resp.getKvs();
+        if (kvs.size() != 1) {
+          break;
+        }
+        return kvs.get(0).getValue().toString(StandardCharsets.UTF_8);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      } catch (ExecutionException e) {
+        logger.info("failed to get mapped Host from PD: " + host, e);
+        break;
+      } catch (Exception ignore) {
+        // ignore
+        break;
+      }
+    }
+    return host;
+  }
+
+  public URI getMappedURI(URI uri) {
+    if (networkMappingName.isEmpty()) {
+      return uri;
+    }
+    return addrToUri(
+        hostMapping.computeIfAbsent(uri.getHost(), this::getMappedHostFromPD)
+            + ":"
+            + uri.getPort());
+  }
+}

src/main/java/org/tikv/common/ExtendedDateTime.java

@@ -1,15 +1,15 @@
 /*
- *
- * Copyright 2019 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *

src/main/java/org/tikv/common/HostMapping.java

@@ -1,87 +1,25 @@
 /*
- * Copyright 2021 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
 
-import static org.tikv.common.pd.PDUtils.addrToUri;
-
-import com.google.common.annotations.Beta;
-import io.etcd.jetcd.ByteSequence;
-import io.etcd.jetcd.Client;
-import io.etcd.jetcd.KeyValue;
-import io.etcd.jetcd.kv.GetResponse;
+import java.io.Serializable;
 import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-public class HostMapping {
-  private static final String NETWORK_MAPPING_PATH = "/client/url-mapping";
-  private final Client etcdClient;
-  private final String networkMappingName;
-  private final ConcurrentMap<String, String> hostMapping;
-  private final Logger logger = LoggerFactory.getLogger(HostMapping.class);
-
-  public HostMapping(Client etcdClient, String networkMappingName) {
-    this.etcdClient = etcdClient;
-    this.networkMappingName = networkMappingName;
-    this.hostMapping = new ConcurrentHashMap<>();
-  }
-
-  private ByteSequence hostToNetworkMappingKey(String host) {
-    String path = NETWORK_MAPPING_PATH + "/" + networkMappingName + "/" + host;
-    return ByteSequence.from(path, StandardCharsets.UTF_8);
-  }
-
-  @Beta
-  private String getMappedHostFromPD(String host) {
-    ByteSequence hostKey = hostToNetworkMappingKey(host);
-    for (int i = 0; i < 5; i++) {
-      CompletableFuture<GetResponse> future = etcdClient.getKVClient().get(hostKey);
-      try {
-        GetResponse resp = future.get();
-        List<KeyValue> kvs = resp.getKvs();
-        if (kvs.size() != 1) {
-          break;
-        }
-        return kvs.get(0).getValue().toString(StandardCharsets.UTF_8);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-      } catch (ExecutionException e) {
-        logger.info("failed to get mapped Host from PD: " + host, e);
-        break;
-      } catch (Exception ignore) {
-        // ignore
-        break;
-      }
-    }
-    return host;
-  }
-
-  public URI getMappedURI(URI uri) {
-    if (networkMappingName.isEmpty()) {
-      return uri;
-    }
-    return addrToUri(
-        hostMapping.computeIfAbsent(uri.getHost(), this::getMappedHostFromPD)
-            + ":"
-            + uri.getPort());
-  }
+public interface HostMapping extends Serializable {
+  URI getMappedURI(URI uri);
 }

src/main/java/org/tikv/common/KVClient.java

@@ -1,15 +1,15 @@
 /*
- *
- * Copyright 2019 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
@@ -65,7 +65,9 @@ public class KVClient implements AutoCloseable {
    * @return a ByteString value if key exists, ByteString.EMPTY if key does not exist
    */
   public ByteString get(ByteString key, long version) throws GrpcException {
-    BackOffer backOffer = ConcreteBackOffer.newGetBackOff();
+    BackOffer backOffer =
+        ConcreteBackOffer.newGetBackOff(
+            clientBuilder.getRegionManager().getPDClient().getClusterId());
     while (true) {
       RegionStoreClient client = clientBuilder.build(key);
       try {

src/main/java/org/tikv/common/Main.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/MetricsServer.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+import io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.client.hotspot.DefaultExports;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MetricsServer {
+  private static final Logger logger = LoggerFactory.getLogger(MetricsServer.class);
+
+  private static MetricsServer METRICS_SERVER_INSTANCE = null;
+  private static int metricsServerRefCount = 0;
+
+  private final int port;
+  private final HTTPServer server;
+
+  public static MetricsServer getInstance(TiConfiguration conf) {
+    if (!conf.isMetricsEnable()) {
+      return null;
+    }
+
+    synchronized (MetricsServer.class) {
+      int port = conf.getMetricsPort();
+      if (METRICS_SERVER_INSTANCE != null) {
+        if (port != METRICS_SERVER_INSTANCE.port) {
+          throw new IllegalArgumentException(
+              String.format(
+                  "Do dot support multiple tikv.metrics.port, which are %d and %d",
+                  port, METRICS_SERVER_INSTANCE.port));
+        }
+      } else {
+        METRICS_SERVER_INSTANCE = new MetricsServer(port);
+      }
+      metricsServerRefCount += 1;
+      return METRICS_SERVER_INSTANCE;
+    }
+  }
+
+  private MetricsServer(int port) {
+    try {
+      this.port = port;
+      DefaultExports.initialize();
+      this.server = new HTTPServer(port, true);
+      logger.info("http server is up " + this.server.getPort());
+    } catch (Exception e) {
+      logger.error("http server not up");
+      throw new RuntimeException(e);
+    }
+  }
+
+  public void close() {
+    synchronized (MetricsServer.class) {
+      if (metricsServerRefCount == 1) {
+        if (server != null) {
+          server.stop();
+          logger.info("Metrics server on " + server.getPort() + " is stopped");
+        }
+        METRICS_SERVER_INSTANCE = null;
+      }
+
+      if (metricsServerRefCount >= 1) {
+        metricsServerRefCount -= 1;
+      }
+    }
+  }
+}

src/main/java/org/tikv/common/PDChecker.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common;
+
+public enum PDChecker {
+  Learner,
+  Replica,
+  Rule,
+  Split,
+  Merge,
+  JointState,
+  Priority;
+
+  public String apiName() {
+    switch (this) {
+      case Learner:
+        return "learner";
+      case Replica:
+        return "replica";
+      case Rule:
+        return "rule";
+      case Split:
+        return "split";
+      case Merge:
+        return "merge";
+      case JointState:
+        return "joint-state";
+      case Priority:
+        return "priority";
+    }
+    throw new IllegalArgumentException();
+  }
+}

src/main/java/org/tikv/common/PDClient.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -21,6 +23,9 @@ import static org.tikv.common.pd.PDError.buildFromPdpbError;
 import static org.tikv.common.pd.PDUtils.addrToUri;
 import static org.tikv.common.pd.PDUtils.uriToAddr;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.protobuf.ByteString;
@@ -30,40 +35,55 @@ import io.etcd.jetcd.KeyValue;
 import io.etcd.jetcd.kv.GetResponse;
 import io.etcd.jetcd.options.GetOption;
 import io.grpc.ManagedChannel;
+import io.grpc.Metadata;
+import io.grpc.stub.MetadataUtils;
 import io.prometheus.client.Histogram;
 import java.net.URI;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.tikv.common.TiConfiguration.KVMode;
-import org.tikv.common.codec.Codec.BytesCodec;
-import org.tikv.common.codec.CodecDataOutput;
+import org.tikv.common.apiversion.RequestKeyCodec;
 import org.tikv.common.codec.KeyUtils;
 import org.tikv.common.exception.GrpcException;
 import org.tikv.common.exception.TiClientInternalException;
 import org.tikv.common.meta.TiTimestamp;
 import org.tikv.common.operation.NoopHandler;
 import org.tikv.common.operation.PDErrorHandler;
-import org.tikv.common.region.TiRegion;
 import org.tikv.common.util.BackOffFunction.BackOffFuncType;
 import org.tikv.common.util.BackOffer;
 import org.tikv.common.util.ChannelFactory;
 import org.tikv.common.util.ConcreteBackOffer;
-import org.tikv.common.util.FutureObserver;
+import org.tikv.common.util.HistogramUtils;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Metapb;
 import org.tikv.kvproto.Metapb.Store;
 import org.tikv.kvproto.PDGrpc;
 import org.tikv.kvproto.PDGrpc.PDBlockingStub;
-import org.tikv.kvproto.PDGrpc.PDStub;
+import org.tikv.kvproto.PDGrpc.PDFutureStub;
+import org.tikv.kvproto.Pdpb;
 import org.tikv.kvproto.Pdpb.Error;
 import org.tikv.kvproto.Pdpb.ErrorType;
 import org.tikv.kvproto.Pdpb.GetAllStoresRequest;
@@ -84,40 +104,55 @@ import org.tikv.kvproto.Pdpb.ScatterRegionResponse;
 import org.tikv.kvproto.Pdpb.Timestamp;
 import org.tikv.kvproto.Pdpb.TsoRequest;
 import org.tikv.kvproto.Pdpb.TsoResponse;
+import org.tikv.kvproto.Pdpb.UpdateServiceGCSafePointRequest;
 
-public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
+public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
     implements ReadOnlyPDClient {
   private static final String TIFLASH_TABLE_SYNC_PROGRESS_PATH = "/tiflash/table/sync";
-  private final Logger logger = LoggerFactory.getLogger(PDClient.class);
+  private static final long MIN_TRY_UPDATE_DURATION = 50;
+  private static final int PAUSE_CHECKER_TIMEOUT = 300; // in seconds
+  private static final int KEEP_CHECKER_PAUSE_PERIOD = PAUSE_CHECKER_TIMEOUT / 5; // in seconds
+  private static final Logger logger = LoggerFactory.getLogger(PDClient.class);
+  private static final ObjectMapper mapper = new ObjectMapper();
+
+  private final RequestKeyCodec codec;
   private RequestHeader header;
   private TsoRequest tsoReq;
-  private volatile LeaderWrapper leaderWrapper;
+  private volatile PDClientWrapper pdClientWrapper;
   private ScheduledExecutorService service;
   private ScheduledExecutorService tiflashReplicaService;
+  private final HashMap<PDChecker, ScheduledExecutorService> pauseCheckerService = new HashMap<>();
   private List<URI> pdAddrs;
   private Client etcdClient;
   private ConcurrentMap<Long, Double> tiflashReplicaMap;
   private HostMapping hostMapping;
+  private long lastUpdateLeaderTime;
+  private final ExecutorService updateLeaderService = Executors.newSingleThreadExecutor();
+  private final AtomicBoolean updateLeaderNotify = new AtomicBoolean();
 
   public static final Histogram PD_GET_REGION_BY_KEY_REQUEST_LATENCY =
-      Histogram.build()
+      HistogramUtils.buildDuration()
           .name("client_java_pd_get_region_by_requests_latency")
           .help("pd getRegionByKey request latency.")
+          .labelNames("cluster")
           .register();
 
-  private PDClient(TiConfiguration conf, ChannelFactory channelFactory) {
+  private PDClient(TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
     super(conf, channelFactory);
     initCluster();
+    this.codec = codec;
     this.blockingStub = getBlockingStub();
     this.asyncStub = getAsyncStub();
   }
 
-  public static ReadOnlyPDClient create(TiConfiguration conf, ChannelFactory channelFactory) {
-    return createRaw(conf, channelFactory);
+  public static ReadOnlyPDClient create(
+      TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
+    return createRaw(conf, codec, channelFactory);
   }
 
-  static PDClient createRaw(TiConfiguration conf, ChannelFactory channelFactory) {
-    return new PDClient(conf, channelFactory);
+  static PDClient createRaw(
+      TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
+    return new PDClient(conf, codec, channelFactory);
   }
 
   public HostMapping getHostMapping() {
@@ -138,12 +173,75 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
     return new TiTimestamp(timestamp.getPhysical(), timestamp.getLogical());
   }
 
+  public synchronized void keepPauseChecker(PDChecker checker) {
+    if (!this.pauseCheckerService.containsKey(checker)) {
+      ScheduledExecutorService newService =
+          Executors.newSingleThreadScheduledExecutor(
+              new ThreadFactoryBuilder()
+                  .setNameFormat(String.format("PDClient-pause-%s-pool-%%d", checker.name()))
+                  .setDaemon(true)
+                  .build());
+      newService.scheduleAtFixedRate(
+          () -> pauseChecker(checker, PAUSE_CHECKER_TIMEOUT),
+          0,
+          KEEP_CHECKER_PAUSE_PERIOD,
+          TimeUnit.SECONDS);
+      this.pauseCheckerService.put(checker, newService);
+    }
+  }
+
+  public synchronized void stopKeepPauseChecker(PDChecker checker) {
+    if (this.pauseCheckerService.containsKey(checker)) {
+      this.pauseCheckerService.get(checker).shutdown();
+      this.pauseCheckerService.remove(checker);
+    }
+  }
+
+  public void resumeChecker(PDChecker checker) {
+    pauseChecker(checker, 0);
+  }
+
+  private void pauseChecker(PDChecker checker, int timeout) {
+    String verb = timeout == 0 ? "resume" : "pause";
+    URI url = pdAddrs.get(0);
+    String api = url.toString() + "/pd/api/v1/checker/" + checker.apiName();
+    HashMap<String, Integer> arguments = new HashMap<>();
+    arguments.put("delay", timeout);
+    try (CloseableHttpClient client = HttpClients.createDefault()) {
+      JsonMapper jsonMapper = new JsonMapper();
+      byte[] body = jsonMapper.writeValueAsBytes(arguments);
+      HttpPost post = new HttpPost(api);
+      post.setEntity(new ByteArrayEntity(body));
+      try (CloseableHttpResponse resp = client.execute(post)) {
+        if (resp.getStatusLine().getStatusCode() != 200) {
+          logger.error("failed to {} checker.", verb);
+        }
+        logger.info("checker {} {}d", checker.apiName(), verb);
+      }
+    } catch (Exception e) {
+      logger.error(String.format("failed to %s checker.", verb), e);
+    }
+  }
+
+  public Boolean isCheckerPaused(PDChecker checker) {
+    URI url = pdAddrs.get(0);
+    String api = url.toString() + "/pd/api/v1/checker/" + checker.apiName();
+    try {
+      HashMap<String, Boolean> status =
+          mapper.readValue(new URL(api), new TypeReference<HashMap<String, Boolean>>() {});
+      return status.get("paused");
+    } catch (Exception e) {
+      logger.error(String.format("failed to get %s checker status.", checker.apiName()), e);
+      return null;
+    }
+  }
+
   /**
    * Sends request to pd to scatter region.
    *
    * @param region represents a region info
    */
-  void scatterRegion(TiRegion region, BackOffer backOffer) {
+  void scatterRegion(Metapb.Region region, BackOffer backOffer) {
     Supplier<ScatterRegionRequest> request =
         () ->
             ScatterRegionRequest.newBuilder().setHeader(header).setRegionId(region.getId()).build();
@@ -167,7 +265,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
    *
    * @param region
    */
-  void waitScatterRegionFinish(TiRegion region, BackOffer backOffer) {
+  void waitScatterRegionFinish(Metapb.Region region, BackOffer backOffer) {
     for (; ; ) {
       GetOperatorResponse resp = getOperator(region.getId());
       if (resp != null) {
@@ -182,7 +280,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
                   "wait scatter region %d at key %s is %s",
                   region.getId(),
                   KeyUtils.formatBytes(resp.getDesc().toByteArray()),
-                  resp.getStatus().toString()));
+                  resp.getStatus()));
         }
       }
     }
@@ -193,7 +291,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
         () -> GetOperatorRequest.newBuilder().setHeader(header).setRegionId(regionId).build();
     // get operator no need to handle error and no need back offer.
     return callWithRetry(
-        ConcreteBackOffer.newCustomBackOff(0),
+        ConcreteBackOffer.newCustomBackOff(0, getClusterId()),
         PDGrpc.getGetOperatorMethod(),
         request,
         new NoopHandler<>());
@@ -220,60 +318,30 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
   }
 
   @Override
-  public TiRegion getRegionByKey(BackOffer backOffer, ByteString key) {
-    Histogram.Timer requestTimer = PD_GET_REGION_BY_KEY_REQUEST_LATENCY.startTimer();
+  public Pair<Metapb.Region, Metapb.Peer> getRegionByKey(BackOffer backOffer, ByteString key) {
+    Histogram.Timer requestTimer =
+        PD_GET_REGION_BY_KEY_REQUEST_LATENCY.labels(getClusterId().toString()).startTimer();
     try {
-      if (conf.getKvMode() == KVMode.TXN) {
-        CodecDataOutput cdo = new CodecDataOutput();
-        BytesCodec.writeBytes(cdo, key.toByteArray());
-        key = cdo.toByteString();
-      }
-      ByteString queryKey = key;
-
       Supplier<GetRegionRequest> request =
-          () -> GetRegionRequest.newBuilder().setHeader(header).setRegionKey(queryKey).build();
+          () ->
+              GetRegionRequest.newBuilder()
+                  .setHeader(header)
+                  .setRegionKey(codec.encodePdQuery(key))
+                  .build();
 
       PDErrorHandler<GetRegionResponse> handler =
           new PDErrorHandler<>(getRegionResponseErrorExtractor, this);
 
       GetRegionResponse resp =
           callWithRetry(backOffer, PDGrpc.getGetRegionMethod(), request, handler);
-      return new TiRegion(
-          resp.getRegion(),
-          resp.getLeader(),
-          conf.getIsolationLevel(),
-          conf.getCommandPriority(),
-          conf.getKvMode(),
-          conf.isReplicaRead());
+      return new Pair<>(codec.decodeRegion(resp.getRegion()), resp.getLeader());
     } finally {
       requestTimer.observeDuration();
     }
   }
 
   @Override
-  public Future<TiRegion> getRegionByKeyAsync(BackOffer backOffer, ByteString key) {
-    FutureObserver<TiRegion, GetRegionResponse> responseObserver =
-        new FutureObserver<>(
-            resp ->
-                new TiRegion(
-                    resp.getRegion(),
-                    resp.getLeader(),
-                    conf.getIsolationLevel(),
-                    conf.getCommandPriority(),
-                    conf.getKvMode(),
-                    conf.isReplicaRead()));
-    Supplier<GetRegionRequest> request =
-        () -> GetRegionRequest.newBuilder().setHeader(header).setRegionKey(key).build();
-
-    PDErrorHandler<GetRegionResponse> handler =
-        new PDErrorHandler<>(getRegionResponseErrorExtractor, this);
-
-    callAsyncWithRetry(backOffer, PDGrpc.getGetRegionMethod(), request, responseObserver, handler);
-    return responseObserver.getFuture();
-  }
-
-  @Override
-  public TiRegion getRegionByID(BackOffer backOffer, long id) {
+  public Pair<Metapb.Region, Metapb.Peer> getRegionByID(BackOffer backOffer, long id) {
     Supplier<GetRegionByIDRequest> request =
         () -> GetRegionByIDRequest.newBuilder().setHeader(header).setRegionId(id).build();
     PDErrorHandler<GetRegionResponse> handler =
@@ -281,37 +349,31 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
 
     GetRegionResponse resp =
         callWithRetry(backOffer, PDGrpc.getGetRegionByIDMethod(), request, handler);
-    // Instead of using default leader instance, explicitly set no leader to null
-    return new TiRegion(
-        resp.getRegion(),
-        resp.getLeader(),
-        conf.getIsolationLevel(),
-        conf.getCommandPriority(),
-        conf.getKvMode(),
-        conf.isReplicaRead());
+    return new Pair<Metapb.Region, Metapb.Peer>(
+        codec.decodeRegion(resp.getRegion()), resp.getLeader());
   }
 
   @Override
-  public Future<TiRegion> getRegionByIDAsync(BackOffer backOffer, long id) {
-    FutureObserver<TiRegion, GetRegionResponse> responseObserver =
-        new FutureObserver<>(
-            resp ->
-                new TiRegion(
-                    resp.getRegion(),
-                    resp.getLeader(),
-                    conf.getIsolationLevel(),
-                    conf.getCommandPriority(),
-                    conf.getKvMode(),
-                    conf.isReplicaRead()));
+  public List<Pdpb.Region> scanRegions(
+      BackOffer backOffer, ByteString startKey, ByteString endKey, int limit) {
+    // no need to backoff because ScanRegions is just for optimization
+    // introduce a warm-up timeout for ScanRegions requests
+    PDGrpc.PDBlockingStub stub =
+        getBlockingStub().withDeadlineAfter(conf.getWarmUpTimeout(), TimeUnit.MILLISECONDS);
+    Pair<ByteString, ByteString> range = codec.encodePdQueryRange(startKey, endKey);
+    Pdpb.ScanRegionsRequest request =
+        Pdpb.ScanRegionsRequest.newBuilder()
+            .setHeader(header)
+            .setStartKey(range.first)
+            .setEndKey(range.second)
+            .setLimit(limit)
+            .build();
+    Pdpb.ScanRegionsResponse resp = stub.scanRegions(request);
+    if (resp == null) {
+      return null;
+    }
 
-    Supplier<GetRegionByIDRequest> request =
-        () -> GetRegionByIDRequest.newBuilder().setHeader(header).setRegionId(id).build();
-    PDErrorHandler<GetRegionResponse> handler =
-        new PDErrorHandler<>(getRegionResponseErrorExtractor, this);
-
-    callAsyncWithRetry(
-        backOffer, PDGrpc.getGetRegionByIDMethod(), request, responseObserver, handler);
-    return responseObserver.getFuture();
+    return codec.decodePdRegions(resp.getRegionsList());
   }
 
   private Supplier<GetStoreRequest> buildGetStoreReq(long storeId) {
@@ -322,6 +384,17 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
     return () -> GetAllStoresRequest.newBuilder().setHeader(header).build();
   }
 
+  private Supplier<UpdateServiceGCSafePointRequest> buildUpdateServiceGCSafePointRequest(
+      ByteString serviceId, long ttl, long safePoint) {
+    return () ->
+        UpdateServiceGCSafePointRequest.newBuilder()
+            .setHeader(header)
+            .setSafePoint(safePoint)
+            .setServiceId(serviceId)
+            .setTTL(ttl)
+            .build();
+  }
+
   private <T> PDErrorHandler<GetStoreResponse> buildPDErrorHandler() {
     return new PDErrorHandler<>(
         r -> r.getHeader().hasError() ? buildFromPdpbError(r.getHeader().getError()) : null, this);
@@ -329,23 +402,16 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
 
   @Override
   public Store getStore(BackOffer backOffer, long storeId) {
-    return callWithRetry(
-            backOffer, PDGrpc.getGetStoreMethod(), buildGetStoreReq(storeId), buildPDErrorHandler())
-        .getStore();
-  }
-
-  @Override
-  public Future<Store> getStoreAsync(BackOffer backOffer, long storeId) {
-    FutureObserver<Store, GetStoreResponse> responseObserver =
-        new FutureObserver<>(GetStoreResponse::getStore);
-
-    callAsyncWithRetry(
-        backOffer,
-        PDGrpc.getGetStoreMethod(),
-        buildGetStoreReq(storeId),
-        responseObserver,
-        buildPDErrorHandler());
-    return responseObserver.getFuture();
+    GetStoreResponse resp =
+        callWithRetry(
+            backOffer,
+            PDGrpc.getGetStoreMethod(),
+            buildGetStoreReq(storeId),
+            buildPDErrorHandler());
+    if (resp != null) {
+      return resp.getStore();
+    }
+    return null;
   }
 
   @Override
@@ -361,8 +427,22 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
   }
 
   @Override
-  public boolean isReplicaRead() {
-    return conf.isReplicaRead();
+  public TiConfiguration.ReplicaRead getReplicaRead() {
+    return conf.getReplicaRead();
+  }
+
+  @Override
+  public Long updateServiceGCSafePoint(
+      String serviceId, long ttl, long safePoint, BackOffer backOffer) {
+    return callWithRetry(
+            backOffer,
+            PDGrpc.getUpdateServiceGCSafePointMethod(),
+            buildUpdateServiceGCSafePointRequest(
+                ByteString.copyFromUtf8(serviceId), ttl, safePoint),
+            new PDErrorHandler<>(
+                r -> r.getHeader().hasError() ? buildFromPdpbError(r.getHeader().getError()) : null,
+                this))
+        .getMinSafePoint();
   }
 
   @Override
@@ -377,6 +457,8 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
     if (channelFactory != null) {
       channelFactory.close();
     }
+
+    updateLeaderService.shutdownNow();
   }
 
   @VisibleForTesting
@@ -385,77 +467,210 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
   }
 
   @VisibleForTesting
-  LeaderWrapper getLeaderWrapper() {
-    return leaderWrapper;
+  PDClientWrapper getPdClientWrapper() {
+    return pdClientWrapper;
   }
 
-  private GetMembersResponse getMembers(URI uri) {
-    try {
-      ManagedChannel probChan = channelFactory.getChannel(uriToAddr(uri), hostMapping);
-      PDGrpc.PDBlockingStub stub = PDGrpc.newBlockingStub(probChan);
-      GetMembersRequest request =
-          GetMembersRequest.newBuilder().setHeader(RequestHeader.getDefaultInstance()).build();
-      GetMembersResponse resp = stub.getMembers(request);
-      // check if the response contains a valid leader
-      if (resp != null && resp.getLeader().getMemberId() == 0) {
-        return null;
+  private GetMembersResponse doGetMembers(BackOffer backOffer, URI uri) {
+    while (true) {
+      backOffer.checkTimeout();
+
+      try {
+        ManagedChannel probChan = channelFactory.getChannel(uriToAddr(uri), hostMapping);
+        PDGrpc.PDBlockingStub stub =
+            PDGrpc.newBlockingStub(probChan).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
+        GetMembersRequest request =
+            GetMembersRequest.newBuilder().setHeader(RequestHeader.getDefaultInstance()).build();
+        GetMembersResponse resp = stub.getMembers(request);
+        // check if the response contains a valid leader
+        if (resp != null && resp.getLeader().getMemberId() == 0) {
+          return null;
+        }
+        return resp;
+      } catch (Exception e) {
+        logger.warn(
+            "failed to get member from pd server from {}, caused by: {}", uri, e.getMessage());
+        backOffer.doBackOff(BackOffFuncType.BoPDRPC, e);
       }
-      return resp;
-    } catch (Exception e) {
-      logger.warn("failed to get member from pd server.", e);
     }
-    return null;
   }
 
-  synchronized boolean switchLeader(List<String> leaderURLs) {
-    if (leaderURLs.isEmpty()) return false;
-    String leaderUrlStr = leaderURLs.get(0);
-    // TODO: Why not strip protocol info on server side since grpc does not need it
-    if (leaderWrapper != null && leaderUrlStr.equals(leaderWrapper.getLeaderInfo())) {
-      return true;
+  private GetMembersResponse getMembers(BackOffer backOffer, URI uri) {
+    try {
+      return doGetMembers(backOffer, uri);
+    } catch (Exception e) {
+      return null;
+    }
+  }
+
+  // return whether the leader has changed to target address `leaderUrlStr`.
+  synchronized boolean trySwitchLeader(String leaderUrlStr) {
+    if (pdClientWrapper != null) {
+      if (leaderUrlStr.equals(pdClientWrapper.getLeaderInfo())) {
+        // The message to leader is not forwarded by follower.
+        if (leaderUrlStr.equals(pdClientWrapper.getStoreAddress())) {
+          return true;
+        }
+      }
+      // If leader has transferred to another member, we can create another leaderWrapper.
     }
     // switch leader
-    return createLeaderWrapper(leaderUrlStr);
+    return createLeaderClientWrapper(leaderUrlStr);
   }
 
-  private boolean createLeaderWrapper(String leaderUrlStr) {
+  private synchronized boolean createLeaderClientWrapper(String leaderUrlStr) {
     try {
-      URI newLeader = addrToUri(leaderUrlStr);
-      leaderUrlStr = uriToAddr(newLeader);
-      if (leaderWrapper != null && leaderUrlStr.equals(leaderWrapper.getLeaderInfo())) {
-        return true;
-      }
-
       // create new Leader
       ManagedChannel clientChannel = channelFactory.getChannel(leaderUrlStr, hostMapping);
-      leaderWrapper =
-          new LeaderWrapper(
-              leaderUrlStr,
-              PDGrpc.newBlockingStub(clientChannel),
-              PDGrpc.newStub(clientChannel),
-              System.nanoTime());
+      pdClientWrapper =
+          new PDClientWrapper(leaderUrlStr, leaderUrlStr, clientChannel, System.nanoTime());
+      timeout = conf.getTimeout();
     } catch (IllegalArgumentException e) {
-      logger.error("Error updating leader. " + leaderUrlStr, e);
       return false;
     }
-    logger.info(String.format("Switched to new leader: %s", leaderWrapper));
+    logger.info(String.format("Switched to new leader: %s", pdClientWrapper));
     return true;
   }
 
-  public void updateLeader() {
+  synchronized boolean createFollowerClientWrapper(
+      BackOffer backOffer, String followerUrlStr, String leaderUrls) {
+    // TODO: Why not strip protocol info on server side since grpc does not need it
+
+    try {
+      if (!checkHealth(backOffer, followerUrlStr, hostMapping)) {
+        return false;
+      }
+
+      // create new Leader
+      ManagedChannel channel = channelFactory.getChannel(followerUrlStr, hostMapping);
+      pdClientWrapper = new PDClientWrapper(leaderUrls, followerUrlStr, channel, System.nanoTime());
+      timeout = conf.getForwardTimeout();
+    } catch (IllegalArgumentException e) {
+      return false;
+    }
+    logger.info(String.format("Switched to new leader by follower forward: %s", pdClientWrapper));
+    return true;
+  }
+
+  public void tryUpdateLeaderOrForwardFollower() {
+    if (updateLeaderNotify.compareAndSet(false, true)) {
+      try {
+        updateLeaderService.submit(
+            () -> {
+              try {
+                updateLeaderOrForwardFollower();
+              } catch (Exception e) {
+                logger.info("update leader or forward follower failed", e);
+                throw e;
+              } finally {
+                updateLeaderNotify.set(false);
+                logger.info("updating leader finish");
+              }
+            });
+      } catch (RejectedExecutionException e) {
+        logger.error("PDClient is shutdown", e);
+        updateLeaderNotify.set(false);
+      }
+    }
+  }
+
+  private synchronized void updateLeaderOrForwardFollower() {
+    logger.warn("updating leader or forward follower");
+    if (System.currentTimeMillis() - lastUpdateLeaderTime < MIN_TRY_UPDATE_DURATION) {
+      return;
+    }
     for (URI url : this.pdAddrs) {
+      BackOffer backOffer = this.probeBackOffer();
       // since resp is null, we need update leader's address by walking through all pd server.
-      GetMembersResponse resp = getMembers(url);
+      GetMembersResponse resp = getMembers(backOffer, url);
       if (resp == null) {
         continue;
       }
+      if (resp.getLeader().getClientUrlsList().isEmpty()) {
+        continue;
+      }
+
+      String leaderUrlStr = resp.getLeader().getClientUrlsList().get(0);
+      leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
+
       // if leader is switched, just return.
-      if (switchLeader(resp.getLeader().getClientUrlsList())) {
+      if (checkHealth(backOffer, leaderUrlStr, hostMapping)
+          && createLeaderClientWrapper(leaderUrlStr)) {
+        lastUpdateLeaderTime = System.currentTimeMillis();
+        return;
+      }
+
+      if (!conf.getEnableGrpcForward()) {
+        continue;
+      }
+
+      logger.info(String.format("can not switch to new leader, try follower forward"));
+      List<Pdpb.Member> members = resp.getMembersList();
+
+      // If we have not used follower forward, try the first follower.
+      boolean hasReachNextMember =
+          pdClientWrapper != null && pdClientWrapper.getStoreAddress().equals(leaderUrlStr);
+
+      for (int i = 0; i < members.size() * 2; i++) {
+        Pdpb.Member member = members.get(i % members.size());
+        if (member.getMemberId() == resp.getLeader().getMemberId()) {
+          continue;
+        }
+        String followerUrlStr = member.getClientUrlsList().get(0);
+        followerUrlStr = uriToAddr(addrToUri(followerUrlStr));
+        if (pdClientWrapper != null && pdClientWrapper.getStoreAddress().equals(followerUrlStr)) {
+          hasReachNextMember = true;
+          continue;
+        }
+        if (hasReachNextMember
+            && createFollowerClientWrapper(backOffer, followerUrlStr, leaderUrlStr)) {
+          logger.warn(
+              String.format("forward request to pd [%s] by pd [%s]", leaderUrlStr, followerUrlStr));
+          return;
+        }
+      }
+    }
+    lastUpdateLeaderTime = System.currentTimeMillis();
+    if (pdClientWrapper == null) {
+      throw new TiClientInternalException(
+          "already tried all address on file, but not leader found yet.");
+    }
+  }
+
+  public void tryUpdateLeader() {
+    logger.info("try update leader");
+    for (URI url : this.pdAddrs) {
+      BackOffer backOffer = this.probeBackOffer();
+      // since resp is null, we need update leader's address by walking through all pd server.
+      GetMembersResponse resp = getMembers(backOffer, url);
+      if (resp == null) {
+        continue;
+      }
+      List<URI> urls =
+          resp.getMembersList()
+              .stream()
+              .map(mem -> addrToUri(mem.getClientUrls(0)))
+              .collect(Collectors.toList());
+      String leaderUrlStr = resp.getLeader().getClientUrlsList().get(0);
+      leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
+
+      // If leader is not change but becomes available, we can cancel follower forward.
+      if (checkHealth(backOffer, leaderUrlStr, hostMapping) && trySwitchLeader(leaderUrlStr)) {
+        if (!urls.equals(this.pdAddrs)) {
+          tryUpdateMembers(urls);
+        }
         return;
       }
     }
-    throw new TiClientInternalException(
-        "already tried all address on file, but not leader found yet.");
+    lastUpdateLeaderTime = System.currentTimeMillis();
+    if (pdClientWrapper == null) {
+      throw new TiClientInternalException(
+          "already tried all address on file, but not leader found yet.");
+    }
+  }
+
+  private synchronized void tryUpdateMembers(List<URI> members) {
+    this.pdAddrs = members;
   }
 
   public void updateTiFlashReplicaStatus() {
@@ -513,74 +728,130 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
 
   @Override
   protected PDBlockingStub getBlockingStub() {
-    if (leaderWrapper == null) {
+    if (pdClientWrapper == null) {
       throw new GrpcException("PDClient may not be initialized");
     }
-    return leaderWrapper.getBlockingStub().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
+    return pdClientWrapper.getBlockingStub().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
   }
 
   @Override
-  protected PDStub getAsyncStub() {
-    if (leaderWrapper == null) {
+  protected PDFutureStub getAsyncStub() {
+    if (pdClientWrapper == null) {
       throw new GrpcException("PDClient may not be initialized");
     }
-    return leaderWrapper.getAsyncStub().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
+    return pdClientWrapper.getAsyncStub().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
   }
 
   private void initCluster() {
+    logger.info("init cluster: start");
     GetMembersResponse resp = null;
-    List<URI> pdAddrs = getConf().getPdAddrs();
+    List<URI> pdAddrs = new ArrayList<>(getConf().getPdAddrs());
+    // shuffle PD addresses so that clients call getMembers from different PD
+    Collections.shuffle(pdAddrs);
     this.pdAddrs = pdAddrs;
-    this.etcdClient = Client.builder().endpoints(pdAddrs).build();
-    this.hostMapping = new HostMapping(this.etcdClient, conf.getNetworkMappingName());
+    this.etcdClient =
+        Client.builder()
+            .endpoints(pdAddrs)
+            .executorService(
+                Executors.newCachedThreadPool(
+                    new ThreadFactoryBuilder()
+                        .setNameFormat("etcd-conn-manager-pool-%d")
+                        .setDaemon(true)
+                        .build()))
+            .build();
+    logger.info("init host mapping: start");
+    this.hostMapping =
+        Optional.ofNullable(getConf().getHostMapping())
+            .orElseGet(() -> new DefaultHostMapping(this.etcdClient, conf.getNetworkMappingName()));
+    logger.info("init host mapping: end");
+    // The first request may cost too much latency
+    long originTimeout = this.timeout;
+    this.timeout = conf.getPdFirstGetMemberTimeout();
     for (URI u : pdAddrs) {
-      resp = getMembers(u);
+      logger.info("get members with pd " + u + ": start");
+      resp = getMembers(defaultBackOffer(), u);
+      logger.info("get members with pd " + u + ": end");
       if (resp != null) {
         break;
       }
     }
+    if (resp == null) {
+      logger.error("Could not get leader member with: " + pdAddrs);
+    }
+
+    this.timeout = originTimeout;
     checkNotNull(resp, "Failed to init client for PD cluster.");
     long clusterId = resp.getHeader().getClusterId();
     header = RequestHeader.newBuilder().setClusterId(clusterId).build();
     tsoReq = TsoRequest.newBuilder().setHeader(header).setCount(1).build();
     this.tiflashReplicaMap = new ConcurrentHashMap<>();
-    createLeaderWrapper(resp.getLeader().getClientUrls(0));
+    this.pdAddrs =
+        resp.getMembersList()
+            .stream()
+            .map(mem -> addrToUri(mem.getClientUrls(0)))
+            .collect(Collectors.toList());
+    logger.info("init cluster with address: " + this.pdAddrs);
+
+    String leaderUrlStr = resp.getLeader().getClientUrls(0);
+    leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
+    logger.info("createLeaderClientWrapper with leader " + leaderUrlStr + ": start");
+    createLeaderClientWrapper(leaderUrlStr);
+    logger.info("createLeaderClientWrapper with leader " + leaderUrlStr + ": end");
     service =
         Executors.newSingleThreadScheduledExecutor(
-            new ThreadFactoryBuilder().setDaemon(true).build());
+            new ThreadFactoryBuilder()
+                .setNameFormat("PDClient-update-leader-pool-%d")
+                .setDaemon(true)
+                .build());
     service.scheduleAtFixedRate(
         () -> {
           // Wrap this with a try catch block in case schedule update fails
           try {
-            updateLeader();
+            tryUpdateLeader();
           } catch (Exception e) {
             logger.warn("Update leader failed", e);
           }
         },
-        1,
-        1,
-        TimeUnit.MINUTES);
-    tiflashReplicaService =
-        Executors.newSingleThreadScheduledExecutor(
-            new ThreadFactoryBuilder().setDaemon(true).build());
-    tiflashReplicaService.scheduleAtFixedRate(
-        this::updateTiFlashReplicaStatus, 10, 10, TimeUnit.SECONDS);
+        10,
+        10,
+        TimeUnit.SECONDS);
+    if (conf.isTiFlashEnabled()) {
+      tiflashReplicaService =
+          Executors.newSingleThreadScheduledExecutor(
+              new ThreadFactoryBuilder()
+                  .setNameFormat("PDClient-tiflash-replica-pool-%d")
+                  .setDaemon(true)
+                  .build());
+      tiflashReplicaService.scheduleAtFixedRate(
+          this::updateTiFlashReplicaStatus, 10, 10, TimeUnit.SECONDS);
+    }
+    logger.info("init cluster: finish");
   }
 
-  static class LeaderWrapper {
+  static class PDClientWrapper {
     private final String leaderInfo;
     private final PDBlockingStub blockingStub;
-    private final PDStub asyncStub;
+    private final PDFutureStub asyncStub;
     private final long createTime;
+    private final String storeAddress;
 
-    LeaderWrapper(
-        String leaderInfo,
-        PDGrpc.PDBlockingStub blockingStub,
-        PDGrpc.PDStub asyncStub,
-        long createTime) {
+    PDClientWrapper(
+        String leaderInfo, String storeAddress, ManagedChannel clientChannel, long createTime) {
+      if (!storeAddress.equals(leaderInfo)) {
+        Metadata header = new Metadata();
+        header.put(TiConfiguration.PD_FORWARD_META_DATA_KEY, addrToUri(leaderInfo).toString());
+        this.blockingStub =
+            PDGrpc.newBlockingStub(clientChannel)
+                .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header));
+        this.asyncStub =
+            PDGrpc.newFutureStub(clientChannel)
+                .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header));
+      } else {
+        this.blockingStub = PDGrpc.newBlockingStub(clientChannel);
+        this.asyncStub = PDGrpc.newFutureStub(clientChannel);
+      }
       this.leaderInfo = leaderInfo;
-      this.blockingStub = blockingStub;
-      this.asyncStub = asyncStub;
+      this.storeAddress = storeAddress;
       this.createTime = createTime;
     }
 
@@ -588,11 +859,15 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
       return leaderInfo;
     }
 
+    String getStoreAddress() {
+      return storeAddress;
+    }
+
     PDBlockingStub getBlockingStub() {
       return blockingStub;
     }
 
-    PDStub getAsyncStub() {
+    PDFutureStub getAsyncStub() {
       return asyncStub;
     }
 
@@ -602,7 +877,28 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
 
     @Override
     public String toString() {
-      return "[leaderInfo: " + leaderInfo + "]";
+      return "[leaderInfo: " + leaderInfo + ", storeAddress: " + storeAddress + "]";
     }
   }
+
+  public Long getClusterId() {
+    return header.getClusterId();
+  }
+
+  public List<URI> getPdAddrs() {
+    return pdAddrs;
+  }
+
+  public RequestKeyCodec getCodec() {
+    return codec;
+  }
+
+  private static BackOffer defaultBackOffer() {
+    return ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF);
+  }
+
+  private BackOffer probeBackOffer() {
+    int maxSleep = (int) getTimeout() * 2;
+    return ConcreteBackOffer.newCustomBackOff(maxSleep);
+  }
 }

src/main/java/org/tikv/common/ReadOnlyPDClient.java

@@ -1,27 +1,31 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
 
 import com.google.protobuf.ByteString;
 import java.util.List;
-import java.util.concurrent.Future;
+import org.tikv.common.apiversion.RequestKeyCodec;
 import org.tikv.common.meta.TiTimestamp;
-import org.tikv.common.region.TiRegion;
 import org.tikv.common.util.BackOffer;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Metapb;
 import org.tikv.kvproto.Metapb.Store;
+import org.tikv.kvproto.Pdpb;
 
 /** Readonly PD client including only reading related interface Supposed for TiDB-like use cases */
 public interface ReadOnlyPDClient {
@@ -38,9 +42,7 @@ public interface ReadOnlyPDClient {
    * @param key key in bytes for locating a region
    * @return the region whose startKey and endKey range covers the given key
    */
-  TiRegion getRegionByKey(BackOffer backOffer, ByteString key);
-
-  Future<TiRegion> getRegionByKeyAsync(BackOffer backOffer, ByteString key);
+  Pair<Metapb.Region, Metapb.Peer> getRegionByKey(BackOffer backOffer, ByteString key);
 
   /**
    * Get Region by Region Id
@@ -48,9 +50,10 @@ public interface ReadOnlyPDClient {
    * @param id Region Id
    * @return the region corresponding to the given Id
    */
-  TiRegion getRegionByID(BackOffer backOffer, long id);
+  Pair<Metapb.Region, Metapb.Peer> getRegionByID(BackOffer backOffer, long id);
 
-  Future<TiRegion> getRegionByIDAsync(BackOffer backOffer, long id);
+  List<Pdpb.Region> scanRegions(
+      BackOffer backOffer, ByteString startKey, ByteString endKey, int limit);
 
   HostMapping getHostMapping();
 
@@ -62,9 +65,23 @@ public interface ReadOnlyPDClient {
    */
   Store getStore(BackOffer backOffer, long storeId);
 
-  Future<Store> getStoreAsync(BackOffer backOffer, long storeId);
-
   List<Store> getAllStores(BackOffer backOffer);
 
-  boolean isReplicaRead();
+  TiConfiguration.ReplicaRead getReplicaRead();
+
+  Long getClusterId();
+
+  RequestKeyCodec getCodec();
+
+  /**
+   * Update ServiceGCSafePoint
+   *
+   * @param serviceId ServiceId
+   * @param ttl TTL in seconds
+   * @param safePoint The TiTimestamp you want to set. Set to start_ts.getPrevious() is a good
+   *     practice
+   * @return the MinSafePoint of all services. If this value is greater than safePoint, it means
+   *     update failed
+   */
+  Long updateServiceGCSafePoint(String serviceId, long ttl, long safePoint, BackOffer backOffer);
 }

src/main/java/org/tikv/common/Snapshot.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -78,7 +80,9 @@ public class Snapshot {
     try (KVClient client = new KVClient(session, session.getRegionStoreClientBuilder())) {
       List<KvPair> kvPairList =
           client.batchGet(
-              ConcreteBackOffer.newCustomBackOff(backOffer), list, timestamp.getVersion());
+              ConcreteBackOffer.newCustomBackOff(backOffer, session.getPDClient().getClusterId()),
+              list,
+              timestamp.getVersion());
       return kvPairList
           .stream()
           .map(

src/main/java/org/tikv/common/StoreVersion.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -25,7 +27,6 @@ import org.tikv.common.util.ConcreteBackOffer;
 import org.tikv.kvproto.Metapb;
 
 public class StoreVersion {
-
   private static final int SCALE = 10000;
   private final Logger logger = LoggerFactory.getLogger(this.getClass());
   private int v0 = 9999;
@@ -60,7 +61,8 @@ public class StoreVersion {
   public static boolean minTiKVVersion(String version, PDClient pdClient) {
     StoreVersion storeVersion = new StoreVersion(version);
 
-    BackOffer bo = ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF);
+    BackOffer bo =
+        ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF, pdClient.getClusterId());
     List<Metapb.Store> storeList =
         pdClient
             .getAllStores(bo)

src/main/java/org/tikv/common/TiBatchWriteUtils.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2019 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/TiConfiguration.java

@@ -1,40 +1,201 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
 
-import static org.tikv.common.ConfigUtils.*;
+import static org.tikv.common.ConfigUtils.DEF_BATCH_DELETE_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.DEF_BATCH_GET_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.DEF_BATCH_PUT_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.DEF_BATCH_SCAN_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.DEF_CHECK_HEALTH_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_DB_PREFIX;
+import static org.tikv.common.ConfigUtils.DEF_DELETE_RANGE_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.DEF_FORWARD_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_GRPC_FORWARD_ENABLE;
+import static org.tikv.common.ConfigUtils.DEF_HEALTH_CHECK_PERIOD_DURATION;
+import static org.tikv.common.ConfigUtils.DEF_INDEX_SCAN_BATCH_SIZE;
+import static org.tikv.common.ConfigUtils.DEF_INDEX_SCAN_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.DEF_KV_CLIENT_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.DEF_MAX_FRAME_SIZE;
+import static org.tikv.common.ConfigUtils.DEF_METRICS_ENABLE;
+import static org.tikv.common.ConfigUtils.DEF_METRICS_PORT;
+import static org.tikv.common.ConfigUtils.DEF_PD_ADDRESSES;
+import static org.tikv.common.ConfigUtils.DEF_REPLICA_READ;
+import static org.tikv.common.ConfigUtils.DEF_SCAN_BATCH_SIZE;
+import static org.tikv.common.ConfigUtils.DEF_SCAN_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_SHOW_ROWID;
+import static org.tikv.common.ConfigUtils.DEF_TABLE_SCAN_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.DEF_TIFLASH_ENABLE;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_API_VERSION;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_BO_REGION_MISS_BASE_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_CONN_RECYCLE_TIME;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_ENABLE_ATOMIC_FOR_CAS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_IDLE_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_INGEST_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_KEEPALIVE_TIME;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_WARM_UP_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_IMPORTER_MAX_KV_BATCH_BYTES;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_IMPORTER_MAX_KV_BATCH_SIZE;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_NETWORK_MAPPING_NAME;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_PD_FIRST_GET_MEMBER_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_READ_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_SCAN_SLOWLOG_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_WRITE_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_SCAN_REGIONS_LIMIT;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_SCATTER_WAIT_SECONDS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_TLS_ENABLE;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_TLS_RELOAD_INTERVAL;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_USE_JKS;
+import static org.tikv.common.ConfigUtils.DEF_TIKV_WARM_UP_ENABLE;
+import static org.tikv.common.ConfigUtils.DEF_TIMEOUT;
+import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT;
+import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE;
+import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUST_VOLUMN_THRESHOLD;
+import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS;
+import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_ENABLE;
+import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS;
+import static org.tikv.common.ConfigUtils.FOLLOWER;
+import static org.tikv.common.ConfigUtils.HIGH_COMMAND_PRIORITY;
+import static org.tikv.common.ConfigUtils.LEADER_AND_FOLLOWER;
+import static org.tikv.common.ConfigUtils.LOW_COMMAND_PRIORITY;
+import static org.tikv.common.ConfigUtils.NORMAL_COMMAND_PRIORITY;
+import static org.tikv.common.ConfigUtils.RAW_KV_MODE;
+import static org.tikv.common.ConfigUtils.READ_COMMITTED_ISOLATION_LEVEL;
+import static org.tikv.common.ConfigUtils.SNAPSHOT_ISOLATION_LEVEL;
+import static org.tikv.common.ConfigUtils.TIFLASH_ENABLE;
+import static org.tikv.common.ConfigUtils.TIKV_API_VERSION;
+import static org.tikv.common.ConfigUtils.TIKV_BATCH_DELETE_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.TIKV_BATCH_GET_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.TIKV_BATCH_PUT_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.TIKV_BATCH_SCAN_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.TIKV_BO_REGION_MISS_BASE_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_CONN_RECYCLE_TIME;
+import static org.tikv.common.ConfigUtils.TIKV_DB_PREFIX;
+import static org.tikv.common.ConfigUtils.TIKV_DELETE_RANGE_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.TIKV_ENABLE_ATOMIC_FOR_CAS;
+import static org.tikv.common.ConfigUtils.TIKV_ENABLE_GRPC_FORWARD;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_FORWARD_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_HEALTH_CHECK_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_IDLE_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_INGEST_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_KEEPALIVE_TIME;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_KEEPALIVE_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_MAX_FRAME_SIZE;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_SCAN_BATCH_SIZE;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_SCAN_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_GRPC_WARM_UP_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_HEALTH_CHECK_PERIOD_DURATION;
+import static org.tikv.common.ConfigUtils.TIKV_IMPORTER_MAX_KV_BATCH_BYTES;
+import static org.tikv.common.ConfigUtils.TIKV_IMPORTER_MAX_KV_BATCH_SIZE;
+import static org.tikv.common.ConfigUtils.TIKV_INDEX_SCAN_BATCH_SIZE;
+import static org.tikv.common.ConfigUtils.TIKV_INDEX_SCAN_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.TIKV_JKS_KEY_PASSWORD;
+import static org.tikv.common.ConfigUtils.TIKV_JKS_KEY_PATH;
+import static org.tikv.common.ConfigUtils.TIKV_JKS_TRUST_PASSWORD;
+import static org.tikv.common.ConfigUtils.TIKV_JKS_TRUST_PATH;
+import static org.tikv.common.ConfigUtils.TIKV_KEY_CERT_CHAIN;
+import static org.tikv.common.ConfigUtils.TIKV_KEY_FILE;
+import static org.tikv.common.ConfigUtils.TIKV_KV_CLIENT_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.TIKV_KV_MODE;
+import static org.tikv.common.ConfigUtils.TIKV_METRICS_ENABLE;
+import static org.tikv.common.ConfigUtils.TIKV_METRICS_PORT;
+import static org.tikv.common.ConfigUtils.TIKV_NETWORK_MAPPING_NAME;
+import static org.tikv.common.ConfigUtils.TIKV_PD_ADDRESSES;
+import static org.tikv.common.ConfigUtils.TIKV_PD_FIRST_GET_MEMBER_TIMEOUT;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_BATCH_READ_SLOWLOG_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_BATCH_WRITE_SLOWLOG_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_READ_SLOWLOG_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_READ_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_SCAN_SLOWLOG_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_SCAN_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_SERVER_SLOWLOG_FACTOR;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_WRITE_SLOWLOG_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_RAWKV_WRITE_TIMEOUT_IN_MS;
+import static org.tikv.common.ConfigUtils.TIKV_REPLICA_READ;
+import static org.tikv.common.ConfigUtils.TIKV_REQUEST_COMMAND_PRIORITY;
+import static org.tikv.common.ConfigUtils.TIKV_REQUEST_ISOLATION_LEVEL;
+import static org.tikv.common.ConfigUtils.TIKV_SCAN_REGIONS_LIMIT;
+import static org.tikv.common.ConfigUtils.TIKV_SCATTER_WAIT_SECONDS;
+import static org.tikv.common.ConfigUtils.TIKV_SHOW_ROWID;
+import static org.tikv.common.ConfigUtils.TIKV_TABLE_SCAN_CONCURRENCY;
+import static org.tikv.common.ConfigUtils.TIKV_TLS_ENABLE;
+import static org.tikv.common.ConfigUtils.TIKV_TLS_RELOAD_INTERVAL;
+import static org.tikv.common.ConfigUtils.TIKV_TRUST_CERT_COLLECTION;
+import static org.tikv.common.ConfigUtils.TIKV_USE_JKS;
+import static org.tikv.common.ConfigUtils.TIKV_WARM_UP_ENABLE;
+import static org.tikv.common.ConfigUtils.TXN_KV_MODE;
+import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT;
+import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE;
+import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD;
+import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS;
+import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_ENABLE;
+import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS;
 
+import com.google.protobuf.ByteString;
+import io.grpc.Metadata;
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.Serializable;
 import java.net.URI;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.tikv.common.pd.PDUtils;
+import org.tikv.common.replica.ReplicaSelector;
+import org.tikv.kvproto.Kvrpcpb;
 import org.tikv.kvproto.Kvrpcpb.CommandPri;
 import org.tikv.kvproto.Kvrpcpb.IsolationLevel;
 
 public class TiConfiguration implements Serializable {
-
   private static final Logger logger = LoggerFactory.getLogger(TiConfiguration.class);
   private static final ConcurrentHashMap<String, String> settings = new ConcurrentHashMap<>();
+  public static final Metadata.Key<String> FORWARD_META_DATA_KEY =
+      Metadata.Key.of("tikv-forwarded-host", Metadata.ASCII_STRING_MARSHALLER);
+  public static final Metadata.Key<String> PD_FORWARD_META_DATA_KEY =
+      Metadata.Key.of("pd-forwarded-host", Metadata.ASCII_STRING_MARSHALLER);
+  public static final ByteString API_V2_RAW_PREFIX = ByteString.copyFromUtf8("r");
+  public static final ByteString API_V2_TXN_PREFIX = ByteString.copyFromUtf8("x");
 
   static {
+    // priority: system environment > config file > default
     loadFromSystemProperties();
+    loadFromConfigurationFile();
     loadFromDefaultProperties();
+    listAll();
   }
 
   private static void loadFromSystemProperties() {
@@ -45,12 +206,43 @@ public class TiConfiguration implements Serializable {
     }
   }
 
+  private static void loadFromConfigurationFile() {
+    try (InputStream input =
+        TiConfiguration.class
+            .getClassLoader()
+            .getResourceAsStream(ConfigUtils.TIKV_CONFIGURATION_FILENAME)) {
+      Properties properties = new Properties();
+
+      if (input == null) {
+        logger.warn("Unable to find " + ConfigUtils.TIKV_CONFIGURATION_FILENAME);
+        return;
+      }
+
+      logger.info("loading " + ConfigUtils.TIKV_CONFIGURATION_FILENAME);
+      properties.load(input);
+      for (String key : properties.stringPropertyNames()) {
+        if (key.startsWith("tikv.")) {
+          String value = properties.getProperty(key);
+          setIfMissing(key, value);
+        }
+      }
+    } catch (IOException e) {
+      logger.error("load config file error", e);
+    }
+  }
+
   private static void loadFromDefaultProperties() {
     setIfMissing(TIKV_PD_ADDRESSES, DEF_PD_ADDRESSES);
     setIfMissing(TIKV_GRPC_TIMEOUT, DEF_TIMEOUT);
+    setIfMissing(TIKV_GRPC_INGEST_TIMEOUT, DEF_TIKV_GRPC_INGEST_TIMEOUT);
+    setIfMissing(TIKV_GRPC_FORWARD_TIMEOUT, DEF_FORWARD_TIMEOUT);
+    setIfMissing(TIKV_GRPC_WARM_UP_TIMEOUT, DEF_TIKV_GRPC_WARM_UP_TIMEOUT);
+    setIfMissing(TIKV_PD_FIRST_GET_MEMBER_TIMEOUT, DEF_TIKV_PD_FIRST_GET_MEMBER_TIMEOUT);
     setIfMissing(TIKV_GRPC_SCAN_TIMEOUT, DEF_SCAN_TIMEOUT);
     setIfMissing(TIKV_GRPC_SCAN_BATCH_SIZE, DEF_SCAN_BATCH_SIZE);
     setIfMissing(TIKV_GRPC_MAX_FRAME_SIZE, DEF_MAX_FRAME_SIZE);
+    setIfMissing(TIKV_CONN_RECYCLE_TIME, DEF_TIKV_CONN_RECYCLE_TIME);
+    setIfMissing(TIKV_TLS_RELOAD_INTERVAL, DEF_TIKV_TLS_RELOAD_INTERVAL);
     setIfMissing(TIKV_INDEX_SCAN_BATCH_SIZE, DEF_INDEX_SCAN_BATCH_SIZE);
     setIfMissing(TIKV_INDEX_SCAN_CONCURRENCY, DEF_INDEX_SCAN_CONCURRENCY);
     setIfMissing(TIKV_TABLE_SCAN_CONCURRENCY, DEF_TABLE_SCAN_CONCURRENCY);
@@ -67,14 +259,54 @@ public class TiConfiguration implements Serializable {
     setIfMissing(TIKV_DB_PREFIX, DEF_DB_PREFIX);
     setIfMissing(TIKV_KV_CLIENT_CONCURRENCY, DEF_KV_CLIENT_CONCURRENCY);
     setIfMissing(TIKV_KV_MODE, TXN_KV_MODE);
-    setIfMissing(TIKV_IS_REPLICA_READ, DEF_IS_REPLICA_READ);
+    setIfMissing(TIKV_REPLICA_READ, DEF_REPLICA_READ);
     setIfMissing(TIKV_METRICS_ENABLE, DEF_METRICS_ENABLE);
     setIfMissing(TIKV_METRICS_PORT, DEF_METRICS_PORT);
     setIfMissing(TIKV_NETWORK_MAPPING_NAME, DEF_TIKV_NETWORK_MAPPING_NAME);
+    setIfMissing(TIKV_ENABLE_GRPC_FORWARD, DEF_GRPC_FORWARD_ENABLE);
+    setIfMissing(TIKV_GRPC_HEALTH_CHECK_TIMEOUT, DEF_CHECK_HEALTH_TIMEOUT);
+    setIfMissing(TIKV_HEALTH_CHECK_PERIOD_DURATION, DEF_HEALTH_CHECK_PERIOD_DURATION);
+    setIfMissing(TIKV_ENABLE_ATOMIC_FOR_CAS, DEF_TIKV_ENABLE_ATOMIC_FOR_CAS);
+    setIfMissing(TIKV_IMPORTER_MAX_KV_BATCH_BYTES, DEF_TIKV_IMPORTER_MAX_KV_BATCH_BYTES);
+    setIfMissing(TIKV_IMPORTER_MAX_KV_BATCH_SIZE, DEF_TIKV_IMPORTER_MAX_KV_BATCH_SIZE);
+    setIfMissing(TIKV_SCATTER_WAIT_SECONDS, DEF_TIKV_SCATTER_WAIT_SECONDS);
+    setIfMissing(TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS, DEF_TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS);
+    setIfMissing(TIKV_GRPC_KEEPALIVE_TIME, DEF_TIKV_GRPC_KEEPALIVE_TIME);
+    setIfMissing(TIKV_GRPC_KEEPALIVE_TIMEOUT, DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT);
+    setIfMissing(TIKV_GRPC_IDLE_TIMEOUT, DEF_TIKV_GRPC_IDLE_TIMEOUT);
+    setIfMissing(TIKV_TLS_ENABLE, DEF_TIKV_TLS_ENABLE);
+    setIfMissing(TIKV_USE_JKS, DEF_TIKV_USE_JKS);
+    setIfMissing(TIFLASH_ENABLE, DEF_TIFLASH_ENABLE);
+    setIfMissing(TIKV_WARM_UP_ENABLE, DEF_TIKV_WARM_UP_ENABLE);
+    setIfMissing(TIKV_RAWKV_READ_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_READ_TIMEOUT_IN_MS);
+    setIfMissing(TIKV_RAWKV_WRITE_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_WRITE_TIMEOUT_IN_MS);
+    setIfMissing(TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS);
+    setIfMissing(TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS);
+    setIfMissing(TIKV_RAWKV_SCAN_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS);
+    setIfMissing(TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS);
+    setIfMissing(TIKV_BO_REGION_MISS_BASE_IN_MS, DEF_TIKV_BO_REGION_MISS_BASE_IN_MS);
+    setIfMissing(TIKV_RAWKV_SCAN_SLOWLOG_IN_MS, DEF_TIKV_RAWKV_SCAN_SLOWLOG_IN_MS);
+    setIfMissing(TiKV_CIRCUIT_BREAK_ENABLE, DEF_TiKV_CIRCUIT_BREAK_ENABLE);
+    setIfMissing(
+        TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS,
+        DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS);
+    setIfMissing(
+        TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE,
+        DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE);
+    setIfMissing(
+        TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD,
+        DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUST_VOLUMN_THRESHOLD);
+    setIfMissing(
+        TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS, DEF_TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS);
+    setIfMissing(
+        TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT, DEF_TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT);
+    setIfMissing(TIKV_SCAN_REGIONS_LIMIT, DEF_TIKV_SCAN_REGIONS_LIMIT);
+
+    setIfMissing(TIKV_API_VERSION, DEF_TIKV_API_VERSION);
   }
 
   public static void listAll() {
-    logger.info(new ArrayList<>(settings.entrySet()).toString());
+    logger.info("static configurations are:" + new ArrayList<>(settings.entrySet()));
   }
 
   private static void set(String key, String value) {
@@ -117,10 +349,14 @@ public class TiConfiguration implements Serializable {
     return option.get();
   }
 
-  private static int getInt(String key) {
+  public static int getInt(String key) {
     return Integer.parseInt(get(key));
   }
 
+  public static Optional<Integer> getIntOption(String key) {
+    return getOption(key).map(Integer::parseInt);
+  }
+
   private static int getInt(String key, int defaultValue) {
     try {
       return getOption(key).map(Integer::parseInt).orElse(defaultValue);
@@ -216,9 +452,25 @@ public class TiConfiguration implements Serializable {
     }
   }
 
+  private static ReplicaRead getReplicaRead(String key) {
+    String value = get(key).toUpperCase(Locale.ROOT);
+    if (FOLLOWER.equals(value)) {
+      return ReplicaRead.FOLLOWER;
+    } else if (LEADER_AND_FOLLOWER.equals(value)) {
+      return ReplicaRead.LEADER_AND_FOLLOWER;
+    } else {
+      return ReplicaRead.LEADER;
+    }
+  }
+
   private long timeout = getTimeAsMs(TIKV_GRPC_TIMEOUT);
+  private long ingestTimeout = getTimeAsMs(TIKV_GRPC_INGEST_TIMEOUT);
+  private long forwardTimeout = getTimeAsMs(TIKV_GRPC_FORWARD_TIMEOUT);
+  private long warmUpTimeout = getTimeAsMs(TIKV_GRPC_WARM_UP_TIMEOUT);
+  private long pdFirstGetMemberTimeout = getTimeAsMs(TIKV_PD_FIRST_GET_MEMBER_TIMEOUT);
   private long scanTimeout = getTimeAsMs(TIKV_GRPC_SCAN_TIMEOUT);
   private int maxFrameSize = getInt(TIKV_GRPC_MAX_FRAME_SIZE);
+  private long connRecycleTime = getTimeAsSeconds(TIKV_CONN_RECYCLE_TIME);
   private List<URI> pdAddrs = getPdAddrs(TIKV_PD_ADDRESSES);
   private int indexScanBatchSize = getInt(TIKV_INDEX_SCAN_BATCH_SIZE);
   private int indexScanConcurrency = getInt(TIKV_INDEX_SCAN_CONCURRENCY);
@@ -233,20 +485,98 @@ public class TiConfiguration implements Serializable {
   private boolean showRowId = getBoolean(TIKV_SHOW_ROWID);
   private String dbPrefix = get(TIKV_DB_PREFIX);
   private KVMode kvMode = getKvMode(TIKV_KV_MODE);
+  private boolean enableGrpcForward = getBoolean(TIKV_ENABLE_GRPC_FORWARD);
 
   private int kvClientConcurrency = getInt(TIKV_KV_CLIENT_CONCURRENCY);
-  private boolean isReplicaRead = getBoolean(TIKV_IS_REPLICA_READ);
+  private ReplicaRead replicaRead = getReplicaRead(TIKV_REPLICA_READ);
+  private ReplicaSelector internalReplicaSelector = getReplicaSelector(replicaRead);
+  private ReplicaSelector replicaSelector;
 
   private boolean metricsEnable = getBoolean(TIKV_METRICS_ENABLE);
   private int metricsPort = getInt(TIKV_METRICS_PORT);
+  private int grpcHealthCheckTimeout = getInt(TIKV_GRPC_HEALTH_CHECK_TIMEOUT);
+  private int healthCheckPeriodDuration = getInt(TIKV_HEALTH_CHECK_PERIOD_DURATION);
 
   private final String networkMappingName = get(TIKV_NETWORK_MAPPING_NAME);
+  private HostMapping hostMapping = null;
+
+  private boolean enableAtomicForCAS = getBoolean(TIKV_ENABLE_ATOMIC_FOR_CAS);
+
+  private int importerMaxKVBatchBytes = getInt(TIKV_IMPORTER_MAX_KV_BATCH_BYTES);
+
+  private int importerMaxKVBatchSize = getInt(TIKV_IMPORTER_MAX_KV_BATCH_SIZE);
+
+  private int scatterWaitSeconds = getInt(TIKV_SCATTER_WAIT_SECONDS);
+
+  private int rawKVDefaultBackoffInMS = getInt(TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS);
+  private int rawKVReadTimeoutInMS = getInt(TIKV_RAWKV_READ_TIMEOUT_IN_MS);
+  private int rawKVWriteTimeoutInMS = getInt(TIKV_RAWKV_WRITE_TIMEOUT_IN_MS);
+  private int rawKVBatchReadTimeoutInMS = getInt(TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS);
+  private int rawKVBatchWriteTimeoutInMS = getInt(TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS);
+  private int rawKVScanTimeoutInMS = getInt(TIKV_RAWKV_SCAN_TIMEOUT_IN_MS);
+  private int rawKVCleanTimeoutInMS = getInt(TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS);
+  private Integer rawKVReadSlowLogInMS = getIntOption(TIKV_RAWKV_READ_SLOWLOG_IN_MS).orElse(null);
+  private Integer rawKVWriteSlowLogInMS = getIntOption(TIKV_RAWKV_WRITE_SLOWLOG_IN_MS).orElse(null);
+  private Integer rawKVBatchReadSlowLogInMS =
+      getIntOption(TIKV_RAWKV_BATCH_READ_SLOWLOG_IN_MS).orElse(null);
+  private Integer rawKVBatchWriteSlowLogInMS =
+      getIntOption(TIKV_RAWKV_BATCH_WRITE_SLOWLOG_IN_MS).orElse(null);
+  private int rawKVScanSlowLogInMS = getInt(TIKV_RAWKV_SCAN_SLOWLOG_IN_MS);
+  private double rawKVServerSlowLogFactor = getDouble(TIKV_RAWKV_SERVER_SLOWLOG_FACTOR, 0.5);
+
+  private boolean tlsEnable = getBoolean(TIKV_TLS_ENABLE);
+  private long certReloadInterval = getTimeAsSeconds(TIKV_TLS_RELOAD_INTERVAL);
+
+  private String trustCertCollectionFile = getOption(TIKV_TRUST_CERT_COLLECTION).orElse(null);
+  private String keyCertChainFile = getOption(TIKV_KEY_CERT_CHAIN).orElse(null);
+  private String keyFile = getOption(TIKV_KEY_FILE).orElse(null);
+
+  private boolean useJks = getBoolean(TIKV_USE_JKS);
+  private String jksKeyPath = getOption(TIKV_JKS_KEY_PATH).orElse(null);
+  private String jksKeyPassword = getOption(TIKV_JKS_KEY_PASSWORD).orElse(null);
+  private String jksTrustPath = getOption(TIKV_JKS_TRUST_PATH).orElse(null);
+  private String jksTrustPassword = getOption(TIKV_JKS_TRUST_PASSWORD).orElse(null);
+
+  private final boolean tiFlashEnable = getBoolean(TIFLASH_ENABLE);
+  private boolean warmUpEnable = getBoolean(TIKV_WARM_UP_ENABLE);
+
+  private boolean isTest = false;
+
+  private int keepaliveTime = getInt(TIKV_GRPC_KEEPALIVE_TIME);
+  private int keepaliveTimeout = getInt(TIKV_GRPC_KEEPALIVE_TIMEOUT);
+  private int idleTimeout = getInt(TIKV_GRPC_IDLE_TIMEOUT);
+
+  private boolean circuitBreakEnable = getBoolean(TiKV_CIRCUIT_BREAK_ENABLE);
+  private int circuitBreakAvailabilityWindowInSeconds =
+      getInt(TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS);
+  private int circuitBreakAvailabilityErrorThresholdPercentage =
+      getInt(TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE);
+  private int circuitBreakAvailabilityRequestVolumnThreshold =
+      getInt(TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD);
+  private int circuitBreakSleepWindowInSeconds = getInt(TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS);
+  private int circuitBreakAttemptRequestCount = getInt(TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT);
+
+  private int scanRegionsLimit = getInt(TIKV_SCAN_REGIONS_LIMIT);
+
+  private ApiVersion apiVersion = ApiVersion.fromInt(getInt(TIKV_API_VERSION));
 
   public enum KVMode {
     TXN,
     RAW
   }
 
+  public enum ReplicaRead {
+    LEADER,
+    FOLLOWER,
+    LEADER_AND_FOLLOWER
+  }
+
+  public TiConfiguration() {
+    if (rawKVServerSlowLogFactor < 0 || rawKVServerSlowLogFactor > 1) {
+      throw new IllegalArgumentException("rawkv_server_slowlog_factor must be in range [0, 1]");
+    }
+  }
+
   public static TiConfiguration createDefault() {
     return new TiConfiguration();
   }
@@ -301,6 +631,40 @@ public class TiConfiguration implements Serializable {
     return this;
   }
 
+  public long getIngestTimeout() {
+    return ingestTimeout;
+  }
+
+  public void setIngestTimeout(long ingestTimeout) {
+    this.ingestTimeout = ingestTimeout;
+  }
+
+  public long getForwardTimeout() {
+    return forwardTimeout;
+  }
+
+  public TiConfiguration setForwardTimeout(long timeout) {
+    this.forwardTimeout = timeout;
+    return this;
+  }
+
+  public long getWarmUpTimeout() {
+    return warmUpTimeout;
+  }
+
+  public TiConfiguration setWarmUpTimeout(long timeout) {
+    this.warmUpTimeout = timeout;
+    return this;
+  }
+
+  public long getPdFirstGetMemberTimeout() {
+    return pdFirstGetMemberTimeout;
+  }
+
+  public void setPdFirstGetMemberTimeout(long pdFirstGetMemberTimeout) {
+    this.pdFirstGetMemberTimeout = pdFirstGetMemberTimeout;
+  }
+
   public long getScanTimeout() {
     return scanTimeout;
   }
@@ -331,6 +695,15 @@ public class TiConfiguration implements Serializable {
     return this;
   }
 
+  public long getConnRecycleTimeInSeconds() {
+    return connRecycleTime;
+  }
+
+  public TiConfiguration setConnRecycleTimeInSeconds(int connRecycleTime) {
+    this.connRecycleTime = connRecycleTime;
+    return this;
+  }
+
   public int getIndexScanBatchSize() {
     return indexScanBatchSize;
   }
@@ -443,6 +816,14 @@ public class TiConfiguration implements Serializable {
     return kvMode;
   }
 
+  public boolean isRawKVMode() {
+    return getKvMode() == TiConfiguration.KVMode.RAW;
+  }
+
+  public boolean isTxnKVMode() {
+    return getKvMode() == KVMode.TXN;
+  }
+
   public TiConfiguration setKvMode(String kvMode) {
     this.kvMode = KVMode.valueOf(kvMode);
     return this;
@@ -457,15 +838,40 @@ public class TiConfiguration implements Serializable {
     return this;
   }
 
-  public boolean isReplicaRead() {
-    return isReplicaRead;
+  public ReplicaRead getReplicaRead() {
+    return replicaRead;
   }
 
-  public TiConfiguration setReplicaRead(boolean isReplicaRead) {
-    this.isReplicaRead = isReplicaRead;
+  public TiConfiguration setReplicaRead(ReplicaRead replicaRead) {
+    this.replicaRead = replicaRead;
+    this.internalReplicaSelector = getReplicaSelector(this.replicaRead);
     return this;
   }
 
+  private ReplicaSelector getReplicaSelector(ReplicaRead replicaRead) {
+    if (TiConfiguration.ReplicaRead.LEADER.equals(replicaRead)) {
+      return ReplicaSelector.LEADER;
+    } else if (TiConfiguration.ReplicaRead.FOLLOWER.equals(replicaRead)) {
+      return ReplicaSelector.FOLLOWER;
+    } else if (TiConfiguration.ReplicaRead.LEADER_AND_FOLLOWER.equals(replicaRead)) {
+      return ReplicaSelector.LEADER_AND_FOLLOWER;
+    } else {
+      return null;
+    }
+  }
+
+  public ReplicaSelector getReplicaSelector() {
+    if (replicaSelector != null) {
+      return replicaSelector;
+    } else {
+      return internalReplicaSelector;
+    }
+  }
+
+  public void setReplicaSelector(ReplicaSelector replicaSelector) {
+    this.replicaSelector = replicaSelector;
+  }
+
   public boolean isMetricsEnable() {
     return metricsEnable;
   }
@@ -487,4 +893,405 @@ public class TiConfiguration implements Serializable {
   public String getNetworkMappingName() {
     return this.networkMappingName;
   }
+
+  public HostMapping getHostMapping() {
+    return hostMapping;
+  }
+
+  public void setHostMapping(HostMapping mapping) {
+    this.hostMapping = mapping;
+  }
+
+  public boolean getEnableGrpcForward() {
+    return this.enableGrpcForward;
+  }
+
+  public void setEnableGrpcForward(boolean enableGrpcForward) {
+    this.enableGrpcForward = enableGrpcForward;
+  }
+
+  public long getGrpcHealthCheckTimeout() {
+    return this.grpcHealthCheckTimeout;
+  }
+
+  public void setGrpcHealthCheckTimeout(int grpcHealthCheckTimeout) {
+    this.grpcHealthCheckTimeout = grpcHealthCheckTimeout;
+  }
+
+  public long getHealthCheckPeriodDuration() {
+    return this.healthCheckPeriodDuration;
+  }
+
+  public void setHealthCheckPeriodDuration(int healthCheckPeriodDuration) {
+    this.healthCheckPeriodDuration = healthCheckPeriodDuration;
+  }
+
+  public boolean isEnableAtomicForCAS() {
+    return enableAtomicForCAS;
+  }
+
+  public void setEnableAtomicForCAS(boolean enableAtomicForCAS) {
+    this.enableAtomicForCAS = enableAtomicForCAS;
+  }
+
+  public int getImporterMaxKVBatchBytes() {
+    return importerMaxKVBatchBytes;
+  }
+
+  public void setImporterMaxKVBatchBytes(int importerMaxKVBatchBytes) {
+    this.importerMaxKVBatchBytes = importerMaxKVBatchBytes;
+  }
+
+  public int getImporterMaxKVBatchSize() {
+    return importerMaxKVBatchSize;
+  }
+
+  public void setImporterMaxKVBatchSize(int importerMaxKVBatchSize) {
+    this.importerMaxKVBatchSize = importerMaxKVBatchSize;
+  }
+
+  public int getScatterWaitSeconds() {
+    return scatterWaitSeconds;
+  }
+
+  public void setScatterWaitSeconds(int scatterWaitSeconds) {
+    this.scatterWaitSeconds = scatterWaitSeconds;
+  }
+
+  public int getRawKVDefaultBackoffInMS() {
+    return rawKVDefaultBackoffInMS;
+  }
+
+  public void setRawKVDefaultBackoffInMS(int rawKVDefaultBackoffInMS) {
+    this.rawKVDefaultBackoffInMS = rawKVDefaultBackoffInMS;
+  }
+
+  public boolean isTest() {
+    return isTest;
+  }
+
+  public void setTest(boolean test) {
+    isTest = test;
+  }
+
+  public int getKeepaliveTime() {
+    return keepaliveTime;
+  }
+
+  public void setKeepaliveTime(int keepaliveTime) {
+    this.keepaliveTime = keepaliveTime;
+  }
+
+  public int getKeepaliveTimeout() {
+    return keepaliveTimeout;
+  }
+
+  public void setKeepaliveTimeout(int timeout) {
+    this.keepaliveTimeout = timeout;
+  }
+
+  public int getIdleTimeout() {
+    return idleTimeout;
+  }
+
+  public void setIdleTimeout(int timeout) {
+    this.idleTimeout = timeout;
+  }
+
+  public boolean isTiFlashEnabled() {
+    return tiFlashEnable;
+  }
+
+  public boolean isWarmUpEnable() {
+    return warmUpEnable;
+  }
+
+  public void setWarmUpEnable(boolean warmUpEnable) {
+    this.warmUpEnable = warmUpEnable;
+  }
+
+  public boolean isTlsEnable() {
+    return tlsEnable;
+  }
+
+  public long getCertReloadIntervalInSeconds() {
+    return certReloadInterval;
+  }
+
+  public TiConfiguration setCertReloadIntervalInSeconds(long interval) {
+    this.certReloadInterval = interval;
+    return this;
+  }
+
+  public void setTlsEnable(boolean tlsEnable) {
+    this.tlsEnable = tlsEnable;
+  }
+
+  public String getTrustCertCollectionFile() {
+    return trustCertCollectionFile;
+  }
+
+  public void setTrustCertCollectionFile(String trustCertCollectionFile) {
+    this.trustCertCollectionFile = trustCertCollectionFile;
+  }
+
+  public String getKeyCertChainFile() {
+    return keyCertChainFile;
+  }
+
+  public void setKeyCertChainFile(String keyCertChainFile) {
+    this.keyCertChainFile = keyCertChainFile;
+  }
+
+  public String getKeyFile() {
+    return keyFile;
+  }
+
+  public void setKeyFile(String keyFile) {
+    this.keyFile = keyFile;
+  }
+
+  public boolean isJksEnable() {
+    return useJks;
+  }
+
+  public void setJksEnable(boolean useJks) {
+    this.useJks = useJks;
+  }
+
+  public String getJksKeyPath() {
+    return jksKeyPath;
+  }
+
+  public void setJksKeyPath(String jksKeyPath) {
+    this.jksKeyPath = jksKeyPath;
+  }
+
+  public String getJksKeyPassword() {
+    return jksKeyPassword;
+  }
+
+  public void setJksKeyPassword(String jksKeyPassword) {
+    this.jksKeyPassword = jksKeyPassword;
+  }
+
+  public String getJksTrustPath() {
+    return jksTrustPath;
+  }
+
+  public void setJksTrustPath(String jksTrustPath) {
+    this.jksTrustPath = jksTrustPath;
+  }
+
+  public String getJksTrustPassword() {
+    return jksTrustPassword;
+  }
+
+  public void setJksTrustPassword(String jksTrustPassword) {
+    this.jksTrustPassword = jksTrustPassword;
+  }
+
+  public int getRawKVReadTimeoutInMS() {
+    return rawKVReadTimeoutInMS;
+  }
+
+  public void setRawKVReadTimeoutInMS(int rawKVReadTimeoutInMS) {
+    this.rawKVReadTimeoutInMS = rawKVReadTimeoutInMS;
+  }
+
+  public int getRawKVWriteTimeoutInMS() {
+    return rawKVWriteTimeoutInMS;
+  }
+
+  public void setRawKVWriteTimeoutInMS(int rawKVWriteTimeoutInMS) {
+    this.rawKVWriteTimeoutInMS = rawKVWriteTimeoutInMS;
+  }
+
+  public int getRawKVBatchReadTimeoutInMS() {
+    return rawKVBatchReadTimeoutInMS;
+  }
+
+  public void setRawKVBatchReadTimeoutInMS(int rawKVBatchReadTimeoutInMS) {
+    this.rawKVBatchReadTimeoutInMS = rawKVBatchReadTimeoutInMS;
+  }
+
+  public int getRawKVBatchWriteTimeoutInMS() {
+    return rawKVBatchWriteTimeoutInMS;
+  }
+
+  public void setRawKVBatchWriteTimeoutInMS(int rawKVBatchWriteTimeoutInMS) {
+    this.rawKVBatchWriteTimeoutInMS = rawKVBatchWriteTimeoutInMS;
+  }
+
+  public int getRawKVScanTimeoutInMS() {
+    return rawKVScanTimeoutInMS;
+  }
+
+  public void setRawKVScanTimeoutInMS(int rawKVScanTimeoutInMS) {
+    this.rawKVScanTimeoutInMS = rawKVScanTimeoutInMS;
+  }
+
+  public int getRawKVCleanTimeoutInMS() {
+    return rawKVCleanTimeoutInMS;
+  }
+
+  public void setRawKVCleanTimeoutInMS(int rawKVCleanTimeoutInMS) {
+    this.rawKVCleanTimeoutInMS = rawKVCleanTimeoutInMS;
+  }
+
+  public Integer getRawKVReadSlowLogInMS() {
+    return Optional.ofNullable(rawKVReadSlowLogInMS).orElse((int) (getTimeout() * 2));
+  }
+
+  public void setRawKVReadSlowLogInMS(Integer rawKVReadSlowLogInMS) {
+    this.rawKVReadSlowLogInMS = rawKVReadSlowLogInMS;
+  }
+
+  public Integer getRawKVWriteSlowLogInMS() {
+    return Optional.ofNullable(rawKVWriteSlowLogInMS).orElse((int) (getTimeout() * 2));
+  }
+
+  public void setRawKVWriteSlowLogInMS(Integer rawKVWriteSlowLogInMS) {
+    this.rawKVWriteSlowLogInMS = rawKVWriteSlowLogInMS;
+  }
+
+  public Integer getRawKVBatchReadSlowLogInMS() {
+    return Optional.ofNullable(rawKVBatchReadSlowLogInMS).orElse((int) (getTimeout() * 2));
+  }
+
+  public void setRawKVBatchReadSlowLogInMS(Integer rawKVBatchReadSlowLogInMS) {
+    this.rawKVBatchReadSlowLogInMS = rawKVBatchReadSlowLogInMS;
+  }
+
+  public Integer getRawKVBatchWriteSlowLogInMS() {
+    return Optional.ofNullable(rawKVBatchWriteSlowLogInMS).orElse((int) (getTimeout() * 2));
+  }
+
+  public void setRawKVBatchWriteSlowLogInMS(Integer rawKVBatchWriteSlowLogInMS) {
+    this.rawKVBatchWriteSlowLogInMS = rawKVBatchWriteSlowLogInMS;
+  }
+
+  public int getRawKVScanSlowLogInMS() {
+    return rawKVScanSlowLogInMS;
+  }
+
+  public void setRawKVScanSlowLogInMS(int rawKVScanSlowLogInMS) {
+    this.rawKVScanSlowLogInMS = rawKVScanSlowLogInMS;
+  }
+
+  public double getRawKVServerSlowLogFactor() {
+    return rawKVServerSlowLogFactor;
+  }
+
+  public void setRawKVServerSlowLogFactor(double rawKVServerSlowLogFactor) {
+    if (rawKVServerSlowLogFactor < 0 || rawKVServerSlowLogFactor > 1) {
+      throw new IllegalArgumentException("rawkv_server_slowlog_factor must be in range [0, 1]");
+    }
+    this.rawKVServerSlowLogFactor = rawKVServerSlowLogFactor;
+  }
+
+  public boolean isCircuitBreakEnable() {
+    return circuitBreakEnable;
+  }
+
+  public void setCircuitBreakEnable(boolean circuitBreakEnable) {
+    this.circuitBreakEnable = circuitBreakEnable;
+  }
+
+  public int getCircuitBreakAvailabilityWindowInSeconds() {
+    return circuitBreakAvailabilityWindowInSeconds;
+  }
+
+  public void setCircuitBreakAvailabilityWindowInSeconds(
+      int circuitBreakAvailabilityWindowInSeconds) {
+    this.circuitBreakAvailabilityWindowInSeconds = circuitBreakAvailabilityWindowInSeconds;
+  }
+
+  public int getCircuitBreakAvailabilityErrorThresholdPercentage() {
+    return circuitBreakAvailabilityErrorThresholdPercentage;
+  }
+
+  public void setCircuitBreakAvailabilityErrorThresholdPercentage(
+      int circuitBreakAvailabilityErrorThresholdPercentage) {
+    this.circuitBreakAvailabilityErrorThresholdPercentage =
+        circuitBreakAvailabilityErrorThresholdPercentage;
+  }
+
+  public int getCircuitBreakAvailabilityRequestVolumnThreshold() {
+    return circuitBreakAvailabilityRequestVolumnThreshold;
+  }
+
+  public void setCircuitBreakAvailabilityRequestVolumnThreshold(
+      int circuitBreakAvailabilityRequestVolumnThreshold) {
+    this.circuitBreakAvailabilityRequestVolumnThreshold =
+        circuitBreakAvailabilityRequestVolumnThreshold;
+  }
+
+  public int getCircuitBreakSleepWindowInSeconds() {
+    return circuitBreakSleepWindowInSeconds;
+  }
+
+  public void setCircuitBreakSleepWindowInSeconds(int circuitBreakSleepWindowInSeconds) {
+    this.circuitBreakSleepWindowInSeconds = circuitBreakSleepWindowInSeconds;
+  }
+
+  public int getCircuitBreakAttemptRequestCount() {
+    return circuitBreakAttemptRequestCount;
+  }
+
+  public void setCircuitBreakAttemptRequestCount(int circuitBreakAttemptRequestCount) {
+    this.circuitBreakAttemptRequestCount = circuitBreakAttemptRequestCount;
+  }
+
+  public int getScanRegionsLimit() {
+    return scanRegionsLimit;
+  }
+
+  public void setScanRegionsLimit(int scanRegionsLimit) {
+    this.scanRegionsLimit = scanRegionsLimit;
+  }
+
+  public ApiVersion getApiVersion() {
+    return apiVersion;
+  }
+
+  public TiConfiguration setApiVersion(ApiVersion version) {
+    this.apiVersion = version;
+    return this;
+  }
+
+  public enum ApiVersion {
+    V1,
+    V2;
+
+    public static ApiVersion fromInt(int version) {
+      switch (version) {
+        case 1:
+          return V1;
+        case 2:
+          return V2;
+        default:
+          throw new IllegalArgumentException("unknown api version " + version);
+      }
+    }
+
+    public boolean isV1() {
+      return this == V1;
+    }
+
+    public boolean isV2() {
+      return this == V2;
+    }
+
+    public Kvrpcpb.APIVersion toPb() {
+      switch (this) {
+        case V1:
+          return Kvrpcpb.APIVersion.V1;
+        case V2:
+          return Kvrpcpb.APIVersion.V2;
+        default:
+          throw new IllegalArgumentException("unknown api version " + this);
+      }
+    }
+  }
 }

src/main/java/org/tikv/common/TiDBJDBCClient.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2019 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -39,6 +41,7 @@ public class TiDBJDBCClient implements AutoCloseable {
   private static final int DELAY_CLEAN_TABLE_LOCK_DEFAULT = 0;
   private static final String TIDB_ROW_FORMAT_VERSION_SQL = "select @@tidb_row_format_version";
   private static final int TIDB_ROW_FORMAT_VERSION_DEFAULT = 1;
+  private static final ObjectMapper objectMapper = new ObjectMapper();
   private final Logger logger = LoggerFactory.getLogger(getClass().getName());
   private final Connection connection;
 
@@ -118,7 +121,6 @@ public class TiDBJDBCClient implements AutoCloseable {
 
   private Map<String, Object> readConfMapFromTiDB() throws SQLException, IOException {
     String configJSON = (String) queryTiDBViaJDBC(SELECT_TIDB_CONFIG_SQL).get(0).get(0);
-    ObjectMapper objectMapper = new ObjectMapper();
     TypeReference<HashMap<String, Object>> typeRef =
         new TypeReference<HashMap<String, Object>>() {};
     return objectMapper.readValue(configJSON, typeRef);

src/main/java/org/tikv/common/TiSession.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;
@@ -20,46 +22,61 @@ import static org.tikv.common.util.ClientUtils.groupKeysByRegion;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.protobuf.ByteString;
-import io.prometheus.client.CollectorRegistry;
-import io.prometheus.client.exporter.HTTPServer;
-import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.tikv.common.apiversion.RequestKeyCodec;
+import org.tikv.common.apiversion.RequestKeyV1RawCodec;
+import org.tikv.common.apiversion.RequestKeyV1TxnCodec;
+import org.tikv.common.apiversion.RequestKeyV2RawCodec;
+import org.tikv.common.apiversion.RequestKeyV2TxnCodec;
 import org.tikv.common.catalog.Catalog;
-import org.tikv.common.event.CacheInvalidateEvent;
 import org.tikv.common.exception.TiKVException;
+import org.tikv.common.importer.ImporterStoreClient;
+import org.tikv.common.importer.SwitchTiKVModeClient;
 import org.tikv.common.key.Key;
 import org.tikv.common.meta.TiTimestamp;
-import org.tikv.common.policy.RetryPolicy;
 import org.tikv.common.region.RegionManager;
 import org.tikv.common.region.RegionStoreClient;
-import org.tikv.common.region.RegionStoreClient.RegionStoreClientBuilder;
 import org.tikv.common.region.TiRegion;
-import org.tikv.common.util.*;
+import org.tikv.common.region.TiStore;
+import org.tikv.common.util.BackOffFunction;
+import org.tikv.common.util.BackOffer;
+import org.tikv.common.util.ChannelFactory;
+import org.tikv.common.util.ConcreteBackOffer;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Errorpb;
+import org.tikv.kvproto.ImportSstpb;
 import org.tikv.kvproto.Metapb;
+import org.tikv.kvproto.Pdpb;
 import org.tikv.raw.RawKVClient;
+import org.tikv.raw.SmartRawKVClient;
+import org.tikv.service.failsafe.CircuitBreaker;
+import org.tikv.service.failsafe.CircuitBreakerImpl;
 import org.tikv.txn.KVClient;
 import org.tikv.txn.TxnKVClient;
 
 /**
  * TiSession is the holder for PD Client, Store pdClient and PD Cache All sessions share common
- * region store connection pool but separated PD conn and cache for better concurrency TiSession is
- * thread-safe but it's also recommended to have multiple session avoiding lock contention
+ * region store connection pool but separated PD conn and cache for better concurrency
+ *
+ * <p>TiSession is thread-safe but it's also recommended to have multiple session avoiding lock
+ * contention
  */
 public class TiSession implements AutoCloseable {
   private static final Logger logger = LoggerFactory.getLogger(TiSession.class);
   private static final Map<String, TiSession> sessionCachedMap = new HashMap<>();
   private final TiConfiguration conf;
+  private final RequestKeyCodec keyCodec;
   private final ChannelFactory channelFactory;
-  private Function<CacheInvalidateEvent, Void> cacheInvalidateCallback;
   // below object creation is either heavy or making connection (pd), pending for lazy loading
   private volatile PDClient client;
   private volatile Catalog catalog;
@@ -71,35 +88,184 @@ public class TiSession implements AutoCloseable {
   private volatile ExecutorService batchScanThreadPool;
   private volatile ExecutorService deleteRangeThreadPool;
   private volatile RegionManager regionManager;
+  private final boolean enableGrpcForward;
   private volatile RegionStoreClient.RegionStoreClientBuilder clientBuilder;
-  private boolean isClosed = false;
-  private HTTPServer server;
-  private CollectorRegistry collectorRegistry;
+  private volatile ImporterStoreClient.ImporterStoreClientBuilder importerClientBuilder;
+  private volatile boolean isClosed = false;
+  private volatile SwitchTiKVModeClient switchTiKVModeClient;
+  private final MetricsServer metricsServer;
+  private final CircuitBreaker circuitBreaker;
+  private static final int MAX_SPLIT_REGION_STACK_DEPTH = 6;
+
+  static {
+    logger.info("Welcome to TiKV Java Client {}", getVersionInfo());
+  }
+
+  private static class VersionInfo {
+
+    private final String buildVersion;
+    private final String commitHash;
+
+    public VersionInfo(String buildVersion, String commitHash) {
+      this.buildVersion = buildVersion;
+      this.commitHash = commitHash;
+    }
+
+    @Override
+    public String toString() {
+      return buildVersion + "@" + commitHash;
+    }
+  }
 
   public TiSession(TiConfiguration conf) {
+    // may throw org.tikv.common.MetricsServer  - http server not up
+    // put it at the beginning of this function to avoid unclosed Thread
+    this.metricsServer = MetricsServer.getInstance(conf);
+
     this.conf = conf;
-    this.channelFactory = new ChannelFactory(conf.getMaxFrameSize());
-    this.client = PDClient.createRaw(conf, channelFactory);
-    if (conf.isMetricsEnable()) {
-      try {
-        this.collectorRegistry = new CollectorRegistry();
-        this.collectorRegistry.register(RawKVClient.RAW_REQUEST_LATENCY);
-        this.collectorRegistry.register(RawKVClient.RAW_REQUEST_FAILURE);
-        this.collectorRegistry.register(RawKVClient.RAW_REQUEST_SUCCESS);
-        this.collectorRegistry.register(RegionStoreClient.GRPC_RAW_REQUEST_LATENCY);
-        this.collectorRegistry.register(RetryPolicy.GRPC_SINGLE_REQUEST_LATENCY);
-        this.collectorRegistry.register(RegionManager.GET_REGION_BY_KEY_REQUEST_LATENCY);
-        this.collectorRegistry.register(PDClient.PD_GET_REGION_BY_KEY_REQUEST_LATENCY);
-        this.server =
-            new HTTPServer(
-                new InetSocketAddress(conf.getMetricsPort()), this.collectorRegistry, true);
-        logger.info("http server is up " + this.server.getPort());
-      } catch (Exception e) {
-        logger.error("http server not up");
-        throw new RuntimeException(e);
+
+    if (conf.getApiVersion().isV1()) {
+      if (conf.isRawKVMode()) {
+        keyCodec = new RequestKeyV1RawCodec();
+      } else {
+        keyCodec = new RequestKeyV1TxnCodec();
+      }
+    } else {
+      if (conf.isRawKVMode()) {
+        keyCodec = new RequestKeyV2RawCodec();
+      } else {
+        keyCodec = new RequestKeyV2TxnCodec();
       }
     }
-    logger.info("TiSession initialized in " + conf.getKvMode() + " mode");
+
+    if (conf.isTlsEnable()) {
+      if (conf.isJksEnable()) {
+        this.channelFactory =
+            new ChannelFactory(
+                conf.getMaxFrameSize(),
+                conf.getKeepaliveTime(),
+                conf.getKeepaliveTimeout(),
+                conf.getIdleTimeout(),
+                conf.getConnRecycleTimeInSeconds(),
+                conf.getCertReloadIntervalInSeconds(),
+                conf.getJksKeyPath(),
+                conf.getJksKeyPassword(),
+                conf.getJksTrustPath(),
+                conf.getJksTrustPassword());
+      } else {
+        this.channelFactory =
+            new ChannelFactory(
+                conf.getMaxFrameSize(),
+                conf.getKeepaliveTime(),
+                conf.getKeepaliveTimeout(),
+                conf.getIdleTimeout(),
+                conf.getConnRecycleTimeInSeconds(),
+                conf.getCertReloadIntervalInSeconds(),
+                conf.getTrustCertCollectionFile(),
+                conf.getKeyCertChainFile(),
+                conf.getKeyFile());
+      }
+    } else {
+      this.channelFactory =
+          new ChannelFactory(
+              conf.getMaxFrameSize(),
+              conf.getKeepaliveTime(),
+              conf.getKeepaliveTimeout(),
+              conf.getIdleTimeout());
+    }
+
+    this.client = PDClient.createRaw(conf, keyCodec, channelFactory);
+    if (conf.getApiVersion().isV2() && !StoreVersion.minTiKVVersion(Version.API_V2, client)) {
+      throw new IllegalStateException(
+          "With API v2, store versions should not older than " + Version.API_V2);
+    }
+
+    this.enableGrpcForward = conf.getEnableGrpcForward();
+    if (this.enableGrpcForward) {
+      logger.info("enable grpc forward for high available");
+    }
+    if (conf.isWarmUpEnable() && conf.isRawKVMode()) {
+      warmUp();
+    }
+    this.circuitBreaker = new CircuitBreakerImpl(conf, client.getClusterId());
+    logger.info(
+        "TiSession initialized in "
+            + conf.getKvMode()
+            + " mode in API version: "
+            + conf.getApiVersion());
+  }
+
+  private static VersionInfo getVersionInfo() {
+    VersionInfo info;
+    try {
+      final Properties properties = new Properties();
+      properties.load(TiSession.class.getClassLoader().getResourceAsStream("git.properties"));
+      String version = properties.getProperty("git.build.version");
+      String commitHash = properties.getProperty("git.commit.id.full");
+      info = new VersionInfo(version, commitHash);
+    } catch (Exception e) {
+      logger.info("Fail to read package info: " + e.getMessage());
+      info = new VersionInfo("unknown", "unknown");
+    }
+    return info;
+  }
+
+  @VisibleForTesting
+  public synchronized void warmUp() {
+    long warmUpStartTime = System.nanoTime();
+    BackOffer backOffer = ConcreteBackOffer.newRawKVBackOff(getPDClient().getClusterId());
+    try {
+      // let JVM ClassLoader load gRPC error related classes
+      // this operation may cost 100ms
+      Errorpb.Error.newBuilder().setNotLeader(Errorpb.NotLeader.newBuilder().build()).build();
+
+      this.client = getPDClient();
+      this.regionManager = getRegionManager();
+      List<Metapb.Store> stores = this.client.getAllStores(backOffer);
+      // warm up store cache
+      for (Metapb.Store store : stores) {
+        this.regionManager.updateStore(
+            null, new TiStore(this.client.getStore(backOffer, store.getId())));
+      }
+
+      // use scan region to load region cache with limit
+      ByteString startKey = ByteString.EMPTY;
+      do {
+        List<Pdpb.Region> regions =
+            regionManager.scanRegions(
+                backOffer, startKey, ByteString.EMPTY, conf.getScanRegionsLimit());
+        if (regions == null || regions.isEmpty()) {
+          // something went wrong, but the warm-up process could continue
+          break;
+        }
+        for (Pdpb.Region region : regions) {
+          regionManager.insertRegionToCache(
+              regionManager.createRegion(region.getRegion(), backOffer));
+        }
+        startKey = regions.get(regions.size() - 1).getRegion().getEndKey();
+      } while (!startKey.isEmpty());
+
+      try (RawKVClient rawKVClient = createRawClient()) {
+        ByteString exampleKey = ByteString.EMPTY;
+        Optional<ByteString> prev = rawKVClient.get(exampleKey);
+        if (prev.isPresent()) {
+          rawKVClient.delete(exampleKey);
+          rawKVClient.putIfAbsent(exampleKey, prev.get());
+          rawKVClient.put(exampleKey, prev.get());
+        } else {
+          rawKVClient.putIfAbsent(exampleKey, ByteString.EMPTY);
+          rawKVClient.put(exampleKey, ByteString.EMPTY);
+          rawKVClient.delete(exampleKey);
+        }
+      }
+    } catch (Exception e) {
+      // ignore error
+      logger.info("warm up fails, ignored ", e);
+    } finally {
+      logger.info(
+          String.format(
+              "warm up duration %d ms", (System.nanoTime() - warmUpStartTime) / 1_000_000));
+    }
   }
 
   @VisibleForTesting
@@ -122,31 +288,66 @@ public class TiSession implements AutoCloseable {
   }
 
   public RawKVClient createRawClient() {
-    RegionStoreClientBuilder builder =
-        new RegionStoreClientBuilder(conf, channelFactory, this.getRegionManager(), client);
-    return new RawKVClient(this, builder);
+    checkIsClosed();
+
+    return new RawKVClient(this, this.getRegionStoreClientBuilder());
+  }
+
+  public SmartRawKVClient createSmartRawClient() {
+    RawKVClient rawKVClient = createRawClient();
+    return new SmartRawKVClient(rawKVClient, circuitBreaker);
   }
 
   public KVClient createKVClient() {
-    RegionStoreClientBuilder builder =
-        new RegionStoreClientBuilder(conf, channelFactory, this.getRegionManager(), client);
-    return new KVClient(conf, builder);
+    checkIsClosed();
+
+    return new KVClient(this.conf, this.getRegionStoreClientBuilder(), this);
   }
 
   public TxnKVClient createTxnClient() {
+    checkIsClosed();
+
     return new TxnKVClient(conf, this.getRegionStoreClientBuilder(), this.getPDClient());
   }
 
   public RegionStoreClient.RegionStoreClientBuilder getRegionStoreClientBuilder() {
-    RegionStoreClient.RegionStoreClientBuilder res = clientBuilder;
+    checkIsClosed();
+
+    if (this.clientBuilder != null) {
+      return this.clientBuilder;
+    }
+
+    // lazily create the clientBuilder for the current TiSession
+    synchronized (this) {
+      if (this.clientBuilder == null) {
+        this.clientBuilder =
+            new RegionStoreClient.RegionStoreClientBuilder(
+                this.conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
+      }
+    }
+    return this.clientBuilder;
+  }
+
+  public ImporterStoreClient.ImporterStoreClientBuilder getImporterRegionStoreClientBuilder() {
+    checkIsClosed();
+
+    ImporterStoreClient.ImporterStoreClientBuilder res = importerClientBuilder;
     if (res == null) {
       synchronized (this) {
-        if (clientBuilder == null) {
-          clientBuilder =
-              new RegionStoreClient.RegionStoreClientBuilder(
-                  conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
+        if (importerClientBuilder == null) {
+          if (conf.isTxnKVMode()) {
+            importerClientBuilder =
+                new ImporterStoreClient.ImporterStoreClientBuilder<
+                    ImportSstpb.WriteRequest, ImportSstpb.WriteRequest>(
+                    conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
+          } else {
+            importerClientBuilder =
+                new ImporterStoreClient.ImporterStoreClientBuilder<
+                    ImportSstpb.RawWriteRequest, ImportSstpb.RawWriteResponse>(
+                    conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
+          }
         }
-        res = clientBuilder;
+        res = importerClientBuilder;
       }
     }
     return res;
@@ -157,23 +358,32 @@ public class TiSession implements AutoCloseable {
   }
 
   public TiTimestamp getTimestamp() {
-    return getPDClient().getTimestamp(ConcreteBackOffer.newTsoBackOff());
+    checkIsClosed();
+
+    return getPDClient()
+        .getTimestamp(ConcreteBackOffer.newTsoBackOff(getPDClient().getClusterId()));
   }
 
   public Snapshot createSnapshot() {
+    checkIsClosed();
+
     return new Snapshot(getTimestamp(), this);
   }
 
   public Snapshot createSnapshot(TiTimestamp ts) {
+    checkIsClosed();
+
     return new Snapshot(ts, this);
   }
 
   public PDClient getPDClient() {
+    checkIsClosed();
+
     PDClient res = client;
     if (res == null) {
       synchronized (this) {
         if (client == null) {
-          client = PDClient.createRaw(this.getConf(), channelFactory);
+          client = PDClient.createRaw(this.getConf(), keyCodec, channelFactory);
         }
         res = client;
       }
@@ -182,6 +392,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public Catalog getCatalog() {
+    checkIsClosed();
+
     Catalog res = catalog;
     if (res == null) {
       synchronized (this) {
@@ -194,12 +406,14 @@ public class TiSession implements AutoCloseable {
     return res;
   }
 
-  public synchronized RegionManager getRegionManager() {
+  public RegionManager getRegionManager() {
+    checkIsClosed();
+
     RegionManager res = regionManager;
     if (res == null) {
       synchronized (this) {
         if (regionManager == null) {
-          regionManager = new RegionManager(getPDClient(), this.cacheInvalidateCallback);
+          regionManager = new RegionManager(getConf(), getPDClient(), this.channelFactory);
         }
         res = regionManager;
       }
@@ -208,6 +422,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForIndexScan() {
+    checkIsClosed();
+
     ExecutorService res = indexScanThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -227,6 +443,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForTableScan() {
+    checkIsClosed();
+
     ExecutorService res = tableScanThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -243,6 +461,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForBatchPut() {
+    checkIsClosed();
+
     ExecutorService res = batchPutThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -262,6 +482,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForBatchGet() {
+    checkIsClosed();
+
     ExecutorService res = batchGetThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -281,6 +503,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForBatchDelete() {
+    checkIsClosed();
+
     ExecutorService res = batchDeleteThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -300,6 +524,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForBatchScan() {
+    checkIsClosed();
+
     ExecutorService res = batchScanThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -319,6 +545,8 @@ public class TiSession implements AutoCloseable {
   }
 
   public ExecutorService getThreadPoolForDeleteRange() {
+    checkIsClosed();
+
     ExecutorService res = deleteRangeThreadPool;
     if (res == null) {
       synchronized (this) {
@@ -339,49 +567,67 @@ public class TiSession implements AutoCloseable {
 
   @VisibleForTesting
   public ChannelFactory getChannelFactory() {
+    checkIsClosed();
+
     return channelFactory;
   }
 
-  public CollectorRegistry getCollectorRegistry() {
-    return collectorRegistry;
-  }
-
   /**
-   * This is used for setting call back function to invalidate cache information
+   * SwitchTiKVModeClient is used for SST Ingest.
    *
-   * @param callBackFunc callback function
+   * @return a SwitchTiKVModeClient
    */
-  public void injectCallBackFunc(Function<CacheInvalidateEvent, Void> callBackFunc) {
-    this.cacheInvalidateCallback = callBackFunc;
+  public SwitchTiKVModeClient getSwitchTiKVModeClient() {
+    checkIsClosed();
+
+    SwitchTiKVModeClient res = switchTiKVModeClient;
+    if (res == null) {
+      synchronized (this) {
+        if (switchTiKVModeClient == null) {
+          switchTiKVModeClient =
+              new SwitchTiKVModeClient(getPDClient(), getImporterRegionStoreClientBuilder());
+        }
+        res = switchTiKVModeClient;
+      }
+    }
+    return res;
   }
 
   /**
    * split region and scatter
    *
    * @param splitKeys
+   * @param splitRegionBackoffMS
+   * @param scatterRegionBackoffMS
+   * @param scatterWaitMS
    */
   public void splitRegionAndScatter(
       List<byte[]> splitKeys,
       int splitRegionBackoffMS,
       int scatterRegionBackoffMS,
       int scatterWaitMS) {
+    checkIsClosed();
+
     logger.info(String.format("split key's size is %d", splitKeys.size()));
     long startMS = System.currentTimeMillis();
 
     // split region
-    List<TiRegion> newRegions =
+    List<Metapb.Region> newRegions =
         splitRegion(
             splitKeys
                 .stream()
-                .map(k -> Key.toRawKey(k).next().toByteString())
+                .map(k -> Key.toRawKey(k).toByteString())
                 .collect(Collectors.toList()),
-            ConcreteBackOffer.newCustomBackOff(splitRegionBackoffMS));
+            ConcreteBackOffer.newCustomBackOff(splitRegionBackoffMS, getPDClient().getClusterId()));
 
     // scatter region
-    for (TiRegion newRegion : newRegions) {
+    for (Metapb.Region newRegion : newRegions) {
       try {
         getPDClient()
-            .scatterRegion(newRegion, ConcreteBackOffer.newCustomBackOff(scatterRegionBackoffMS));
+            .scatterRegion(
+                newRegion,
+                ConcreteBackOffer.newCustomBackOff(
+                    scatterRegionBackoffMS, getPDClient().getClusterId()));
       } catch (Exception e) {
         logger.warn(String.format("failed to scatter region: %d", newRegion.getId()), e);
       }
@@ -391,14 +637,16 @@ public class TiSession implements AutoCloseable {
     if (scatterWaitMS > 0) {
       logger.info("start to wait scatter region finish");
       long scatterRegionStartMS = System.currentTimeMillis();
-      for (TiRegion newRegion : newRegions) {
+      for (Metapb.Region newRegion : newRegions) {
         long remainMS = (scatterRegionStartMS + scatterWaitMS) - System.currentTimeMillis();
         if (remainMS <= 0) {
           logger.warn("wait scatter region timeout");
           return;
         }
         getPDClient()
-            .waitScatterRegionFinish(newRegion, ConcreteBackOffer.newCustomBackOff((int) remainMS));
+            .waitScatterRegionFinish(
+                newRegion,
+                ConcreteBackOffer.newCustomBackOff((int) remainMS, getPDClient().getClusterId()));
       }
     } else {
       logger.info("skip to wait scatter region finish");
@@ -408,17 +656,36 @@ public class TiSession implements AutoCloseable {
     logger.info("splitRegionAndScatter cost {} seconds", (endMS - startMS) / 1000);
   }
 
-  private List<TiRegion> splitRegion(List<ByteString> splitKeys, BackOffer backOffer) {
-    List<TiRegion> regions = new ArrayList<>();
+  /**
+   * split region and scatter
+   *
+   * @param splitKeys
+   */
+  public void splitRegionAndScatter(List<byte[]> splitKeys) {
+    checkIsClosed();
+
+    int splitRegionBackoffMS = BackOffer.SPLIT_REGION_BACKOFF;
+    int scatterRegionBackoffMS = BackOffer.SCATTER_REGION_BACKOFF;
+    int scatterWaitMS = conf.getScatterWaitSeconds() * 1000;
+    splitRegionAndScatter(splitKeys, splitRegionBackoffMS, scatterRegionBackoffMS, scatterWaitMS);
+  }
+
+  private List<Metapb.Region> splitRegion(List<ByteString> splitKeys, BackOffer backOffer) {
+    return splitRegion(splitKeys, backOffer, 1);
+  }
+
+  private List<Metapb.Region> splitRegion(
+      List<ByteString> splitKeys, BackOffer backOffer, int depth) {
+    List<Metapb.Region> regions = new ArrayList<>();
 
     Map<TiRegion, List<ByteString>> groupKeys =
-        groupKeysByRegion(regionManager, splitKeys, backOffer);
+        groupKeysByRegion(getRegionManager(), splitKeys, backOffer);
     for (Map.Entry<TiRegion, List<ByteString>> entry : groupKeys.entrySet()) {
 
-      Pair<TiRegion, Metapb.Store> pair =
+      Pair<TiRegion, TiStore> pair =
           getRegionManager().getRegionStorePairByKey(entry.getKey().getStartKey());
       TiRegion region = pair.first;
-      Metapb.Store store = pair.second;
+      TiStore store = pair.second;
       List<ByteString> splits =
           entry
               .getValue()
@@ -431,15 +698,25 @@ public class TiSession implements AutoCloseable {
             "split key equal to region start key or end key. Region splitting is not needed.");
       } else {
         logger.info("start to split region id={}, split size={}", region.getId(), splits.size());
-        List<TiRegion> newRegions;
+        List<Metapb.Region> newRegions;
         try {
           newRegions = getRegionStoreClientBuilder().build(region, store).splitRegion(splits);
+          // invalidate old region
+          getRegionManager().invalidateRegion(region);
         } catch (final TiKVException e) {
           // retry
           logger.warn("ReSplitting ranges for splitRegion", e);
-          clientBuilder.getRegionManager().invalidateRegion(region);
+          getRegionManager().invalidateRegion(region);
           backOffer.doBackOff(BackOffFunction.BackOffFuncType.BoRegionMiss, e);
-          newRegions = splitRegion(splits, backOffer);
+          if (depth >= MAX_SPLIT_REGION_STACK_DEPTH) {
+            logger.warn(
+                String.format(
+                    "Skip split region because MAX_SPLIT_REGION_STACK_DEPTH(%d) reached!",
+                    MAX_SPLIT_REGION_STACK_DEPTH));
+            newRegions = new ArrayList<>();
+          } else {
+            newRegions = splitRegion(splits, backOffer, depth + 1);
+          }
         }
         logger.info("region id={}, new region size={}", region.getId(), newRegions.size());
         regions.addAll(newRegions);
@@ -450,49 +727,119 @@ public class TiSession implements AutoCloseable {
     return regions;
   }
 
-  @Override
-  public synchronized void close() throws Exception {
+  private void checkIsClosed() {
     if (isClosed) {
-      logger.warn("this TiSession is already closed!");
-      return;
-    }
-
-    if (server != null) {
-      server.stop();
-      logger.info("Metrics server on " + server.getPort() + " is stopped");
-    }
-
-    isClosed = true;
-    synchronized (sessionCachedMap) {
-      sessionCachedMap.remove(conf.getPdAddrsString());
-    }
-
-    if (tableScanThreadPool != null) {
-      tableScanThreadPool.shutdownNow();
-    }
-    if (indexScanThreadPool != null) {
-      indexScanThreadPool.shutdownNow();
-    }
-    if (batchGetThreadPool != null) {
-      batchGetThreadPool.shutdownNow();
-    }
-    if (batchPutThreadPool != null) {
-      batchPutThreadPool.shutdownNow();
-    }
-    if (batchDeleteThreadPool != null) {
-      batchDeleteThreadPool.shutdownNow();
-    }
-    if (batchScanThreadPool != null) {
-      batchScanThreadPool.shutdownNow();
-    }
-    if (deleteRangeThreadPool != null) {
-      deleteRangeThreadPool.shutdownNow();
-    }
-    if (client != null) {
-      getPDClient().close();
-    }
-    if (catalog != null) {
-      getCatalog().close();
+      throw new RuntimeException("this TiSession is closed!");
     }
   }
+
+  public synchronized void closeAwaitTermination(long timeoutMS) throws Exception {
+    shutdown(false);
+
+    long startMS = System.currentTimeMillis();
+    while (true) {
+      if (isTerminatedExecutorServices()) {
+        cleanAfterTerminated();
+        return;
+      }
+
+      if (System.currentTimeMillis() - startMS > timeoutMS) {
+        shutdown(true);
+        return;
+      }
+      Thread.sleep(500);
+    }
+  }
+
+  @Override
+  public synchronized void close() throws Exception {
+    shutdown(true);
+  }
+
+  private synchronized void shutdown(boolean now) throws Exception {
+    if (!isClosed) {
+      isClosed = true;
+      synchronized (sessionCachedMap) {
+        sessionCachedMap.remove(conf.getPdAddrsString());
+      }
+
+      if (metricsServer != null) {
+        metricsServer.close();
+      }
+
+      if (circuitBreaker != null) {
+        circuitBreaker.close();
+      }
+    }
+
+    if (now) {
+      shutdownNowExecutorServices();
+      cleanAfterTerminated();
+    } else {
+      shutdownExecutorServices();
+    }
+  }
+
+  private synchronized void cleanAfterTerminated() throws InterruptedException {
+    if (regionManager != null) {
+      regionManager.close();
+    }
+    if (client != null) {
+      client.close();
+    }
+    if (catalog != null) {
+      catalog.close();
+    }
+
+    if (switchTiKVModeClient != null) {
+      switchTiKVModeClient.stopKeepTiKVToImportMode();
+    }
+  }
+
+  private List<ExecutorService> getExecutorServices() {
+    List<ExecutorService> executorServiceList = new ArrayList<>();
+    if (tableScanThreadPool != null) {
+      executorServiceList.add(tableScanThreadPool);
+    }
+    if (indexScanThreadPool != null) {
+      executorServiceList.add(indexScanThreadPool);
+    }
+    if (batchGetThreadPool != null) {
+      executorServiceList.add(batchGetThreadPool);
+    }
+    if (batchPutThreadPool != null) {
+      executorServiceList.add(batchPutThreadPool);
+    }
+    if (batchDeleteThreadPool != null) {
+      executorServiceList.add(batchDeleteThreadPool);
+    }
+    if (batchScanThreadPool != null) {
+      executorServiceList.add(batchScanThreadPool);
+    }
+    if (deleteRangeThreadPool != null) {
+      executorServiceList.add(deleteRangeThreadPool);
+    }
+    return executorServiceList;
+  }
+
+  private void shutdownExecutorServices() {
+    for (ExecutorService executorService : getExecutorServices()) {
+      executorService.shutdown();
+    }
+  }
+
+  private void shutdownNowExecutorServices() {
+    for (ExecutorService executorService : getExecutorServices()) {
+      executorService.shutdownNow();
+    }
+  }
+
+  private boolean isTerminatedExecutorServices() {
+    for (ExecutorService executorService : getExecutorServices()) {
+      if (!executorService.isTerminated()) {
+        return false;
+      }
+    }
+    return true;
+  }
 }

src/main/java/org/tikv/common/Utils.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2021 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common;

src/main/java/org/tikv/common/Version.java

@@ -1,15 +1,15 @@
 /*
- *
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
@@ -25,4 +25,6 @@ public class Version {
   public static final String RESOLVE_LOCK_V4 = "4.0.0";
 
   public static final String BATCH_WRITE = "3.0.14";
+
+  public static final String API_V2 = "6.1.0";
 }

src/main/java/org/tikv/common/allocator/RowIDAllocator.java

@@ -1,258 +0,0 @@
-/*
- * Copyright 2019 PingCAP, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.tikv.common.allocator;
-
-import com.google.common.primitives.UnsignedLongs;
-import com.google.protobuf.ByteString;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.function.Function;
-import org.tikv.common.Snapshot;
-import org.tikv.common.TiSession;
-import org.tikv.common.codec.CodecDataInput;
-import org.tikv.common.codec.CodecDataOutput;
-import org.tikv.common.codec.MetaCodec;
-import org.tikv.common.exception.AllocateRowIDOverflowException;
-import org.tikv.common.exception.TiBatchWriteException;
-import org.tikv.common.meta.TiTableInfo;
-import org.tikv.common.util.BackOffer;
-import org.tikv.common.util.ConcreteBackOffer;
-import org.tikv.txn.TwoPhaseCommitter;
-
-/**
- * RowIDAllocator read current start from TiKV and write back 'start+step' back to TiKV. It designs
- * to allocate all id for data to be written at once, hence it does not need run inside a txn.
- *
- * <p>(start, end] is allocated
- */
-public final class RowIDAllocator implements Serializable {
-  private final long maxShardRowIDBits;
-  private final long dbId;
-  private final TiSession session;
-  private final long step;
-  private long end;
-
-  private RowIDAllocator(long maxShardRowIDBits, long dbId, long step, TiSession session) {
-    this.maxShardRowIDBits = maxShardRowIDBits;
-    this.dbId = dbId;
-    this.step = step;
-    this.session = session;
-  }
-
-  /**
-   * @param index should >= 1
-   * @return
-   */
-  public long getShardRowId(long index) {
-    return getShardRowId(maxShardRowIDBits, index, index + getStart());
-  }
-
-  static long getShardRowId(long maxShardRowIDBits, long partitionIndex, long rowID) {
-    if (maxShardRowIDBits <= 0 || maxShardRowIDBits >= 16) {
-      return rowID;
-    }
-
-    // assert rowID < Math.pow(2, 64 - maxShardRowIDBits)
-
-    long partition = partitionIndex & ((1L << maxShardRowIDBits) - 1);
-    return rowID | (partition << (64 - maxShardRowIDBits - 1));
-  }
-
-  public static RowIDAllocator create(
-      long dbId, TiTableInfo table, TiSession session, boolean unsigned, long step) {
-    RowIDAllocator allocator =
-        new RowIDAllocator(table.getMaxShardRowIDBits(), dbId, step, session);
-    if (unsigned) {
-      allocator.initUnsigned(session.createSnapshot(), table.getId(), table.getMaxShardRowIDBits());
-    } else {
-      allocator.initSigned(session.createSnapshot(), table.getId(), table.getMaxShardRowIDBits());
-    }
-
-    return allocator;
-  }
-
-  public long getStart() {
-    return end - step;
-  }
-
-  public long getEnd() {
-    return end;
-  }
-
-  // set key value pair to tikv via two phase committer protocol.
-  private void set(ByteString key, byte[] value) {
-    TwoPhaseCommitter twoPhaseCommitter =
-        new TwoPhaseCommitter(session, session.getTimestamp().getVersion());
-
-    twoPhaseCommitter.prewritePrimaryKey(
-        ConcreteBackOffer.newCustomBackOff(BackOffer.PREWRITE_MAX_BACKOFF),
-        key.toByteArray(),
-        value);
-
-    twoPhaseCommitter.commitPrimaryKey(
-        ConcreteBackOffer.newCustomBackOff(BackOffer.BATCH_COMMIT_BACKOFF),
-        key.toByteArray(),
-        session.getTimestamp().getVersion());
-
-    try {
-      twoPhaseCommitter.close();
-    } catch (Throwable ignored) {
-    }
-  }
-
-  private void updateMeta(ByteString key, byte[] oldVal, Snapshot snapshot) {
-    // 1. encode hash meta key
-    // 2. load meta via hash meta key from TiKV
-    // 3. update meta's filed count and set it back to TiKV
-    CodecDataOutput cdo = new CodecDataOutput();
-    ByteString metaKey = MetaCodec.encodeHashMetaKey(cdo, key.toByteArray());
-    long fieldCount;
-    ByteString metaVal = snapshot.get(metaKey);
-
-    // decode long from bytes
-    // big endian the 8 bytes
-    fieldCount = new CodecDataInput(metaVal.toByteArray()).readLong();
-
-    // update meta field count only oldVal is null
-    if (oldVal == null || oldVal.length == 0) {
-      fieldCount++;
-      cdo.reset();
-      cdo.writeLong(fieldCount);
-
-      set(metaKey, cdo.toBytes());
-    }
-  }
-
-  private long updateHash(
-      ByteString key,
-      ByteString field,
-      Function<byte[], byte[]> calculateNewVal,
-      Snapshot snapshot) {
-    // 1. encode hash data key
-    // 2. get value in byte from get operation
-    // 3. calculate new value via calculateNewVal
-    // 4. check old value equals to new value or not
-    // 5. set the new value back to TiKV via 2pc
-    // 6. encode a hash meta key
-    // 7. update a hash meta field count if needed
-
-    CodecDataOutput cdo = new CodecDataOutput();
-    MetaCodec.encodeHashDataKey(cdo, key.toByteArray(), field.toByteArray());
-    ByteString dataKey = cdo.toByteString();
-    byte[] oldVal = snapshot.get(dataKey.toByteArray());
-
-    byte[] newVal = calculateNewVal.apply(oldVal);
-    if (Arrays.equals(newVal, oldVal)) {
-      // not need to update
-      return 0L;
-    }
-
-    set(dataKey, newVal);
-    updateMeta(key, oldVal, snapshot);
-    return Long.parseLong(new String(newVal));
-  }
-
-  private static boolean isDBExisted(long dbId, Snapshot snapshot) {
-    ByteString dbKey = MetaCodec.encodeDatabaseID(dbId);
-    ByteString json = MetaCodec.hashGet(MetaCodec.KEY_DBs, dbKey, snapshot);
-    return json != null && !json.isEmpty();
-  }
-
-  private static boolean isTableExisted(long dbId, long tableId, Snapshot snapshot) {
-    ByteString dbKey = MetaCodec.encodeDatabaseID(dbId);
-    ByteString tableKey = MetaCodec.tableKey(tableId);
-    return !MetaCodec.hashGet(dbKey, tableKey, snapshot).isEmpty();
-  }
-
-  public static boolean shardRowBitsOverflow(
-      long base, long step, long shardRowBits, boolean reservedSignBit) {
-    long signBit = reservedSignBit ? 1 : 0;
-    long mask = ((1L << shardRowBits) - 1) << (64 - shardRowBits - signBit);
-    if (reservedSignBit) {
-      return ((base + step) & mask) > 0;
-    } else {
-      return Long.compareUnsigned((base + step) & mask, 0) > 0;
-    }
-  }
-
-  /**
-   * read current row id from TiKV and write the calculated value back to TiKV. The calculation rule
-   * is start(read from TiKV) + step.
-   */
-  public long udpateAllocateId(
-      long dbId, long tableId, long step, Snapshot snapshot, long shard, boolean hasSignedBit) {
-    if (isDBExisted(dbId, snapshot) && isTableExisted(dbId, tableId, snapshot)) {
-      return updateHash(
-          MetaCodec.encodeDatabaseID(dbId),
-          MetaCodec.autoTableIDKey(tableId),
-          (oldVal) -> {
-            long base = 0;
-            if (oldVal != null && oldVal.length != 0) {
-              base = Long.parseLong(new String(oldVal));
-            }
-            if (shard >= 1 && shardRowBitsOverflow(base, step, shard, hasSignedBit)) {
-              throw new AllocateRowIDOverflowException(base, step, shard);
-            }
-            base += step;
-            return String.valueOf(base).getBytes();
-          },
-          snapshot);
-    }
-
-    throw new IllegalArgumentException("table or database is not existed");
-  }
-
-  /** read current row id from TiKV according to database id and table id. */
-  public static long getAllocateId(long dbId, long tableId, Snapshot snapshot) {
-    if (isDBExisted(dbId, snapshot) && isTableExisted(dbId, tableId, snapshot)) {
-      ByteString dbKey = MetaCodec.encodeDatabaseID(dbId);
-      ByteString tblKey = MetaCodec.autoTableIDKey(tableId);
-      ByteString val = MetaCodec.hashGet(dbKey, tblKey, snapshot);
-      if (val.isEmpty()) return 0L;
-      return Long.parseLong(val.toStringUtf8());
-    }
-
-    throw new IllegalArgumentException("table or database is not existed");
-  }
-
-  private void initSigned(Snapshot snapshot, long tableId, long shard) {
-    // get new start from TiKV, and calculate new end and set it back to TiKV.
-    long newStart = getAllocateId(dbId, tableId, snapshot);
-    long tmpStep = Math.min(Long.MAX_VALUE - newStart, step);
-    if (tmpStep != step) {
-      throw new TiBatchWriteException("cannot allocate ids for this write");
-    }
-    if (newStart == Long.MAX_VALUE) {
-      throw new TiBatchWriteException("cannot allocate more ids since it ");
-    }
-    end = udpateAllocateId(dbId, tableId, tmpStep, snapshot, shard, true);
-  }
-
-  private void initUnsigned(Snapshot snapshot, long tableId, long shard) {
-    // get new start from TiKV, and calculate new end and set it back to TiKV.
-    long newStart = getAllocateId(dbId, tableId, snapshot);
-    // for unsigned long, -1L is max value.
-    long tmpStep = UnsignedLongs.min(-1L - newStart, step);
-    if (tmpStep != step) {
-      throw new TiBatchWriteException("cannot allocate ids for this write");
-    }
-    // when compare unsigned long, the min value is largest value.
-    if (UnsignedLongs.compare(newStart, -1L) == 0) {
-      throw new TiBatchWriteException(
-          "cannot allocate more ids since the start reaches " + "unsigned long's max value ");
-    }
-    end = udpateAllocateId(dbId, tableId, tmpStep, snapshot, shard, false);
-  }
-}

src/main/java/org/tikv/common/apiversion/CodecUtils.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import org.tikv.common.codec.Codec.BytesCodec;
+import org.tikv.common.codec.CodecDataInput;
+import org.tikv.common.codec.CodecDataOutput;
+
+// TODO(iosmanthus): use ByteString.wrap to avoid once more copying.
+public class CodecUtils {
+  public static ByteString encode(ByteString key) {
+    CodecDataOutput cdo = new CodecDataOutput();
+    BytesCodec.writeBytes(cdo, key.toByteArray());
+    return cdo.toByteString();
+  }
+
+  public static ByteString decode(ByteString key) {
+    return ByteString.copyFrom(BytesCodec.readBytes(new CodecDataInput(key)));
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyCodec.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Kvrpcpb.KvPair;
+import org.tikv.kvproto.Kvrpcpb.Mutation;
+import org.tikv.kvproto.Metapb;
+import org.tikv.kvproto.Pdpb;
+
+public interface RequestKeyCodec {
+  ByteString encodeKey(ByteString key);
+
+  default List<ByteString> encodeKeys(List<ByteString> keys) {
+    return keys.stream().map(this::encodeKey).collect(Collectors.toList());
+  }
+
+  default List<Mutation> encodeMutations(List<Mutation> mutations) {
+    return mutations
+        .stream()
+        .map(mut -> Mutation.newBuilder().mergeFrom(mut).setKey(encodeKey(mut.getKey())).build())
+        .collect(Collectors.toList());
+  }
+
+  ByteString decodeKey(ByteString key);
+
+  default KvPair decodeKvPair(KvPair pair) {
+    return KvPair.newBuilder().mergeFrom(pair).setKey(decodeKey(pair.getKey())).build();
+  }
+
+  default List<KvPair> decodeKvPairs(List<KvPair> pairs) {
+    return pairs.stream().map(this::decodeKvPair).collect(Collectors.toList());
+  }
+
+  Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end);
+
+  ByteString encodePdQuery(ByteString key);
+
+  Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end);
+
+  Metapb.Region decodeRegion(Metapb.Region region);
+
+  default List<Pdpb.Region> decodePdRegions(List<Pdpb.Region> regions) {
+    return regions
+        .stream()
+        .map(
+            r ->
+                Pdpb.Region.newBuilder()
+                    .mergeFrom(r)
+                    .setRegion(this.decodeRegion(r.getRegion()))
+                    .build())
+        .collect(Collectors.toList());
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV1Codec.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import java.util.List;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Kvrpcpb.KvPair;
+import org.tikv.kvproto.Kvrpcpb.Mutation;
+import org.tikv.kvproto.Metapb.Region;
+import org.tikv.kvproto.Pdpb;
+
+public class RequestKeyV1Codec implements RequestKeyCodec {
+  @Override
+  public ByteString encodeKey(ByteString key) {
+    return key;
+  }
+
+  @Override
+  public List<ByteString> encodeKeys(List<ByteString> keys) {
+    return keys;
+  }
+
+  @Override
+  public List<Mutation> encodeMutations(List<Mutation> mutations) {
+    return mutations;
+  }
+
+  @Override
+  public ByteString decodeKey(ByteString key) {
+    return key;
+  }
+
+  @Override
+  public KvPair decodeKvPair(KvPair pair) {
+    return pair;
+  }
+
+  @Override
+  public List<KvPair> decodeKvPairs(List<KvPair> pairs) {
+    return pairs;
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end) {
+    return Pair.create(start, end);
+  }
+
+  @Override
+  public ByteString encodePdQuery(ByteString key) {
+    return key;
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
+    return Pair.create(start, end);
+  }
+
+  @Override
+  public Region decodeRegion(Region region) {
+    return region;
+  }
+
+  @Override
+  public List<Pdpb.Region> decodePdRegions(List<Pdpb.Region> regions) {
+    return regions;
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV1RawCodec.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+public class RequestKeyV1RawCodec extends RequestKeyV1Codec implements RequestKeyCodec {
+  public RequestKeyV1RawCodec() {}
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV1TxnCodec.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Metapb;
+
+public class RequestKeyV1TxnCodec extends RequestKeyV1Codec implements RequestKeyCodec {
+  public RequestKeyV1TxnCodec() {}
+
+  @Override
+  public ByteString encodePdQuery(ByteString key) {
+    return CodecUtils.encode(key);
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
+    if (!start.isEmpty()) {
+      start = CodecUtils.encode(start);
+    }
+
+    if (!end.isEmpty()) {
+      end = CodecUtils.encode(end);
+    }
+
+    return Pair.create(start, end);
+  }
+
+  @Override
+  public Metapb.Region decodeRegion(Metapb.Region region) {
+    Metapb.Region.Builder builder = Metapb.Region.newBuilder().mergeFrom(region);
+    ByteString start = region.getStartKey();
+    ByteString end = region.getEndKey();
+
+    if (!start.isEmpty()) {
+      start = CodecUtils.decode(start);
+    }
+
+    if (!end.isEmpty()) {
+      end = CodecUtils.decode(end);
+    }
+
+    return builder.setStartKey(start).setEndKey(end).build();
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV2Codec.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+import com.google.protobuf.ByteString;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Metapb;
+import org.tikv.kvproto.Metapb.Region;
+
+public class RequestKeyV2Codec implements RequestKeyCodec {
+  protected static final ByteString RAW_DEFAULT_PREFIX =
+      ByteString.copyFrom(new byte[] {'r', 0, 0, 0});
+  protected static final ByteString RAW_DEFAULT_END =
+      ByteString.copyFrom(new byte[] {'r', 0, 0, 1});
+  protected static final ByteString TXN_DEFAULT_PREFIX =
+      ByteString.copyFrom(new byte[] {'x', 0, 0, 0});
+  protected static final ByteString TXN_DEFAULT_END =
+      ByteString.copyFrom(new byte[] {'x', 0, 0, 1});
+  protected ByteString keyPrefix;
+  protected ByteString infiniteEndKey;
+
+  @Override
+  public ByteString encodeKey(ByteString key) {
+    return keyPrefix.concat(key);
+  }
+
+  @Override
+  public ByteString decodeKey(ByteString key) {
+    if (key.isEmpty()) {
+      return key;
+    }
+
+    if (!key.startsWith(keyPrefix)) {
+      throw new IllegalArgumentException("key corrupted, wrong prefix");
+    }
+
+    return key.substring(keyPrefix.size());
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end) {
+    start = encodeKey(start);
+
+    end = end.isEmpty() ? infiniteEndKey : encodeKey(end);
+
+    return Pair.create(start, end);
+  }
+
+  @Override
+  public ByteString encodePdQuery(ByteString key) {
+    return CodecUtils.encode(encodeKey(key));
+  }
+
+  @Override
+  public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
+    Pair<ByteString, ByteString> range = encodeRange(start, end);
+    return Pair.create(CodecUtils.encode(range.first), CodecUtils.encode(range.second));
+  }
+
+  @Override
+  public Region decodeRegion(Region region) {
+    Metapb.Region.Builder builder = Metapb.Region.newBuilder().mergeFrom(region);
+
+    ByteString start = region.getStartKey();
+    ByteString end = region.getEndKey();
+
+    if (!start.isEmpty()) {
+      start = CodecUtils.decode(start);
+    }
+
+    if (!end.isEmpty()) {
+      end = CodecUtils.decode(end);
+    }
+
+    if (ByteString.unsignedLexicographicalComparator().compare(start, infiniteEndKey) >= 0
+        || (!end.isEmpty()
+            && ByteString.unsignedLexicographicalComparator().compare(end, keyPrefix) <= 0)) {
+      throw new IllegalArgumentException("region out of keyspace" + region.toString());
+    }
+
+    start = start.startsWith(keyPrefix) ? start.substring(keyPrefix.size()) : ByteString.EMPTY;
+    end = end.startsWith(keyPrefix) ? end.substring(keyPrefix.size()) : ByteString.EMPTY;
+
+    return builder.setStartKey(start).setEndKey(end).build();
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV2RawCodec.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+public class RequestKeyV2RawCodec extends RequestKeyV2Codec {
+  public RequestKeyV2RawCodec() {
+    super();
+
+    this.keyPrefix = RAW_DEFAULT_PREFIX;
+    this.infiniteEndKey = RAW_DEFAULT_END;
+  }
+}

src/main/java/org/tikv/common/apiversion/RequestKeyV2TxnCodec.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2022 TiKV Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.common.apiversion;
+
+public class RequestKeyV2TxnCodec extends RequestKeyV2Codec {
+  public RequestKeyV2TxnCodec() {
+    super();
+
+    this.keyPrefix = TXN_DEFAULT_PREFIX;
+    this.infiniteEndKey = TXN_DEFAULT_END;
+  }
+}

src/main/java/org/tikv/common/catalog/Catalog.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.catalog;

src/main/java/org/tikv/common/catalog/CatalogTransaction.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.catalog;
@@ -38,6 +40,7 @@ import org.tikv.common.util.Pair;
 
 public class CatalogTransaction {
   protected static final Logger logger = LoggerFactory.getLogger(CatalogTransaction.class);
+  private static final ObjectMapper mapper = new ObjectMapper();
   private final Snapshot snapshot;
 
   CatalogTransaction(Snapshot snapshot) {
@@ -49,7 +52,6 @@ public class CatalogTransaction {
     Objects.requireNonNull(cls, "cls is null");
 
     logger.debug(String.format("Parse Json %s : %s", cls.getSimpleName(), json.toStringUtf8()));
-    ObjectMapper mapper = new ObjectMapper();
     try {
       return mapper.readValue(json.toStringUtf8(), cls);
     } catch (JsonParseException | JsonMappingException e) {

src/main/java/org/tikv/common/codec/Codec.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/CodecDataInput.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/CodecDataInputLittleEndian.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/CodecDataOutput.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/CodecDataOutputLittleEndian.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2020 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;

src/main/java/org/tikv/common/codec/KeyUtils.java

@@ -1,16 +1,18 @@
 /*
- * Copyright 2017 PingCAP, Inc.
+ * Copyright 2021 TiKV Project Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package org.tikv.common.codec;