Compare commits

..

17 Commits

Author SHA1 Message Date
birdstorm 41b24bb877 release v3.1.0
Signed-off-by: birdstorm <samuelwyf@hotmail.com>
2021-06-24 13:11:11 +08:00
ti-srebot 05f1559eab
cherry pick #207 to release-3.1 (#210)
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: Wallace <bupt2013211450@gmail.com>
Co-authored-by: birdstorm <samuelwyf@hotmail.com>
2021-06-23 21:53:42 +08:00
ti-srebot c7d9ff151b
pd-client grpc forward (#203) (#206)
* pd-client grpc forward (#203)

* forward pd leader

Signed-off-by: Little-Wallace <bupt2013211450@gmail.com>
Signed-off-by: birdstorm <samuelwyf@hotmail.com>

* fix test

Signed-off-by: birdstorm <samuelwyf@hotmail.com>

Co-authored-by: Wallace <bupt2013211450@gmail.com>
Co-authored-by: birdstorm <samuelwyf@hotmail.com>
2021-06-23 16:21:17 +08:00
ti-srebot 3aa9a5665b
Support grpc forward (#198) (#208)
* Support grpc forward (#198)

* support grpc forward for tikv client

Signed-off-by: Little-Wallace <bupt2013211450@gmail.com>
Signed-off-by: birdstorm <samuelwyf@hotmail.com>

* fix compile

Signed-off-by: birdstorm <samuelwyf@hotmail.com>

Co-authored-by: Wallace <bupt2013211450@gmail.com>
Co-authored-by: birdstorm <samuelwyf@hotmail.com>
2021-06-23 15:43:37 +08:00
ti-srebot 35493c468a
cherry pick #151 to release-3.1 (#211)
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: Liangliang Gu <marsishandsome@gmail.com>
2021-06-23 15:14:22 +08:00
ti-srebot 031745b41b
refactor follower read (#126) (#209)
* cherry pick #126 to release-3.1

Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

* fix conflict

Signed-off-by: birdstorm <samuelwyf@hotmail.com>

Co-authored-by: Liangliang Gu <marsishandsome@gmail.com>
Co-authored-by: birdstorm <samuelwyf@hotmail.com>
2021-06-23 15:02:51 +08:00
ti-srebot 7f468277c3
cherry pick #199 to release-3.1 (#200)
Signed-off-by: birdstorm <samuelwyf@hotmail.com>

Co-authored-by: birdstorm <samuelwyf@hotmail.com>
2021-06-18 11:48:17 +08:00
ti-srebot 71de93d73d
cherry pick #185 to release-3.1 (#201)
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: Lifu Wu <purelind@users.noreply.github.com>
2021-06-17 16:31:02 +08:00
ti-srebot 2913008410
cherry pick #182 to release-3.1 (#189)
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: birdstorm <samuelwyf@hotmail.com>
2021-06-09 16:10:34 +08:00
ti-srebot 6a5ea6fb8b
cherry pick #183 to release-3.1 (#184)
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: birdstorm <samuelwyf@hotmail.com>
2021-06-07 10:02:41 +08:00
ti-srebot 01b391ff2f
cherry pick #172 to release-3.1 (#175)
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: birdstorm <samuelwyf@hotmail.com>
2021-06-04 12:24:23 +08:00
birdstorm 490b4a1e01
invalidate all region cache on store when store id not found (#170)
Signed-off-by: birdstorm <samuelwyf@hotmail.com>
2021-05-20 15:39:31 +08:00
ti-srebot c6398badfe
cherry pick #167 to release-3.1 (#168)
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: Qishang Zhong <zhongqishang@gmail.com>
2021-04-26 11:16:31 +08:00
ti-srebot 4db2ac1ad7
Remove unused files (#165) #166
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: tison <wander4096@gmail.com>
2021-04-19 16:02:23 +08:00
ti-srebot cd3ddc121a
cherry pick #160 to release-3.1 (#161) 2021-04-01 19:03:03 +08:00
birdstorm 06da8d0830
add issue and pr template (#157) (#158)
Signed-off-by: birdstorm <samuelwyf@hotmail.com>
2021-03-30 13:53:19 +08:00
birdstorm 131693bf8a
Update version to 3.1.0-SNAPSHAOT (#152) (#156)
Signed-off-by: birdstorm <samuelwyf@hotmail.com>
2021-03-29 16:21:00 +08:00
395 changed files with 4286 additions and 19324 deletions

63
.ci/build.groovy Normal file
View File

@ -0,0 +1,63 @@
def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) {
catchError {
node ('build') {
container("java") {
stage('Prepare') {
dir("/home/jenkins/agent/git/client-java") {
sh """
rm -rf /maven/.m2/repository/*
rm -rf /maven/.m2/settings.xml
rm -rf ~/.m2/settings.xml
archive_url=http://fileserver.pingcap.net/download/builds/pingcap/client-java/cache/tikv-client-java-m2-cache-latest.tar.gz
if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -sL \$archive_url | tar -zx -C /maven || true; fi
"""
if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) {
deleteDir()
}
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: '+refs/pull/*:refs/remotes/origin/pr/*', url: 'git@github.com:tikv/client-java.git']]]
sh "git checkout -f ${ghprbActualCommit}"
}
}
stage('Format') {
dir("/home/jenkins/agent/git/client-java") {
sh """
mvn com.coveo:fmt-maven-plugin:format
git diff --quiet
formatted="\$?"
if [[ "\${formatted}" -eq 1 ]]
then
echo "code format error, please run the following commands:"
echo " mvn com.coveo:fmt-maven-plugin:format"
exit 1
fi
"""
}
}
stage('Build') {
dir("/home/jenkins/agent/git/client-java") {
timeout(30) {
sh ".ci/build.sh"
}
}
}
}
}
currentBuild.result = "SUCCESS"
}
stage('Summary') {
def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" +
"${ghprbPullLink}" + "\n" +
"${ghprbPullDescription}" + "\n" +
"Build Result: `${currentBuild.result}`" + "\n" +
"Elapsed Time: `${duration} mins` " + "\n" +
"${env.RUN_DISPLAY_URL}"
print msg
}
}
return this

6
.ci/build.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
set -x
set -euo pipefail
mvn clean compile

115
.ci/integration_test.groovy Normal file
View File

@ -0,0 +1,115 @@
def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) {
def TIDB_BRANCH = "release-4.0"
def TIKV_BRANCH = "release-4.0"
def PD_BRANCH = "release-4.0"
// parse tidb branch
def m1 = ghprbCommentBody =~ /tidb\s*=\s*([^\s\\]+)(\s|\\|$)/
if (m1) {
TIDB_BRANCH = "${m1[0][1]}"
}
m1 = null
println "TIDB_BRANCH=${TIDB_BRANCH}"
// parse pd branch
def m2 = ghprbCommentBody =~ /pd\s*=\s*([^\s\\]+)(\s|\\|$)/
if (m2) {
PD_BRANCH = "${m2[0][1]}"
}
m2 = null
println "PD_BRANCH=${PD_BRANCH}"
// parse tikv branch
def m3 = ghprbCommentBody =~ /tikv\s*=\s*([^\s\\]+)(\s|\\|$)/
if (m3) {
TIKV_BRANCH = "${m3[0][1]}"
}
m3 = null
println "TIKV_BRANCH=${TIKV_BRANCH}"
catchError {
node ('build') {
container("java") {
stage('Prepare') {
dir("/home/jenkins/agent/git/client-java") {
sh """
rm -rf /maven/.m2/repository/*
rm -rf /maven/.m2/settings.xml
rm -rf ~/.m2/settings.xml
archive_url=http://fileserver.pingcap.net/download/builds/pingcap/client-java/cache/tikv-client-java-m2-cache-latest.tar.gz
if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -sL \$archive_url | tar -zx -C /maven || true; fi
"""
if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) {
deleteDir()
}
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: '+refs/pull/*:refs/remotes/origin/pr/*', url: 'git@github.com:tikv/client-java.git']]]
sh "git checkout -f ${ghprbActualCommit}"
}
dir("/home/jenkins/agent/git/client-java/_run") {
// tidb
def tidb_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tidb/${TIDB_BRANCH}/sha1").trim()
sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz | tar xz"
// tikv
def tikv_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tikv/${TIKV_BRANCH}/sha1").trim()
sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz | tar xz"
// pd
def pd_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/pd/${PD_BRANCH}/sha1").trim()
sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz | tar xz"
sh """
killall -9 tidb-server || true
killall -9 tikv-server || true
killall -9 pd-server || true
killall -9 java || true
sleep 10
bin/pd-server --name=pd --data-dir=pd --config=../config/pd.toml &>pd.log &
sleep 10
bin/tikv-server --pd=127.0.0.1:2379 -s tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 --config=../config/tikv.toml &>tikv.log &
sleep 10
ps aux | grep '-server' || true
curl -s 127.0.0.1:2379/pd/api/v1/status || true
bin/tidb-server --store=tikv --path="127.0.0.1:2379" --config=../config/tidb.toml &>tidb.log &
sleep 60
"""
}
}
stage('Test') {
dir("/home/jenkins/agent/git/client-java") {
try {
timeout(30) {
sh ".ci/test.sh"
}
} catch (err) {
sh """
ps aux | grep '-server' || true
curl -s 127.0.0.1:2379/pd/api/v1/status || true
"""
sh "cat _run/pd.log"
sh "cat _run/tikv.log"
sh "cat _run/tidb.log"
throw err
}
}
}
}
}
currentBuild.result = "SUCCESS"
}
stage('Summary') {
def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" +
"${ghprbPullLink}" + "\n" +
"${ghprbPullDescription}" + "\n" +
"Integration Common Test Result: `${currentBuild.result}`" + "\n" +
"Elapsed Time: `${duration} mins` " + "\n" +
"${env.RUN_DISPLAY_URL}"
print msg
}
}
return this

6
.ci/test.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
set -x
set -euo pipefail
mvn clean test

View File

@ -1,24 +1,36 @@
---
name: "\U0001F41B Bug Report"
about: As a User, I want to report a Bug.
name: Bug report
about: Create a report to help us improve
title: "[BUG] Title of Bug Report"
labels: type/bug
assignees: ''
---
## Bug Report
### 1. Describe the bug
**Describe the bug**
<!-- A clear and concise description of what the bug is. -->
### 2. Minimal reproduce step (Required)
**What did you do**
<!--
If possible, please provide a code receipt to produce this issue.
-->
<!-- a step by step guide for reproducing the bug. -->
**What do you expect**
<!-- A clear and concise description of what you expected to happen. -->
### 3. What did you see instead (Required)
**What happens instead**
<!-- If an error occurs, please provide complete error stack. -->
### 4. What did you expect to see? (Required)
<!--
**Screenshots**
If applicable, add screenshots to help explain your problem.
-->
### 5. What are your Java Client and TiKV versions? (Required)
**Java Client and TiDB/TiKV version info**
<!-- What version of Spark and TiSpark are you using? (Provide Spark version and run `spark.sql(“select ti_version()”).show(false)` in spark-shell) -->
- Client Java:
- TiKV:
<!--
**Additional context**
Add any other context about the problem here.
You may also provide TiDB version here if it is related to the issue.
-->

View File

@ -1,23 +1,20 @@
---
name: "\U0001F680 Feature Request"
about: As a user, I want to request a New Feature on the product.
name: Feature request
about: Suggest an idea for this project
title: ''
labels: type/feature-request
assignees: ''
---
## Feature Request
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Is your feature request related to a problem? Please describe:**
<!-- A clear and concise description of what the problem is. Ex. I'm always
frustrated when [...] -->
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe the feature you'd like:**
<!-- A clear and concise description of what you want to happen. -->
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Describe alternatives you've considered:**
<!-- A clear and concise description of any alternative solutions or features
you've considered. -->
**Teachability, Documentation, Adoption, Migration Strategy:**
<!-- If you can, explain some scenarios of how users might use this, situations it
would be helpful in. Any API designs, mockups, or diagrams are also helpful.
-->
**Additional context**
Add any other context or screenshots about the feature request here.

View File

@ -1,21 +0,0 @@
# TiKV Configuration.
[raftstore]
# set store capacity, if no set, use disk capacity.
capacity = "6G"
pd-heartbeat-tick-interval = "2s"
pd-store-heartbeat-tick-interval = "5s"
split-region-check-tick-interval = "1s"
[rocksdb]
max-open-files = 10000
[raftdb]
max-open-files = 10000
[storage.block-cache]
capacity = "128MB"
[storage]
reserve-space = "0MB"
enable-ttl = true

View File

@ -1,20 +0,0 @@
# TiKV Configuration.
[raftstore]
# set store capacity, if no set, use disk capacity.
capacity = "6G"
pd-heartbeat-tick-interval = "2s"
pd-store-heartbeat-tick-interval = "5s"
split-region-check-tick-interval = "1s"
[rocksdb]
max-open-files = 10000
[raftdb]
max-open-files = 10000
[storage.block-cache]
capacity = "128MB"
[storage]
reserve-space = "0MB"

View File

@ -1,17 +0,0 @@
# TiKV Configuration.
[raftstore]
pd-heartbeat-tick-interval = "2s"
pd-store-heartbeat-tick-interval = "5s"
split-region-check-tick-interval = "1s"
[rocksdb]
max-open-files = 10000
[raftdb]
max-open-files = 10000
[storage]
reserve-space = "0MB"
api-version = 2
enable-ttl = true

17
.github/labeler.yml vendored
View File

@ -1,17 +0,0 @@
component/config:
- config/*
component/br:
- src/main/java/org/tikv/br/*
component/cdc:
- src/main/java/org/tikv/cdc/*
component/common:
- src/main/java/org/tikv/common/*
component/rawkv:
- src/main/java/org/tikv/raw/*
component/txnkv:
- src/main/java/org/tikv/txn/*

View File

@ -1,26 +0,0 @@
header:
license:
spdx-id: Apache-2.0
copyright-owner: TiKV Project Authors.
paths-ignore:
- 'config/'
- 'dev/'
- 'docs/'
- 'metrics/'
- 'LICENSE'
- 'Makefile'
- 'pom.xml'
- 'shell.nix'
- '.ci/'
- '.gitignore'
- '.gitattributes'
- '.github/'
- '**/*.md'
- '**/*.properties'
- '**/*.json'
- '**/*.pem'
- '**/*.crt'
- '**/*.g4'
- 'src/main/java/io/grpc/'
- 'src/main/java/io/netty/'
comment: on-failure

View File

@ -1,46 +1,33 @@
<!-- Thank you for contributing to TiKV Java Client!
PR Title Format: "[close/to/fix #issue_number] summary" -->
### What problem does this PR solve?
Issue Number: close #issue_number
Problem Description: **TBD**
### What is changed and how does it work?
### What problem does this PR solve? <!--add issue link with summary if exists-->
### What is changed and how it works?
### Code changes
<!-- REMOVE the items that are not applicable -->
- Has exported function/method change
- Has exported variable/fields change
- Has methods of interface change
- Has persistent data change
- No code
### Check List <!--REMOVE the items that are not applicable-->
### Check List for Tests
Tests <!-- At least one of them must be included. -->
This PR has been tested by at least one of the following methods:
- Unit test
- Integration test
- Manual test (add detailed scripts or steps below)
- No code
- Unit test
- Integration test
- Manual test (add detailed scripts or steps below)
- No code
### Side effects
Code changes
<!-- REMOVE the items that are not applicable -->
- Possible performance regression, WHY: **TBD**
- Increased code complexity, WHY: **TBD**
- Breaking backward compatibility, WHY: **TBD**
- NO side effects
- Has exported function/method change
- Has exported variable/fields change
- Has interface methods change
- Has persistent data change
### Related changes
Side effects
<!-- REMOVE the items that are not applicable -->
- Need to cherry-pick to the release branch
- Need to update the documentation
- Need to be included in the release note
- NO related changes
- Possible performance regression
- Increased code complexity
- Breaking backward compatibility
Related changes
- Need to cherry-pick to the release branch
- Need to update the documentation
- Need to be included in the release note

View File

@ -1,84 +0,0 @@
name: CI
on:
pull_request:
push:
branches:
- master
jobs:
format:
name: Check Format
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up JDK 8
uses: actions/setup-java@v2
with:
java-version: '8.0'
distribution: 'adopt'
- run: |
./dev/javafmt
if [[ $(git diff) != "" ]]
then
echo "code format error, please run the following commands:"
echo " ./dev/javafmt"
exit 1
fi
integration-test:
name: Integration Test - ${{ matrix.tikv_version }}
runs-on: ubuntu-latest
strategy:
matrix:
tikv_version: [v5.0.6, v5.3.4, v5.4.3]
fail-fast: false
steps:
- uses: actions/checkout@v2
- name: Set up JDK 8
uses: actions/setup-java@v2
with:
java-version: '8.0'
distribution: 'adopt'
- name: Install TiUP
run: |
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
/home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
- name: Start TiUP Playground
run: |
# Start TiKV in APIV1TTL
touch tiup-v1ttl.log
/home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag rawkv --mode tikv-slim --kv 1 --without-monitor --kv.port 20160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup-v1ttl.log &
timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1ttl.log)
cat tiup-v1ttl.log
echo "Wait for bootstrap"
sleep 10s
# Start TiKV in APIV1
touch tiup-v1.log
/home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag txnkv --mode tikv-slim --kv 1 --without-monitor --kv.port 30160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2381 2>&1 >> tiup-v1.log &
timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1.log)
cat tiup-v1.log
echo "Wait for bootstrap"
sleep 10s
# Get PD address
echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
echo "TXNKV_PD_ADDRESSES=127.0.0.1:2381" >> $GITHUB_ENV
- name: Run Integration Test
run: mvn clean test
- name: Print TiKV logs
if: failure()
run: |
echo "RawKV TiKV logs"
cat /home/runner/.tiup/data/rawkv/tikv-0/tikv.log
echo "TxnKV TiKV logs"
cat /home/runner/.tiup/data/txnkv/tikv-0/tikv.log
- name: Upload coverage
uses: codecov/codecov-action@v2
with:
files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
fail_ci_if_error: true
verbose: true

View File

@ -1,52 +0,0 @@
name: CI (APIv2)
on:
pull_request:
push:
branches:
- master
jobs:
integration-test:
name: Integration Test - ${{ matrix.tikv_version }}
runs-on: ubuntu-latest
strategy:
matrix:
tikv_version: [v6.5.3, v7.1.1, nightly]
fail-fast: false
steps:
- uses: actions/checkout@v2
- name: Set up JDK 8
uses: actions/setup-java@v2
with:
java-version: '8.0'
distribution: 'adopt'
- name: Install TiUP
run: |
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
/home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
- name: Start TiUP Playground
run: |
# Start TiKV in APIV2
touch tiup.log
/home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --tag kv --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_v2.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup.log &
timeout 300 grep -q "PD Endpoints:" <(tail -f tiup.log)
cat tiup.log
# Get PD address
echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
echo "TXNKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
- name: Run Integration Test
run: mvn clean test
- name: Print TiKV logs
if: failure()
run: |
echo "TiKV logs"
cat /home/runner/.tiup/data/kv/tikv-0/tikv.log
- name: Upload coverage
uses: codecov/codecov-action@v2
with:
files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
fail_ci_if_error: true
verbose: true

View File

@ -1,38 +0,0 @@
name: GitHub Pages
on:
push:
branches:
- master
jobs:
deploy:
runs-on: ubuntu-20.04
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
steps:
- uses: actions/checkout@v2
- name: Setup mdBook
uses: peaceiris/actions-mdbook@v1
with:
mdbook-version: '0.4.8'
# mdbook-version: 'latest'
- run: mdbook build ./docs
- name: Set up JDK 8
uses: actions/setup-java@v2
with:
java-version: '8'
distribution: 'adopt'
- name: Maven site
run: mvn clean compile site && mv ./target/site ./docs/book/site
- name: Deploy
uses: peaceiris/actions-gh-pages@v3
if: ${{ github.ref == 'refs/heads/master' }}
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./docs/book

View File

@ -1,11 +0,0 @@
name: "PR Labeler"
on:
- pull_request_target
jobs:
triage:
runs-on: ubuntu-latest
steps:
- uses: actions/labeler@v3
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"

View File

@ -1,24 +0,0 @@
name: License checker
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
check-license:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Check License Header
uses: apache/skywalking-eyes@v0.3.0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
log: info
config: .github/license-checker.yml

View File

@ -1,14 +0,0 @@
name: 'PR Title Checker'
on:
pull_request:
types: [edited, opened, synchronize, reopened]
jobs:
title-check:
runs-on: ubuntu-latest
steps:
- uses: naveenk1223/action-pr-title@master
with:
regex: '\[to|fix|close #[0-9]+\] .+'
prefix_case_sensitive: true
max_length: -1

View File

@ -1,30 +0,0 @@
name: Spotbugs
on:
push:
branches:
- master
pull_request_target:
branches:
- master
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- uses: actions/cache@v1
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Build with Maven
run: mvn -B verify spotbugs:spotbugs -Dmaven.test.skip=true
- uses: jwgmeligmeyling/spotbugs-github-action@v1
with:
path: "**/spotbugsXml.xml"

7
.gitignore vendored
View File

@ -5,7 +5,6 @@ pub.sh
# ignore compiled classes
target
.classpath
# ignore version info
src/main/java/com/pingcap/tikv/TiVersion.java
@ -71,9 +70,3 @@ out/
# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898
# gradle/wrapper/gradle-wrapper.properties
# vscode
.settings
# mdbook
docs/book

View File

@ -1,16 +0,0 @@
.DEFAULT: build
.PHONY: clean
build:
mvn clean package -Dmaven.test.skip=true
fmt:
./dev/javafmt
test:
mvn clean test
clean:
mvn clean
rm -rf target tipb raft-rs kvproto

185
README.md
View File

@ -1,89 +1,182 @@
[![Maven Central](https://img.shields.io/maven-central/v/org.tikv/tikv-client-java.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22org.tikv%22%20AND%20a:%22tikv-client-java%22)
[![Slack](https://img.shields.io/badge/chat-on%20Slack-brightgreen.svg)](https://slack.tidb.io/invite?team=tikv-wg&channel=client)
[![codecov](https://codecov.io/gh/tikv/client-java/branch/master/graph/badge.svg?token=nSAjGaN0EH)](https://codecov.io/gh/tikv/client-java)
## TiKV JAVA Client
A Java client for [TiKV](https://github.com/tikv/tikv).
A Java client for [TiDB](https://github.com/pingcap/tidb)/[TiKV](https://github.com/tikv/tikv).
It is supposed to:
+ Communicate via [gRPC](http://www.grpc.io/)
+ Talk to Placement Driver searching for a region
+ Talk to TiKV for reading/writing data
+ Talk to TiKV for reading/writing data and the resulted data is encoded/decoded just like what we do in TiDB.
+ Talk to Coprocessor for calculation pushdown
## Quick Start
## How to build
> TiKV Java Client is designed to communicate with [PD](https://github.com/tikv/pd) and [TiKV](https://github.com/tikv/tikv), please run PD and TiKV in advance.
### Maven
Build Java client from source file:
The alternative way to build a usable jar for testing will be
```sh
```
mvn clean install -Dmaven.test.skip=true
```
Add maven dependency to `pom.xml`:
The following command can install dependencies for you.
```
mvn package
```
The jar can be found in `./target/`
## Usage
This project is designed to hook with `pd` and `tikv` which you can find in `PingCAP` github page.
When you work with this project, you have to communicate with `pd` and `tikv`. Please run TiKV and PD in advance.
## Component: Raw Ti-Client in Java
Java Implementation of Raw TiKV-Client to support RawKVClient commands.
Demo is avaliable in [KVRawClientTest](https://github.com/birdstorm/KVRawClientTest/)
### Build
```
mvn clean install -Dmaven.test.skip=true
```
### Add to dependency
#### Use jar for binary
Add your jar built with all dependencies into you project's library to use `tikv-client-java` as dependency
#### Use as maven dependency
After building, add following lines into your `pom.xml` if you are using Maven
```xml
<dependency>
<groupId>org.tikv</groupId>
<artifactId>tikv-client-java</artifactId>
<version>3.3.0</version>
<version>3.0.0</version>
</dependency>
```
Create a transactional `KVClient` and communicates with TiKV:
### Entrance
`org.tikv.raw.RawKVClient`
### Create a RawKVClient
```java
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
import org.tikv.txn.KVClient;
public class Main {
public static void main(String[] args) throws Exception {
TiConfiguration conf = TiConfiguration.createDefault(YOUR_PD_ADDRESSES);
TiSession session = TiSession.create(conf);
KVClient client = session.createKVClient();
}
}
```
Or create a `RawKVClient` if you don't need the transaction semantic:
```java
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
import org.tikv.raw.RawKVClient;
public class Main {
public static void main(String[] args) throws Exception {
public static void main() {
// You MUST create a raw configuration if you are using RawKVClient.
TiConfiguration conf = TiConfiguration.createRawDefault(YOUR_PD_ADDRESSES);
TiSession session = TiSession.create(conf);
RawKVClient client = session.createRawClient();
RawKVClient = session.createRawKVClient();
}
}
```
Find more demo in [TiKV Java Client User Documents](https://tikv.github.io/client-java/examples/introduction.html)
### API
## Documentation
```java
/**
* Put a raw key-value pair to TiKV
*
* @param key raw key
* @param value raw value
*/
void put(ByteString key, ByteString value)
```
See [Java Client Documents](/docs/README.md) for references about how to config and monitor Java Client.
```java
/**
* Get a raw key-value pair from TiKV if key exists
*
* @param key raw key
* @return a ByteString value if key exists, ByteString.EMPTY if key does not exist
*/
ByteString get(ByteString key)
```
A [Maven site](https://tikv.github.io/client-java/site) is also available. It includes:
1. [API reference](https://tikv.github.io/client-java/site/apidocs/index.html)
2. [Spotbugs Reports](https://tikv.github.io/client-java/site/spotbugs.html)
3. [Source Code Xref](https://tikv.github.io/client-java/site/xref/index.html)
```java
/**
* Scan raw key-value pairs from TiKV in range [startKey, endKey)
*
* @param startKey raw start key, inclusive
* @param endKey raw end key, exclusive
* @param limit limit of key-value pairs scanned, should be less than {@link #MAX_RAW_SCAN_LIMIT}
* @return list of key-value pairs in range
*/
List<Kvrpcpb.KvPair> scan(ByteString startKey, ByteString endKey, int limit)
```
## Community
```java
/**
* Scan raw key-value pairs from TiKV in range [startKey, endKey)
*
* @param startKey raw start key, inclusive
* @param limit limit of key-value pairs scanned, should be less than {@link #MAX_RAW_SCAN_LIMIT}
* @return list of key-value pairs in range
*/
List<Kvrpcpb.KvPair> scan(ByteString startKey, int limit)
```
### Forum
```java
/**
* Delete a raw key-value pair from TiKV if key exists
*
* @param key raw key to be deleted
*/
void delete(ByteString key)
```
- User forum: [AskTUG](https://asktug.com/)
- Contributor forum: [https://internals.tidb.io/](https://internals.tidb.io/)
## Java Client 配置参数
### Contribute to TiKV Java Client
本文介绍了与部署使用 Java Client 相关的配置参数。
### 常用配置 JVM 参数
以下包括常用配置的 JVM 相关参数。
####tikv.pd.addresses
- pd 集群的地址,逗号分隔
- 默认值 127.0.0.1:2379
####tikv.grpc.timeout_in_ms
- grpc 请求的 timeout 时间
- 默认值 600ms
####tikv.grpc.scan_timeout_in_ms
- scan/delete range grpc 请求的 timeout 时间
- 默认值 20s
### ThreadPool 配置 JVM 参数
以下包括 ThreadPool 相关的参数及其默认配置,可通过 JVM 参数传入。
####tikv.batch_get_concurrency
- Client 端 batchGet 请求的线程池大小
- 默认值 20
####tikv.batch_put_concurrency
- Client 端 batchPut 请求的线程池大小
- 默认值 20
####tikv.batch_delete_concurrency
- Client 端 batchDelete 请求的线程池大小
- 默认值 20
####tikv.batch_scan_concurrency
- Client 端 batchScan 请求的线程池大小
- 默认值 5
####tikv.delete_range_concurrency
- Client 端 deleteRange 请求的线程池大小
- 默认值 20
See [Contribution Guide](https://tikv.github.io/client-java/contribution/introduction.html) for references about how to contribute to this project.
## License
Apache 2.0 license. See the [LICENSE](./LICENSE) file for details.

View File

@ -1,4 +1,4 @@
# PD Configuration.
[replication]
enable-placement-rules = true
max-replicas = 1
max-replicas = 1

1
config/tidb.toml Normal file
View File

@ -0,0 +1 @@
# TiDB Configuration.

5
config/tikv.toml Normal file
View File

@ -0,0 +1,5 @@
# TiKV Configuration.
[raftstore]
# set store capacity, if no set, use disk capacity.
capacity = "8G"

View File

@ -1,6 +1,8 @@
# TiKV Java Client Dev Tools
# TiSpark Dev Tools Guide
## Code Formatting
## Formatting
### Java Format
TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https://github.com/coveooss/fmt-maven-plugin) which follows Google's code styleguide. It is also checked on CI before build.
@ -16,4 +18,4 @@ TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https:
```shell script
./dev/javafmt
```
```

View File

@ -2,222 +2,222 @@
<code_scheme name="GoogleStyle">
<option name="OTHER_INDENT_OPTIONS">
<value>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="TAB_SIZE" value="2"/>
<option name="USE_TAB_CHARACTER" value="false"/>
<option name="SMART_TABS" value="false"/>
<option name="LABEL_INDENT_SIZE" value="0"/>
<option name="LABEL_INDENT_ABSOLUTE" value="false"/>
<option name="USE_RELATIVE_INDENTS" value="false"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="2" />
<option name="USE_TAB_CHARACTER" value="false" />
<option name="SMART_TABS" value="false" />
<option name="LABEL_INDENT_SIZE" value="0" />
<option name="LABEL_INDENT_ABSOLUTE" value="false" />
<option name="USE_RELATIVE_INDENTS" value="false" />
</value>
</option>
<option name="INSERT_INNER_CLASS_IMPORTS" value="true"/>
<option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999"/>
<option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999"/>
<option name="INSERT_INNER_CLASS_IMPORTS" value="true" />
<option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
<option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
<option name="PACKAGES_TO_USE_IMPORT_ON_DEMAND">
<value/>
<value />
</option>
<option name="IMPORT_LAYOUT_TABLE">
<value>
<package name="" withSubpackages="true" static="true"/>
<emptyLine/>
<package name="" withSubpackages="true" static="false"/>
<package name="" withSubpackages="true" static="true" />
<emptyLine />
<package name="" withSubpackages="true" static="false" />
</value>
</option>
<option name="RIGHT_MARGIN" value="100"/>
<option name="JD_ALIGN_PARAM_COMMENTS" value="false"/>
<option name="JD_ALIGN_EXCEPTION_COMMENTS" value="false"/>
<option name="JD_P_AT_EMPTY_LINES" value="false"/>
<option name="JD_KEEP_EMPTY_PARAMETER" value="false"/>
<option name="JD_KEEP_EMPTY_EXCEPTION" value="false"/>
<option name="JD_KEEP_EMPTY_RETURN" value="false"/>
<option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false"/>
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="0"/>
<option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
<option name="BLANK_LINES_AFTER_CLASS_HEADER" value="0"/>
<option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
<option name="ALIGN_MULTILINE_FOR" value="false"/>
<option name="CALL_PARAMETERS_WRAP" value="1"/>
<option name="METHOD_PARAMETERS_WRAP" value="1"/>
<option name="EXTENDS_LIST_WRAP" value="1"/>
<option name="THROWS_KEYWORD_WRAP" value="1"/>
<option name="METHOD_CALL_CHAIN_WRAP" value="1"/>
<option name="BINARY_OPERATION_WRAP" value="1"/>
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
<option name="TERNARY_OPERATION_WRAP" value="1"/>
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
<option name="FOR_STATEMENT_WRAP" value="1"/>
<option name="ARRAY_INITIALIZER_WRAP" value="1"/>
<option name="WRAP_COMMENTS" value="true"/>
<option name="IF_BRACE_FORCE" value="3"/>
<option name="DOWHILE_BRACE_FORCE" value="3"/>
<option name="WHILE_BRACE_FORCE" value="3"/>
<option name="FOR_BRACE_FORCE" value="3"/>
<option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true"/>
<option name="RIGHT_MARGIN" value="100" />
<option name="JD_ALIGN_PARAM_COMMENTS" value="false" />
<option name="JD_ALIGN_EXCEPTION_COMMENTS" value="false" />
<option name="JD_P_AT_EMPTY_LINES" value="false" />
<option name="JD_KEEP_EMPTY_PARAMETER" value="false" />
<option name="JD_KEEP_EMPTY_EXCEPTION" value="false" />
<option name="JD_KEEP_EMPTY_RETURN" value="false" />
<option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="0" />
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="BLANK_LINES_AFTER_CLASS_HEADER" value="0" />
<option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
<option name="ALIGN_MULTILINE_FOR" value="false" />
<option name="CALL_PARAMETERS_WRAP" value="1" />
<option name="METHOD_PARAMETERS_WRAP" value="1" />
<option name="EXTENDS_LIST_WRAP" value="1" />
<option name="THROWS_KEYWORD_WRAP" value="1" />
<option name="METHOD_CALL_CHAIN_WRAP" value="1" />
<option name="BINARY_OPERATION_WRAP" value="1" />
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
<option name="TERNARY_OPERATION_WRAP" value="1" />
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
<option name="FOR_STATEMENT_WRAP" value="1" />
<option name="ARRAY_INITIALIZER_WRAP" value="1" />
<option name="WRAP_COMMENTS" value="true" />
<option name="IF_BRACE_FORCE" value="3" />
<option name="DOWHILE_BRACE_FORCE" value="3" />
<option name="WHILE_BRACE_FORCE" value="3" />
<option name="FOR_BRACE_FORCE" value="3" />
<option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true" />
<AndroidXmlCodeStyleSettings>
<option name="USE_CUSTOM_SETTINGS" value="true"/>
<option name="USE_CUSTOM_SETTINGS" value="true" />
<option name="LAYOUT_SETTINGS">
<value>
<option name="INSERT_BLANK_LINE_BEFORE_TAG" value="false"/>
<option name="INSERT_BLANK_LINE_BEFORE_TAG" value="false" />
</value>
</option>
</AndroidXmlCodeStyleSettings>
<JSCodeStyleSettings>
<option name="INDENT_CHAINED_CALLS" value="false"/>
<option name="INDENT_CHAINED_CALLS" value="false" />
</JSCodeStyleSettings>
<Python>
<option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true"/>
<option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true" />
</Python>
<TypeScriptCodeStyleSettings>
<option name="INDENT_CHAINED_CALLS" value="false"/>
<option name="INDENT_CHAINED_CALLS" value="false" />
</TypeScriptCodeStyleSettings>
<XML>
<option name="XML_ALIGN_ATTRIBUTES" value="false"/>
<option name="XML_LEGACY_SETTINGS_IMPORTED" value="true"/>
<option name="XML_ALIGN_ATTRIBUTES" value="false" />
<option name="XML_LEGACY_SETTINGS_IMPORTED" value="true" />
</XML>
<codeStyleSettings language="CSS">
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="ECMA Script Level 4">
<option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
<option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
<option name="ALIGN_MULTILINE_FOR" value="false"/>
<option name="CALL_PARAMETERS_WRAP" value="1"/>
<option name="METHOD_PARAMETERS_WRAP" value="1"/>
<option name="EXTENDS_LIST_WRAP" value="1"/>
<option name="BINARY_OPERATION_WRAP" value="1"/>
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
<option name="TERNARY_OPERATION_WRAP" value="1"/>
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
<option name="FOR_STATEMENT_WRAP" value="1"/>
<option name="ARRAY_INITIALIZER_WRAP" value="1"/>
<option name="IF_BRACE_FORCE" value="3"/>
<option name="DOWHILE_BRACE_FORCE" value="3"/>
<option name="WHILE_BRACE_FORCE" value="3"/>
<option name="FOR_BRACE_FORCE" value="3"/>
<option name="PARENT_SETTINGS_INSTALLED" value="true"/>
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
<option name="ALIGN_MULTILINE_FOR" value="false" />
<option name="CALL_PARAMETERS_WRAP" value="1" />
<option name="METHOD_PARAMETERS_WRAP" value="1" />
<option name="EXTENDS_LIST_WRAP" value="1" />
<option name="BINARY_OPERATION_WRAP" value="1" />
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
<option name="TERNARY_OPERATION_WRAP" value="1" />
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
<option name="FOR_STATEMENT_WRAP" value="1" />
<option name="ARRAY_INITIALIZER_WRAP" value="1" />
<option name="IF_BRACE_FORCE" value="3" />
<option name="DOWHILE_BRACE_FORCE" value="3" />
<option name="WHILE_BRACE_FORCE" value="3" />
<option name="FOR_BRACE_FORCE" value="3" />
<option name="PARENT_SETTINGS_INSTALLED" value="true" />
</codeStyleSettings>
<codeStyleSettings language="HTML">
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="JAVA">
<option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false"/>
<option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
<option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1"/>
<option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
<option name="ALIGN_MULTILINE_RESOURCES" value="false"/>
<option name="ALIGN_MULTILINE_FOR" value="false"/>
<option name="CALL_PARAMETERS_WRAP" value="1"/>
<option name="METHOD_PARAMETERS_WRAP" value="1"/>
<option name="EXTENDS_LIST_WRAP" value="1"/>
<option name="THROWS_KEYWORD_WRAP" value="1"/>
<option name="METHOD_CALL_CHAIN_WRAP" value="1"/>
<option name="BINARY_OPERATION_WRAP" value="1"/>
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
<option name="TERNARY_OPERATION_WRAP" value="1"/>
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
<option name="FOR_STATEMENT_WRAP" value="1"/>
<option name="ARRAY_INITIALIZER_WRAP" value="1"/>
<option name="WRAP_COMMENTS" value="true"/>
<option name="IF_BRACE_FORCE" value="3"/>
<option name="DOWHILE_BRACE_FORCE" value="3"/>
<option name="WHILE_BRACE_FORCE" value="3"/>
<option name="FOR_BRACE_FORCE" value="3"/>
<option name="PARENT_SETTINGS_INSTALLED" value="true"/>
<option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
<option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
<option name="ALIGN_MULTILINE_RESOURCES" value="false" />
<option name="ALIGN_MULTILINE_FOR" value="false" />
<option name="CALL_PARAMETERS_WRAP" value="1" />
<option name="METHOD_PARAMETERS_WRAP" value="1" />
<option name="EXTENDS_LIST_WRAP" value="1" />
<option name="THROWS_KEYWORD_WRAP" value="1" />
<option name="METHOD_CALL_CHAIN_WRAP" value="1" />
<option name="BINARY_OPERATION_WRAP" value="1" />
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
<option name="TERNARY_OPERATION_WRAP" value="1" />
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
<option name="FOR_STATEMENT_WRAP" value="1" />
<option name="ARRAY_INITIALIZER_WRAP" value="1" />
<option name="WRAP_COMMENTS" value="true" />
<option name="IF_BRACE_FORCE" value="3" />
<option name="DOWHILE_BRACE_FORCE" value="3" />
<option name="WHILE_BRACE_FORCE" value="3" />
<option name="FOR_BRACE_FORCE" value="3" />
<option name="PARENT_SETTINGS_INSTALLED" value="true" />
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="JSON">
<indentOptions>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="TAB_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="JavaScript">
<option name="RIGHT_MARGIN" value="80"/>
<option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
<option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
<option name="ALIGN_MULTILINE_FOR" value="false"/>
<option name="CALL_PARAMETERS_WRAP" value="1"/>
<option name="METHOD_PARAMETERS_WRAP" value="1"/>
<option name="BINARY_OPERATION_WRAP" value="1"/>
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
<option name="TERNARY_OPERATION_WRAP" value="1"/>
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true"/>
<option name="FOR_STATEMENT_WRAP" value="1"/>
<option name="ARRAY_INITIALIZER_WRAP" value="1"/>
<option name="IF_BRACE_FORCE" value="3"/>
<option name="DOWHILE_BRACE_FORCE" value="3"/>
<option name="WHILE_BRACE_FORCE" value="3"/>
<option name="FOR_BRACE_FORCE" value="3"/>
<option name="PARENT_SETTINGS_INSTALLED" value="true"/>
<option name="RIGHT_MARGIN" value="80" />
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
<option name="ALIGN_MULTILINE_FOR" value="false" />
<option name="CALL_PARAMETERS_WRAP" value="1" />
<option name="METHOD_PARAMETERS_WRAP" value="1" />
<option name="BINARY_OPERATION_WRAP" value="1" />
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
<option name="TERNARY_OPERATION_WRAP" value="1" />
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
<option name="FOR_STATEMENT_WRAP" value="1" />
<option name="ARRAY_INITIALIZER_WRAP" value="1" />
<option name="IF_BRACE_FORCE" value="3" />
<option name="DOWHILE_BRACE_FORCE" value="3" />
<option name="WHILE_BRACE_FORCE" value="3" />
<option name="FOR_BRACE_FORCE" value="3" />
<option name="PARENT_SETTINGS_INSTALLED" value="true" />
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="PROTO">
<option name="RIGHT_MARGIN" value="80"/>
<option name="RIGHT_MARGIN" value="80" />
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="2"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="2" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="protobuf">
<option name="RIGHT_MARGIN" value="80"/>
<option name="RIGHT_MARGIN" value="80" />
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="2"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="2" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="Python">
<option name="KEEP_BLANK_LINES_IN_CODE" value="1"/>
<option name="RIGHT_MARGIN" value="80"/>
<option name="ALIGN_MULTILINE_PARAMETERS" value="false"/>
<option name="PARENT_SETTINGS_INSTALLED" value="true"/>
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="RIGHT_MARGIN" value="80" />
<option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
<option name="PARENT_SETTINGS_INSTALLED" value="true" />
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="SASS">
<indentOptions>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="TAB_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="SCSS">
<indentOptions>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="TAB_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="TypeScript">
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="XML">
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="2"/>
<option name="TAB_SIZE" value="2"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="2" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
<arrangement>
<rules>
@ -226,7 +226,7 @@
<match>
<AND>
<NAME>xmlns:android</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
@ -237,7 +237,7 @@
<match>
<AND>
<NAME>xmlns:.*</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
@ -249,7 +249,7 @@
<match>
<AND>
<NAME>.*:id</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -260,7 +260,7 @@
<match>
<AND>
<NAME>style</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
@ -271,7 +271,7 @@
<match>
<AND>
<NAME>.*</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
@ -283,7 +283,7 @@
<match>
<AND>
<NAME>.*:.*Style</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -295,7 +295,7 @@
<match>
<AND>
<NAME>.*:layout_width</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -306,7 +306,7 @@
<match>
<AND>
<NAME>.*:layout_height</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -317,7 +317,7 @@
<match>
<AND>
<NAME>.*:layout_weight</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -328,7 +328,7 @@
<match>
<AND>
<NAME>.*:layout_margin</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -339,7 +339,7 @@
<match>
<AND>
<NAME>.*:layout_marginTop</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -350,7 +350,7 @@
<match>
<AND>
<NAME>.*:layout_marginBottom</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -361,7 +361,7 @@
<match>
<AND>
<NAME>.*:layout_marginStart</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -372,7 +372,7 @@
<match>
<AND>
<NAME>.*:layout_marginEnd</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -383,7 +383,7 @@
<match>
<AND>
<NAME>.*:layout_marginLeft</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -394,7 +394,7 @@
<match>
<AND>
<NAME>.*:layout_marginRight</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -405,7 +405,7 @@
<match>
<AND>
<NAME>.*:layout_.*</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -417,7 +417,7 @@
<match>
<AND>
<NAME>.*:padding</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -428,7 +428,7 @@
<match>
<AND>
<NAME>.*:paddingTop</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -439,7 +439,7 @@
<match>
<AND>
<NAME>.*:paddingBottom</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -450,7 +450,7 @@
<match>
<AND>
<NAME>.*:paddingStart</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -461,7 +461,7 @@
<match>
<AND>
<NAME>.*:paddingEnd</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -472,7 +472,7 @@
<match>
<AND>
<NAME>.*:paddingLeft</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -483,7 +483,7 @@
<match>
<AND>
<NAME>.*:paddingRight</NAME>
<XML_ATTRIBUTE/>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
@ -537,62 +537,62 @@
</arrangement>
</codeStyleSettings>
<Objective-C>
<option name="INDENT_NAMESPACE_MEMBERS" value="0"/>
<option name="INDENT_C_STRUCT_MEMBERS" value="2"/>
<option name="INDENT_CLASS_MEMBERS" value="2"/>
<option name="INDENT_VISIBILITY_KEYWORDS" value="1"/>
<option name="INDENT_INSIDE_CODE_BLOCK" value="2"/>
<option name="KEEP_STRUCTURES_IN_ONE_LINE" value="true"/>
<option name="FUNCTION_PARAMETERS_WRAP" value="5"/>
<option name="FUNCTION_CALL_ARGUMENTS_WRAP" value="5"/>
<option name="TEMPLATE_CALL_ARGUMENTS_WRAP" value="5"/>
<option name="TEMPLATE_CALL_ARGUMENTS_ALIGN_MULTILINE" value="true"/>
<option name="ALIGN_INIT_LIST_IN_COLUMNS" value="false"/>
<option name="SPACE_BEFORE_SUPERCLASS_COLON" value="false"/>
<option name="INDENT_NAMESPACE_MEMBERS" value="0" />
<option name="INDENT_C_STRUCT_MEMBERS" value="2" />
<option name="INDENT_CLASS_MEMBERS" value="2" />
<option name="INDENT_VISIBILITY_KEYWORDS" value="1" />
<option name="INDENT_INSIDE_CODE_BLOCK" value="2" />
<option name="KEEP_STRUCTURES_IN_ONE_LINE" value="true" />
<option name="FUNCTION_PARAMETERS_WRAP" value="5" />
<option name="FUNCTION_CALL_ARGUMENTS_WRAP" value="5" />
<option name="TEMPLATE_CALL_ARGUMENTS_WRAP" value="5" />
<option name="TEMPLATE_CALL_ARGUMENTS_ALIGN_MULTILINE" value="true" />
<option name="ALIGN_INIT_LIST_IN_COLUMNS" value="false" />
<option name="SPACE_BEFORE_SUPERCLASS_COLON" value="false" />
</Objective-C>
<Objective-C-extensions>
<option name="GENERATE_INSTANCE_VARIABLES_FOR_PROPERTIES" value="ASK"/>
<option name="RELEASE_STYLE" value="IVAR"/>
<option name="TYPE_QUALIFIERS_PLACEMENT" value="BEFORE"/>
<option name="GENERATE_INSTANCE_VARIABLES_FOR_PROPERTIES" value="ASK" />
<option name="RELEASE_STYLE" value="IVAR" />
<option name="TYPE_QUALIFIERS_PLACEMENT" value="BEFORE" />
<file>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Import"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Macro"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Typedef"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Enum"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Constant"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Global"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Struct"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="FunctionPredecl"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Function"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Import" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Macro" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Typedef" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Enum" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Constant" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Global" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Struct" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="FunctionPredecl" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Function" />
</file>
<class>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Property"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Synthesize"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InitMethod"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="StaticMethod"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InstanceMethod"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="DeallocMethod"/>
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Property" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="Synthesize" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InitMethod" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="StaticMethod" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="InstanceMethod" />
<option name="com.jetbrains.cidr.lang.util.OCDeclarationKind" value="DeallocMethod" />
</class>
<extensions>
<pair source="cc" header="h"/>
<pair source="c" header="h"/>
<pair source="cc" header="h" />
<pair source="c" header="h" />
</extensions>
</Objective-C-extensions>
<codeStyleSettings language="ObjectiveC">
<option name="RIGHT_MARGIN" value="80"/>
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1"/>
<option name="BLANK_LINES_BEFORE_IMPORTS" value="0"/>
<option name="BLANK_LINES_AFTER_IMPORTS" value="0"/>
<option name="BLANK_LINES_AROUND_CLASS" value="0"/>
<option name="BLANK_LINES_AROUND_METHOD" value="0"/>
<option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="0"/>
<option name="ALIGN_MULTILINE_BINARY_OPERATION" value="false"/>
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true"/>
<option name="FOR_STATEMENT_WRAP" value="1"/>
<option name="ASSIGNMENT_WRAP" value="1"/>
<option name="RIGHT_MARGIN" value="80" />
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />
<option name="BLANK_LINES_BEFORE_IMPORTS" value="0" />
<option name="BLANK_LINES_AFTER_IMPORTS" value="0" />
<option name="BLANK_LINES_AROUND_CLASS" value="0" />
<option name="BLANK_LINES_AROUND_METHOD" value="0" />
<option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="0" />
<option name="ALIGN_MULTILINE_BINARY_OPERATION" value="false" />
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
<option name="FOR_STATEMENT_WRAP" value="1" />
<option name="ASSIGNMENT_WRAP" value="1" />
<indentOptions>
<option name="INDENT_SIZE" value="2"/>
<option name="CONTINUATION_INDENT_SIZE" value="4"/>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="4" />
</indentOptions>
</codeStyleSettings>
</code_scheme>

View File

@ -2,5 +2,4 @@
echo $MAVEN_HOME
mvn com.coveo:fmt-maven-plugin:format
mvn xml-format:xml-format
mvn com.coveo:fmt-maven-plugin:format

View File

@ -1,40 +0,0 @@
#!/usr/bin/env bash
#
# Copyright 2017 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
#
proto_dir="proto"
if [ -d $proto_dir ]; then
rm -r $proto_dir
fi
repos=("https://github.com/pingcap/kvproto" "https://github.com/pingcap/raft-rs" "https://github.com/pingcap/tipb")
commits=(3056ca36e6f2a71a9fc7ba7135e6b119fd977553 b9891b673573fad77ebcf9bbe0969cf945841926 c4d518eb1d60c21f05b028b36729e64610346dac)
for i in "${!repos[@]}"; do
repo_name=$(basename ${repos[$i]})
git_command="git -C $repo_name"
if [ -d "$repo_name" ]; then
$git_command checkout `basename $($git_command symbolic-ref --short refs/remotes/origin/HEAD)`
$git_command fetch --all
$git_command pull --all
else
git clone ${repos[$i]} $repo_name
$git_command fetch -p
fi
$git_command checkout ${commits[$i]}
done

View File

@ -1,16 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<FindBugsFilter>
<Match>
<Package name="~org\.tikv\.(?!kvproto).*"/>
<Not>
<Package name="~org\.tikv\.common\.parser.*"/>
</Not>
<Not>
<Or>
<Bug pattern="EI_EXPOSE_REP"/>
<Bug pattern="EI_EXPOSE_REP2"/>
</Or>
</Not>
<Rank value="1"/>
</Match>
</FindBugsFilter>

View File

@ -1,28 +0,0 @@
# Client Java Docs
NOTE: See [tikv.github.io/client-java](https://tikv.github.io/client-java/) for
the rendered developer and contributor documents.
This directory contains all the source of developer and contributor
documentations in markdown format. The table of content can be found in
[./src/SUMMARY.md](./src/SUMMARY.md).
## How to contribute to the document
The rendered document is generated by `mdbook`, to get it:
```sh
cargo install mdbook
```
Build the source in your development machine:
```sh
mdbook build
```
Get more information about how to use `mdbook`:
```sh
mdbook help
```

View File

@ -1,6 +0,0 @@
[book]
authors = ["Jian Zhang"]
language = "en"
multilingual = false
src = "src"
title = "TiKV Java Client User Documents"

View File

@ -1,33 +0,0 @@
# Summary
<!-- Links to empty page is replaced by empty link `() `, so that we can prevent readers to click into these unfinished page. Once the corresponding page is done, we can delete the empty parenthesis and then the pages will be visible. -->
- [Introduction](./introduction/introduction.md)
- [Production Readiness](./production-readiness.md)
- [Start With Examples](./examples/introduction.md)
- [Quick Start](./examples/quick-start.md)
- [Interact with TiKV RawKV API](./examples/rawkv.md)
- [Interact with TiKV TxnKV API](./examples/txnkv.md)
- [TiKV RawKV Bulk Load]() (./examples/bulk-load.md)
- [Performance]() (./performance/introduction.md)
- [YCSB Benchmarks]() (./performance/ycsb.md)
- [Administration](./administration/introduction.md)
- [Configuration](./administration/configuration.md)
- [Monitoring](./administration/monitoring.md)
- [Troubleshooting](./troubleshooting/introduction.md)
- [Slow Request Diagnosis](./troubleshooting/slow-request.md)
- [Error Request Diagnosis]() (./troubleshooting/error-request.md)
- [Architecture and Internals](./architecture/introduction.md)
- [The Lifecycle of A Request](./architecture/request-lifecycle.md)
- [Backoff and Retry Policy](./architecture/availability.md)
- [Slow Log and Metrics]() (./architecture/observability.md)
- [Region Cache]() (./architecture/region-cache.md)
- [Contribution Guide](./contribution/introduction.md)
- [Bug Severity Guidelines](./contribution/bug-severity-guide.md)

View File

@ -1,121 +0,0 @@
## Java Client Configuration Parameter
### JVM Parameter
The following includes JVM related parameters.
#### tikv.pd.addresses
- pd addresses, separated by comma
- default: 127.0.0.1:2379
#### tikv.grpc.timeout_in_ms
- timeout of grpc request
- default: 600ms
#### tikv.grpc.scan_timeout_in_ms
- timeout of scan/delete range grpc request
- default: 20s
#### tikv.importer.max_kv_batch_bytes
- Maximal package size transporting from clients to TiKV Server (ingest API)
- default: 1048576 (1M)
#### tikv.importer.max_kv_batch_size
- Maximal batch size transporting from clients to TiKV Server (ingest API)
- default: 32768 (32K)
#### tikv.scatter_wait_seconds
- time to wait for scattering regions
- default: 300 (5min)
#### tikv.rawkv.default_backoff_in_ms
- RawKV default backoff in milliseconds
- default: 20000 (20 seconds)
### Metrics Parameter
#### tikv.metrics.enable
- whether to enable metrics exporting
- default: false
#### tikv.metrics.port
- the metrics exporting http port
- default: 3140
### ThreadPool Parameter
The following includes ThreadPool related parameters, which can be passed in through JVM parameters.
#### tikv.batch_get_concurrency
- the thread pool size of batchGet on client side
- default: 20
#### tikv.batch_put_concurrency
- the thread pool size of batchPut on client side
- default: 20
#### tikv.batch_delete_concurrency
- the thread pool size of batchDelete on client side
- default: 20
#### tikv.batch_scan_concurrency
- the thread pool size of batchScan on client side
- default: 5
#### tikv.delete_range_concurrency
- the thread pool size of deleteRange on client side
- default: 20
#### tikv.enable_atomic_for_cas
- whether to enable `Compare And Set`, set true if using `RawKVClient.compareAndSet` or `RawKVClient.putIfAbsent`
- default: false
### TLS
#### tikv.tls_enable
- whether to enable TLS
- default: false
#### tikv.trust_cert_collection
- Trusted certificates for verifying the remote endpoint's certificate, e.g. /home/tidb/ca.pem. The file should contain an X.509 certificate collection in PEM format.
- default: null
#### tikv.key_cert_chain
- an X.509 certificate chain file in PEM format, e.g. /home/tidb/client.pem.
- default: null
#### tikv.key_file
- a PKCS#8 private key file in PEM format. e.g. /home/tidb/client-key.pem.
- default: null
#### tikv.tls.reload_interval
- The interval in seconds to poll the change of TLS context, if a change is detected, the TLS context will be rebuilded.
- default: `"10s"`, `"0s"` means disable TLS context reload.
#### tikv.conn.recycle_time
- After a TLS context reloading, the old connections will be forced to shutdown after `tikv.conn.recycle_time` to prevent channel leak.
- default: `"60s"`.
#### tikv.rawkv.read_timeout_in_ms
- RawKV read timeout in milliseconds. This parameter controls the timeout of `get` `getKeyTTL`.
- default: 2000 (2 seconds)
#### tikv.rawkv.write_timeout_in_ms
- RawKV write timeout in milliseconds. This parameter controls the timeout of `put` `putAtomic` `putIfAbsent` `delete` `deleteAtomic`.
- default: 2000 (2 seconds)
#### tikv.rawkv.batch_read_timeout_in_ms
- RawKV batch read timeout in milliseconds. This parameter controls the timeout of `batchGet`.
- default: 2000 (2 seconds)
#### tikv.rawkv.batch_write_timeout_in_ms
- RawKV batch write timeout in milliseconds. This parameter controls the timeout of `batchPut` `batchDelete` `batchDeleteAtomic`.
- default: 2000 (2 seconds)
#### tikv.rawkv.scan_timeout_in_ms
- RawKV scan timeout in milliseconds. This parameter controls the timeout of `batchScan` `scan` `scanPrefix`.
- default: 10000 (10 seconds)
#### tikv.rawkv.clean_timeout_in_ms
- RawKV clean timeout in milliseconds. This parameter controls the timeout of `deleteRange` `deletePrefix`.
- default: 600000 (10 minutes)

View File

@ -1 +0,0 @@
# Administration

View File

@ -1,33 +0,0 @@
## Java Client Metrics
Client Java supports exporting metrics to Prometheus using poll mode and viewing on Grafana. The following steps shows how to enable this function.
### Step 1: Enable metrics exporting
- set the config `tikv.metrics.enable` to `true`
- call TiConfiguration.setMetricsEnable(true)
### Step 2: Set the metrics port
- set the config `tikv.metrics.port`
- call TiConfiguration.setMetricsPort
Default port is 3140.
### Step 3: Config Prometheus
Add the following config to `conf/prometheus.yml` and restart Prometheus.
```yaml
- job_name: "tikv-client"
honor_labels: true
static_configs:
- targets:
- '127.0.0.1:3140'
- '127.0.0.2:3140'
- '127.0.0.3:3140'
```
### Step 4: Config Grafana
Import the [Client-Java-Summary dashboard config](/metrics/grafana/client_java_summary.json) to Grafana.

View File

@ -1,47 +0,0 @@
# Availability: Backoff and Retry Policy
## BackOffer
The retry and timeout mechanism for a request is controlled by a `BackOffer` object, which is created one per `RawKVClient` method. The `BackOffer` will decide how much time the next sleep and retry should spend, and whether to timeout the request if not enough time is left for retrying the request.
If we need a back off sleep, we call backOffer.doBackOff(funcType, exception), and the current thread will sleep for a decided time. If the current operation will timeout after sleep, the doBackOff simply throw an exception to abort the operation.
## callWithRetry
RegionStoreClient.callWithRetry inherits from AbstractGRPCClient.callWithRetry. The concrete logic is in RetryPolicy.callWithRetry, which implements a retry mechanism, but the specific retry strategy is determined by the ErrorHandler.
ErrorHandlers handler{Request, Response}Error function returns a boolean value indicating whether to retry inside callWithRetry.
The control flow for callWithRetry is as follows:
![callWithRetry](./callWithRetry.jpg)
The error handler is chosen obeying the following table:
| gPRC request | the result | handler |
| -- | -- | -- |
| throws exception | - | handleRequestError
| no exception | is null | handleRequestError
| no exception | is error | handleResponseError
| no exception | normal | normal return
The handleRequestError function copes with the following situations:
| situation | retry within callWithRetry | note |
|----------|---------------|------------------------------|
| invalid store in region manager | true | refresh ClientStub |
| region has not got multiple copies | false | |
| successfully switched to new leader | true | |
| seekProxyStore | true if success | only when `tikv.enable_grpc_forward` is set |
| other | false | |
The handleResponseError function copes with the following gRPC errors:
| error | retry within callWithRetry |
|----------------------|----------------------------|
| NotLeader | true if leader unchanged |
| StoreNotMatch | false |
| EphochNotMatch | true if region epoch in `ctx` is ahead of TiKV's |
| ServerIsBusy | true |
| StaleCommand | true |
| RaftEntryTooLarge | throw |
| KeyNotInRegion | throw |
| Raft ProposalDropped | true |
| other | false |

Binary file not shown.

Before

Width:  |  Height:  |  Size: 89 KiB

View File

@ -1,3 +0,0 @@
# Architecture
This section includes in-depthA description of the client architecture.

View File

@ -1 +0,0 @@
# Observability: Slow Log and Metrics

View File

@ -1 +0,0 @@
# Region Cache

View File

@ -1,7 +0,0 @@
# The Lifecycle of A Request
![time graph](./timegraph.png)
The client talks to TiKV store directly using gRPC requests, which are created in RegionStoreClient. If a request failed, the client could retry after a back off sleep. The retry logic is delegated to AbstractGRPCClient::callWithRetry method. callWithRetry may decide to retry request within the function, or, if the RegionStoreClient must be recreated (due to, for example, region split), return a failure and let outermost RawKVClient to do the retry.
![request-overview](./request-overview.jpg)

Binary file not shown.

Before

Width:  |  Height:  |  Size: 95 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 90 KiB

View File

@ -1,49 +0,0 @@
## Bug Severity Guidelines
This is a **working-in-progress** guide about determining defects severity on
TiKV Java Client according to the impact on the online service. The higher
effect the defect has on the overall functionality or performance, the higher
the severity is. There are 4 severity levels:
1. Critical
2. Major
3. Moderate
4. Minor
Each severity is described with examples in the remaining contents.
### Critical Defects
A defect that affects critical data or functionality and leaves users
with no workaround is classified as a critical defect. These defects are
labeled with `type/bug` and `severity/critical`, can be found
[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fcritical)
Guideline 1. A defect that breaks the API definition is regarded as critical.
For example:
* [client-java/issues/412](https://github.com/tikv/client-java/issues/412)
in this defect, gRPC timeout is not set for certain requests, which causes the
requests can not be terminated as expected when the processing time is too long.
### Major Defects
A defect that affects critical data or functionality and forces users to employ
a workaround is classified as a major defect. These defects are labeled with
`type/bug` and `severity/major`, can be found
[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fmajor)
### Moderate Defects
A defect that affects non-critical data or functionality and forces users to
employ a workaround is classified as moderate defect. These defects are labeled
with `type/bug` and `severity/moderate`, can be found
[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fmoderate)
### Minor Defects
A defect that does not affect data or functionality. It does not even need a
workaround. It does not impact productivity or efficiency. It is merely an
inconvenience. These defects are labeled with `type/bug` and `severity/minor`,
can be found
[here](https://github.com/tikv/client-java/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Aseverity%2Fminor)

View File

@ -1,21 +0,0 @@
# Contribution Guide
## Build the package
```
mvn clean package -Dmaven.test.skip=true
```
## Install the package to local maven repository
```
mvn clean install -Dmaven.test.skip=true
```
## Run tests
```
export RAWKV_PD_ADDRESSES=127.0.0.1:2379
export TXNKV_PD_ADDRESSES=127.0.0.1:2379
mvn clean test
```

View File

@ -1,3 +0,0 @@
# Start With Examples
This section contains examples to demonstrate basic usages of the java client.

View File

@ -1,109 +0,0 @@
# Quick Start
The package is hosted on maven central repository. To build from source, refer to the [Contribution Guide](../contribution/introduction.html).
## Create a maven project
First download [maven] and follow the [installation instructoins][install]. Then `mvn` command should be available in the `$PATH`.
[maven]: https://maven.apache.org/download.html
[install]: https://maven.apache.org/install.html
create a maven project by following command:
```
mvn archetype:generate -DgroupId=com.example -DartifactId=java-client-example -DarchetypeArtifactId=maven-archetype-quickstart -DinteractiveMode=false
cd java-client-example
```
## Add dependency
Add maven dependency to `pom.xml`.
```xml
<dependency>
<groupId>org.tikv</groupId>
<artifactId>tikv-client-java</artifactId>
<version>3.3.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.32</version>
</dependency>
```
Now `pom.xml` should look like this:
```xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.example</groupId>
<artifactId>java-project</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<name>java-project</name>
<url>http://maven.apache.org</url>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.tikv</groupId>
<artifactId>tikv-client-java</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.32</version>
</dependency>
</dependencies>
</project>
```
## Writing code
To interact with TiKV, we should first create a `TiConfiguration` with PD address, create a `TiSession` using `TiSession.create`, and then create a client.
For example, if we want to put a `World` in `Hello` key in RawKV, write the following code in `src/main/java/com/example/App.java`.
```java
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
import org.tikv.raw.RawKVClient;
import org.tikv.shade.com.google.protobuf.ByteString;
public class App {
public static void main(String[] args) throws Exception {
String pdAddr = "127.0.0.1:2379";
// You MUST create a raw configuration if you are using RawKVClient.
TiConfiguration conf = TiConfiguration.createRawDefault(pdAddr);
try (TiSession session = TiSession.create(conf)) {
try (RawKVClient client = session.createRawClient()) {
client.put(ByteString.copyFromUtf8("Hello"), ByteString.copyFromUtf8("World"));
ByteString value = client.get(ByteString.copyFromUtf8("Hello"));
System.out.println(value);
}
}
}
}
```
More examples for RawKV and TxnKV are in following chapters.
## Running program
Run following command:
```
mvn assembly:assembly -DdescriptorId=jar-with-dependencies
java -cp target/java-client-example-1.0-SNAPSHOT-jar-with-dependencies.jar com.example.App
```

View File

@ -1,100 +0,0 @@
# RawKV
Below is the basic usages of RawKV. See [API document] to see a full list of methods available.
[API document]: https://tikv.github.io/client-java/apidocs/org/tikv/raw/RawKVClient
```java
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
import org.tikv.kvproto.Kvrpcpb;
import org.tikv.raw.RawKVClient;
import org.tikv.shade.com.google.protobuf.ByteString;
public class Main {
public static void main(String[] args) throws Exception {
// You MUST create a raw configuration if you are using RawKVClient.
TiConfiguration conf = TiConfiguration.createRawDefault("127.0.0.1:2379");
TiSession session = TiSession.create(conf);
RawKVClient client = session.createRawClient();
// put
client.put(ByteString.copyFromUtf8("k1"), ByteString.copyFromUtf8("Hello"));
client.put(ByteString.copyFromUtf8("k2"), ByteString.copyFromUtf8(","));
client.put(ByteString.copyFromUtf8("k3"), ByteString.copyFromUtf8("World"));
client.put(ByteString.copyFromUtf8("k4"), ByteString.copyFromUtf8("!"));
client.put(ByteString.copyFromUtf8("k5"), ByteString.copyFromUtf8("Raw KV"));
// get
Optional<ByteString> result = client.get(ByteString.copyFromUtf8("k1"));
System.out.println(result.get().toStringUtf8());
// batch get
List<Kvrpcpb.KvPair> list = client.batchGet(new ArrayList<ByteString>() {{
add(ByteString.copyFromUtf8("k1"));
add(ByteString.copyFromUtf8("k3"));
}});
System.out.println(list);
// scan
list = client.scan(ByteString.copyFromUtf8("k1"), ByteString.copyFromUtf8("k6"), 10);
System.out.println(list);
// close
client.close();
session.close();
}
}
```
## API V2
With TiKV version >= 6.1.0, we release a new feature called "TiKV API V2" which provides a new raw key-value storage format allowing the coexistence of RawKV and TxnKV. Please refer to [v6.10 release notes](https://docs.pingcap.com/tidb/stable/release-6.1.0#ease-of-use) for detail.
To enable the API V2 mode, users need to specify the API version of the client.
```java
// import ...
import org.tikv.common.TiConfiguration.ApiVersion;
public class Main {
public static void main(String[] args) throws Exception {
TiConfiguration conf = TiConfiguration.createRawDefault("127.0.0.1:2379");
conf.setApiVersion(ApiVersion.V2);
try(TiSession session = TiSession.create(conf)) {
try(RawKVClient client = session.createRawClient()) {
// The client will read and write date in the format of API V2, which is
// transparent to the users.
client.put(ByteString.copyFromUtf8("hello"), ByteString.copyFromUtf8("world"));
// other client operations.
}
}
}
}
```
### Compatibility
The V2 Client should not access the cluster other than V2, this requires users to [enable the API V2](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#api-version-new-in-v610) for the cluster:
```toml
[storage]
# The V2 cluster must enable ttl for RawKV explicitly
enable-ttl = true
api-version = 2
```
If V2 client accesses a V1 cluster or V1 cluster accesses a V2 cluster, the requests will be denied by the cluster. You can check the compatibility via the following matrix.
| | V1 Server | V1TTL Server | V2 Server |
| --------------------- | --------- | ------------ | --------- |
| V1 RawClient | Raw | Raw | Error |
| V1 RawClient with TTL | Error | Raw | Error |
| V1 TxnClient | Txn | Error | Error |
| V1 TiDB | TiDB Data | Error | TiDB Data |
| V2 RawClient | Error | Error | Raw |
| V2 TxnClient | Error | Error | Txn |

View File

@ -1,69 +0,0 @@
# TxnKV
Below is the basic usages of TxnKV.
Data should be written into TxnKV using [`TwoPhaseCommitter`], and be read using [`org.tikv.txn.KVClient`][KVClient].
[`TwoPhaseCommitter`]: https://tikv.github.io/client-java/apidocs/org/tikv/txn/TwoPhaseCommitter.html
[KVClient]: https://tikv.github.io/client-java/apidocs/org/tikv/txn/KVClient.html
```java
import java.util.Arrays;
import java.util.List;
import org.tikv.common.BytePairWrapper;
import org.tikv.common.ByteWrapper;
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
import org.tikv.common.util.BackOffer;
import org.tikv.common.util.ConcreteBackOffer;
import org.tikv.kvproto.Kvrpcpb.KvPair;
import org.tikv.shade.com.google.protobuf.ByteString;
import org.tikv.txn.KVClient;
import org.tikv.txn.TwoPhaseCommitter;
public class App {
public static void main(String[] args) throws Exception {
TiConfiguration conf = TiConfiguration.createDefault("127.0.0.1:2379");
try (TiSession session = TiSession.create(conf)) {
// two-phrase write
long startTS = session.getTimestamp().getVersion();
try (TwoPhaseCommitter twoPhaseCommitter = new TwoPhaseCommitter(session, startTS)) {
BackOffer backOffer = ConcreteBackOffer.newCustomBackOff(1000);
byte[] primaryKey = "key1".getBytes("UTF-8");
byte[] key2 = "key2".getBytes("UTF-8");
// first phrase: prewrite
twoPhaseCommitter.prewritePrimaryKey(backOffer, primaryKey, "val1".getBytes("UTF-8"));
List<BytePairWrapper> pairs = Arrays
.asList(new BytePairWrapper(key2, "val2".getBytes("UTF-8")));
twoPhaseCommitter.prewriteSecondaryKeys(primaryKey, pairs.iterator(), 1000);
// second phrase: commit
long commitTS = session.getTimestamp().getVersion();
twoPhaseCommitter.commitPrimaryKey(backOffer, primaryKey, commitTS);
List<ByteWrapper> keys = Arrays.asList(new ByteWrapper(key2));
twoPhaseCommitter.commitSecondaryKeys(keys.iterator(), commitTS, 1000);
}
try (KVClient kvClient = session.createKVClient()) {
long version = session.getTimestamp().getVersion();
ByteString key1 = ByteString.copyFromUtf8("key1");
ByteString key2 = ByteString.copyFromUtf8("key2");
// get value of a single key
ByteString val = kvClient.get(key1, version);
System.out.println(val);
// get value of multiple keys
BackOffer backOffer = ConcreteBackOffer.newCustomBackOff(1000);
List<KvPair> kvPairs = kvClient.batchGet(backOffer, Arrays.asList(key1, key2), version);
System.out.println(kvPairs);
// get value of a range of keys
kvPairs = kvClient.scan(key1, ByteString.copyFromUtf8("key3"), version);
System.out.println(kvPairs);
}
}
}
}
```

View File

@ -1,15 +0,0 @@
# Introduction
Welcome to the documents of TiKV Java Client. This document is oriented to:
1. Application developers who needs to integrate java client to their
application code. There are documents about best practices, administrations,
troubleshootings, and internals of TiKV Java Client for developers.
2. TiKV contributors who wish to contribute to TiKV Java Client, become a TiKV
reviewer, committer, or maintainer. There are documents about setting up
environment, submitting PR, triage issues, and manage releases in the
community.
Wish you have a good journey in developing application or contributing to the
TiKV Java Client.

View File

@ -1 +0,0 @@
# Performance

View File

@ -1 +0,0 @@
# YCSB Benchmarks

View File

@ -1,18 +0,0 @@
# Production Readiness
In general, the latest [release](https://github.com/tikv/client-java/releases) of TiKV Java Client is ready for production use. But it is not battle-tested as full featured client for TiKV in all use cases. This page will give you more details.
## RawKV
All RawKV APIs are covered by [CI](https://github.com/tikv/client-java/actions/workflows/ci.yml).
At this time, RawKV has been used in the production environment of some commercial customers in latency sensitive systems. But they only use part of the RawKV APIs (mainly including `raw_put`, `raw_get`, `raw_compare_and_swap`, and `raw_batch_put`).
## TxnKV
All TxnKV APIs are covered by [CI](https://github.com/tikv/client-java/actions/workflows/ci.yml).
In addition, TxnKV has been used in the [TiSpark](https://docs.pingcap.com/tidb/stable/tispark-overview) and [TiBigData](https://github.com/tidb-incubator/TiBigData) project to integrate data from TiDB to Big Data ecosystem. TiSpark and TiBigData are used in the production system of some commercial customers and internet companies.
Similar to RawKV, only part of APIs are used in this scenario (mainly including `prewrite/commit` and `coprocessor`). And this use case doesn't care about latency but throughput and reliability.
## TiDB Cloud
Directly using TiKV is not possible on TiDB Cloud due to the fact that client has to access the whole cluster, which has security issues. And TiKV managed service is not coming soon as it's not contained in [roadmap](https://docs.pingcap.com/tidbcloud/tidb-cloud-roadmap) yet.

View File

@ -1 +0,0 @@
# Error Request Diagnosis

View File

@ -1 +0,0 @@
# Troubleshooting

View File

@ -1,34 +0,0 @@
# Slow Request Diagnosis
If a request take too much time, we can collect the detailed time spend in each component in a “slow log”.
<!-- wrap text in the code block -->
<pre>
<code class="hljs" style="white-space: pre-wrap;">
2022-02-11 11:07:56 WARN SlowLogImpl:88 - A request spent 55 ms. start=11:07:56.938, end=11:07:56.993, SlowLog:{"trace_id":4361090673996453790,"spans":[{"event":"put","begin":"11:07:56.938","duration_ms":55,"properties":{"region":"{Region[2] ConfVer[5] Version[60] Store[1] KeyRange[]:[]}","key":"Hello"}},{"event":"getRegionByKey","begin":"11:07:56.938","duration_ms":0},{"event":"callWithRetry","begin":"11:07:56.943","duration_ms":49,"properties":{"method":"tikvpb.Tikv/RawPut"}},{"event":"gRPC","begin":"11:07:56.943","duration_ms":49,"properties":{"method":"tikvpb.Tikv/RawPut"}}]}
</code>
</pre>
## Slow log configurations
| SlowLog settings | default value |
| -- | -- |
| tikv.rawkv.read_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
| tikv.rawkv.write_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
| tikv.rawkv.batch_read_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
| tikv.rawkv.batch_write_slowlog_in_ms | tikv.grpc.timeout_in_ms * 2 |
| tikv.rawkv.scan_slowlog_in_ms | 5s |
Each settings can be set by system properties, configuration files or `set...` method of `TiConfiguration`.
System properties can be set by `-D` parameter of `java` command.
```
java -cp target/java-client-example-1.0-SNAPSHOT-jar-with-dependencies.jar -Dtikv.rawkv.read_slowlog_in_ms=100 com.example.App
```
Configuration file is `src/main/resources/tikv.properties` in maven projects.
## Visualize slow log
TBD

File diff suppressed because it is too large Load Diff

254
pom.xml
View File

@ -1,13 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.tikv</groupId>
<artifactId>tikv-client-java</artifactId>
<version>3.3.4-SNAPSHOT</version>
<version>3.1.0</version>
<packaging>jar</packaging>
<name>TiKV Java Client</name>
<description>A Java Client for TiKV</description>
<url>http://github.com/tikv/client-java</url>
<licenses>
<license>
<name>Apache 2.0 License</name>
@ -15,9 +18,11 @@
<distribution>repo</distribution>
</license>
</licenses>
<organization>
<name>PingCAP</name>
</organization>
<developers>
<developer>
<name>Xiaoyu Ma</name>
@ -44,54 +49,33 @@
<organizationUrl>https://www.pingcap.com</organizationUrl>
</developer>
</developers>
<scm>
<connection>scm:git:git://github.com/tikv/client-java.git</connection>
<developerConnection>scm:git:ssh://github.com:tikv/client-java.git</developerConnection>
<url>https://github.com/tikv/client-java/tree/master</url>
</scm>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<protobuf.version>3.18.0</protobuf.version>
<protobuf.version>3.5.1</protobuf.version>
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.16</slf4j.version>
<grpc.version>1.60.0</grpc.version>
<netty.tcnative.version>2.0.34.Final</netty.tcnative.version>
<gson.version>2.8.9</gson.version>
<grpc.version>1.24.0</grpc.version>
<powermock.version>1.6.6</powermock.version>
<jackson-annotations.version>2.13.2</jackson-annotations.version>
<jackson.version>2.13.4.2</jackson.version>
<jackson.version>2.10.0</jackson.version>
<trove4j.version>3.0.1</trove4j.version>
<jetcd.version>0.7.7</jetcd.version>
<jetcd.version>0.4.1</jetcd.version>
<joda-time.version>2.9.9</joda-time.version>
<joda-convert.version>1.9.2</joda-convert.version>
<proto.folder>${basedir}/proto</proto.folder>
<gpg.keyname>fake gpg key name</gpg.keyname>
<gpg.skip>true</gpg.skip>
<javadoc.skip>true</javadoc.skip>
</properties>
<dependencies>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.19.6</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>3.19.6</version>
</dependency>
<dependency>
<groupId>io.perfmark</groupId>
<artifactId>perfmark-api</artifactId>
<version>0.24.0</version>
</dependency>
<dependency>
<groupId>io.perfmark</groupId>
<artifactId>perfmark-traceviewer</artifactId>
<version>0.24.0</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
@ -136,12 +120,6 @@
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
@ -152,17 +130,6 @@
<groupId>io.grpc</groupId>
<artifactId>grpc-services</artifactId>
<version>${grpc.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
<version>${netty.tcnative.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
@ -170,15 +137,10 @@
<version>${grpc.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson-annotations.version}</version>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@ -188,6 +150,20 @@
<dependency>
<groupId>io.etcd</groupId>
<artifactId>jetcd-core</artifactId>
<exclusions>
<exclusion>
<groupId>io.etcd</groupId>
<artifactId>jetcd-resolver</artifactId>
</exclusion>
<exclusion>
<groupId>io.etcd</groupId>
<artifactId>jetcd-common</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-grpclb</artifactId>
</exclusion>
</exclusions>
<version>${jetcd.version}</version>
</dependency>
<dependency>
@ -209,7 +185,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.10</version>
<version>3.9</version>
<scope>test</scope>
</dependency>
<dependency>
@ -218,16 +194,6 @@
<version>3.9</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.15</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
@ -252,31 +218,6 @@
<version>0.10.0</version>
</dependency>
</dependencies>
<reporting>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.3.1</version>
</plugin>
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>4.5.2.0</version>
<configuration>
<xmlOutput>true</xmlOutput>
<!-- Optional directory to put spotbugs xdoc xml report -->
<xmlOutputDirectory>target/site</xmlOutputDirectory>
<includeFilterFile>dev/spotbugs-include.xml</includeFilterFile>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jxr-plugin</artifactId>
<version>3.1.1</version>
</plugin>
</plugins>
</reporting>
<build>
<resources>
<resource>
@ -320,7 +261,7 @@
<execution>
<id>clone proto files</id>
<configuration>
<executable>${basedir}/dev/proto.sh</executable>
<executable>${basedir}/scripts/proto.sh</executable>
</configuration>
<phase>validate</phase>
<goals>
@ -371,6 +312,7 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
@ -393,6 +335,7 @@
</execution>
</executions>
</plugin>
<!-- Compiler Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -444,6 +387,26 @@
</filesets>
</configuration>
</plugin>
<!-- Javadoc Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<configuration>
<skip>${javadoc.skip}</skip>
</configuration>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
<configuration> <!-- add this to disable checking -->
<additionalparam>-Xdoclint:none</additionalparam>
</configuration>
</execution>
</executions>
</plugin>
<!--- Needs to shade Protobuf 3 since other projects might use other version -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -480,10 +443,6 @@
<pattern>io.prometheus</pattern>
<shadedPattern>org.tikv.shade.io.prometheus</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>org.tikv.shade.com.fasterxml.jackson</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
@ -503,7 +462,7 @@
</execution>
<execution>
<id>jacoco-site</id>
<phase>test</phase>
<phase>package</phase>
<goals>
<goal>report</goal>
</goals>
@ -560,110 +519,9 @@
<configuration>
<serverId>ossrh</serverId>
<nexusUrl>https://oss.sonatype.org/</nexusUrl>
<autoReleaseAfterClose>true</autoReleaseAfterClose>
</configuration>
</plugin>
<plugin>
<groupId>io.github.git-commit-id</groupId>
<artifactId>git-commit-id-maven-plugin</artifactId>
<version>4.9.9</version>
<executions>
<execution>
<id>get-the-git-infos</id>
<goals>
<goal>revision</goal>
</goals>
<phase>initialize</phase>
</execution>
<execution>
<id>validate-the-git-infos</id>
<goals>
<goal>validateRevision</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
<configuration>
<verbose>true</verbose>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<generateGitPropertiesFilename>${project.build.outputDirectory}/git.properties</generateGitPropertiesFilename>
<includeOnlyProperties>
<includeOnlyProperty>^git.build.(time|version)$</includeOnlyProperty>
<includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
</includeOnlyProperties>
<commitIdGenerationMode>full</commitIdGenerationMode>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M5</version>
<configuration>
<forkMode>always</forkMode>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
<version>3.7.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>4.5.2.0</version>
<configuration>
<includeFilterFile>dev/spotbugs-include.xml</includeFilterFile>
<xmlOutput>true</xmlOutput>
<failOnError>false</failOnError>
</configuration>
<dependencies>
<!-- overwrite dependency on spotbugs if you want to specify the version of spotbugs -->
<dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs</artifactId>
<version>4.5.3</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>au.com.acegi</groupId>
<artifactId>xml-format-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<id>xml-format</id>
<phase>validate</phase>
<goals>
<goal>xml-format</goal>
</goals>
</execution>
</executions>
<configuration>
<indentSize>4</indentSize>
<autoReleaseAfterClose>false</autoReleaseAfterClose>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>jdk9plus</id>
<activation>
<!-- activated when building with JDK9+ -->
<jdk>!1.8</jdk>
</activation>
<dependencies>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
<version>1.3.2</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
</profiles>
</project>

45
scripts/proto.sh Executable file
View File

@ -0,0 +1,45 @@
#!/bin/sh
#
# Copyright 2017 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
#
CURRENT_DIR=`pwd`
TIKV_CLIENT_HOME="$(cd "`dirname "$0"`"/..; pwd)"
cd $TIKV_CLIENT_HOME
kvproto_hash=6ed99a08e262d8a32d6355dcba91cf99cb92074a
raft_rs_hash=b9891b673573fad77ebcf9bbe0969cf945841926
tipb_hash=c4d518eb1d60c21f05b028b36729e64610346dac
if [ -d "kvproto" ]; then
cd kvproto; git fetch -p; git checkout ${kvproto_hash}; cd ..
else
git clone https://github.com/pingcap/kvproto; cd kvproto; git checkout ${kvproto_hash}; cd ..
fi
if [ -d "raft-rs" ]; then
cd raft-rs; git fetch -p; git checkout ${raft_rs_hash}; cd ..
else
git clone https://github.com/pingcap/raft-rs; cd raft-rs; git checkout ${raft_rs_hash}; cd ..
fi
if [ -d "tipb" ]; then
cd tipb; git fetch -p; git checkout ${tipb_hash}; cd ..
else
git clone https://github.com/pingcap/tipb; cd tipb; git checkout ${tipb_hash}; cd ..
fi
cd $CURRENT_DIR

View File

@ -1,77 +0,0 @@
#!/usr/bin/env python3
#!coding:utf-8
# Copyright 2022 TiKV Project Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
import json
import argparse
def main():
pattern = r'.*SlowLog:.*'
slowstr = "SlowLog:"
grpc_pattern = "gRPC tikvpb.Tikv"
backoff_pattern = "backoff "
args = parse_args()
items = []
with open(args.slowlog, encoding = 'utf-8') as f:
for line in f.readlines():
matched = re.match(pattern, line, re.M|re.I)
if matched is not None:
log = json.loads(line[(line.index(slowstr) + len(slowstr)):])
item = {
'req': log['func'],
'start': log['start'],
'tot_lat': latency_ms(log),
'tot_grpc': 0,
'tot_bo': 0,
}
items.append(item)
for span in log['spans']:
if grpc_pattern in span['name'] and span['duration'] != 'N/A':
item['tot_grpc'] += latency_ms(span)
elif backoff_pattern in span['name'] and span['duration'] != 'N/A':
item['tot_bo'] += latency_ms(span)
if args.order == "total":
items = sorted(items, key=lambda d: d['tot_lat'], reverse=True)
elif args.order == "grpc":
items = sorted(items, key=lambda d: d['tot_grpc'], reverse=True)
elif args.order == "backoff":
items = sorted(items, key=lambda d: d['tot_bo'], reverse=True)
else:
print("unsupported order option, use default value: total")
items = sorted(items, key=lambda d: d['tot_lat'], reverse=True)
fmtStr = "{:<12} {:<14} {:<14} {:<20} {:<20}"
print(fmtStr.format("Request", "Start", "Total Lat(ms)", "Total gRPC Lat(ms)", "Total Backoff Lat(ms)"))
for item in items:
print(fmtStr.format(item['req'], item['start'], item['tot_lat'], item['tot_grpc'], item['tot_bo']))
def latency_ms(span):
return int(span['duration'][:len(span['duration'])-2])
def parse_args():
parser = argparse.ArgumentParser(description="rstats: A TiKV Java Client Request Stats Analyzer")
parser.add_argument("-o", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff")
parser.add_argument("slowlog", help="slow log file")
return parser.parse_args()
if __name__ == '__main__':
main()

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
@ -25,7 +23,6 @@ import io.grpc.MethodDescriptor;
import io.grpc.health.v1.HealthCheckRequest;
import io.grpc.health.v1.HealthCheckResponse;
import io.grpc.health.v1.HealthGrpc;
import io.grpc.stub.AbstractFutureStub;
import io.grpc.stub.AbstractStub;
import io.grpc.stub.ClientCalls;
import io.grpc.stub.StreamObserver;
@ -37,20 +34,18 @@ import org.tikv.common.operation.ErrorHandler;
import org.tikv.common.policy.RetryMaxMs.Builder;
import org.tikv.common.policy.RetryPolicy;
import org.tikv.common.streaming.StreamingResponse;
import org.tikv.common.util.BackOffFunction.BackOffFuncType;
import org.tikv.common.util.BackOffer;
import org.tikv.common.util.ChannelFactory;
public abstract class AbstractGRPCClient<
BlockingStubT extends AbstractStub<BlockingStubT>,
FutureStubT extends AbstractFutureStub<FutureStubT>>
BlockingStubT extends AbstractStub<BlockingStubT>, StubT extends AbstractStub<StubT>>
implements AutoCloseable {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
protected final ChannelFactory channelFactory;
protected TiConfiguration conf;
protected long timeout;
protected BlockingStubT blockingStub;
protected FutureStubT asyncStub;
protected StubT asyncStub;
protected AbstractGRPCClient(TiConfiguration conf, ChannelFactory channelFactory) {
this.conf = conf;
@ -62,7 +57,7 @@ public abstract class AbstractGRPCClient<
TiConfiguration conf,
ChannelFactory channelFactory,
BlockingStubT blockingStub,
FutureStubT asyncStub) {
StubT asyncStub) {
this.conf = conf;
this.timeout = conf.getTimeout();
this.channelFactory = channelFactory;
@ -83,16 +78,17 @@ public abstract class AbstractGRPCClient<
if (logger.isTraceEnabled()) {
logger.trace(String.format("Calling %s...", method.getFullMethodName()));
}
RetryPolicy<RespT> policy = new Builder<RespT>(backOffer).create(handler);
RetryPolicy.Builder<RespT> builder = new Builder<>(backOffer);
RespT resp =
policy.callWithRetry(
() -> {
BlockingStubT stub = getBlockingStub();
return ClientCalls.blockingUnaryCall(
stub.getChannel(), method, stub.getCallOptions(), requestFactory.get());
},
method.getFullMethodName(),
backOffer);
builder
.create(handler)
.callWithRetry(
() -> {
BlockingStubT stub = getBlockingStub();
return ClientCalls.blockingUnaryCall(
stub.getChannel(), method, stub.getCallOptions(), requestFactory.get());
},
method.getFullMethodName());
if (logger.isTraceEnabled()) {
logger.trace(String.format("leaving %s...", method.getFullMethodName()));
@ -108,18 +104,19 @@ public abstract class AbstractGRPCClient<
ErrorHandler<RespT> handler) {
logger.debug(String.format("Calling %s...", method.getFullMethodName()));
RetryPolicy<RespT> policy = new Builder<RespT>(backOffer).create(handler);
policy.callWithRetry(
() -> {
FutureStubT stub = getAsyncStub();
ClientCalls.asyncUnaryCall(
stub.getChannel().newCall(method, stub.getCallOptions()),
requestFactory.get(),
responseObserver);
return null;
},
method.getFullMethodName(),
backOffer);
RetryPolicy.Builder<RespT> builder = new Builder<>(backOffer);
builder
.create(handler)
.callWithRetry(
() -> {
StubT stub = getAsyncStub();
ClientCalls.asyncUnaryCall(
stub.getChannel().newCall(method, stub.getCallOptions()),
requestFactory.get(),
responseObserver);
return null;
},
method.getFullMethodName());
logger.debug(String.format("leaving %s...", method.getFullMethodName()));
}
@ -130,17 +127,17 @@ public abstract class AbstractGRPCClient<
ErrorHandler<StreamObserver<ReqT>> handler) {
logger.debug(String.format("Calling %s...", method.getFullMethodName()));
RetryPolicy<StreamObserver<ReqT>> policy =
new Builder<StreamObserver<ReqT>>(backOffer).create(handler);
RetryPolicy.Builder<StreamObserver<ReqT>> builder = new Builder<>(backOffer);
StreamObserver<ReqT> observer =
policy.callWithRetry(
() -> {
FutureStubT stub = getAsyncStub();
return asyncBidiStreamingCall(
stub.getChannel().newCall(method, stub.getCallOptions()), responseObserver);
},
method.getFullMethodName(),
backOffer);
builder
.create(handler)
.callWithRetry(
() -> {
StubT stub = getAsyncStub();
return asyncBidiStreamingCall(
stub.getChannel().newCall(method, stub.getCallOptions()), responseObserver);
},
method.getFullMethodName());
logger.debug(String.format("leaving %s...", method.getFullMethodName()));
return observer;
}
@ -152,18 +149,18 @@ public abstract class AbstractGRPCClient<
ErrorHandler<StreamingResponse> handler) {
logger.debug(String.format("Calling %s...", method.getFullMethodName()));
RetryPolicy<StreamingResponse> policy =
new Builder<StreamingResponse>(backOffer).create(handler);
RetryPolicy.Builder<StreamingResponse> builder = new Builder<>(backOffer);
StreamingResponse response =
policy.callWithRetry(
() -> {
BlockingStubT stub = getBlockingStub();
return new StreamingResponse(
blockingServerStreamingCall(
stub.getChannel(), method, stub.getCallOptions(), requestFactory.get()));
},
method.getFullMethodName(),
backOffer);
builder
.create(handler)
.callWithRetry(
() -> {
BlockingStubT stub = getBlockingStub();
return new StreamingResponse(
blockingServerStreamingCall(
stub.getChannel(), method, stub.getCallOptions(), requestFactory.get()));
},
method.getFullMethodName());
logger.debug(String.format("leaving %s...", method.getFullMethodName()));
return response;
}
@ -178,32 +175,21 @@ public abstract class AbstractGRPCClient<
protected abstract BlockingStubT getBlockingStub();
protected abstract FutureStubT getAsyncStub();
protected abstract StubT getAsyncStub();
private boolean doCheckHealth(BackOffer backOffer, String addressStr, HostMapping hostMapping) {
while (true) {
backOffer.checkTimeout();
try {
ManagedChannel channel = channelFactory.getChannel(addressStr, hostMapping);
HealthGrpc.HealthBlockingStub stub =
HealthGrpc.newBlockingStub(channel)
.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
HealthCheckRequest req = HealthCheckRequest.newBuilder().build();
HealthCheckResponse resp = stub.check(req);
return resp.getStatus() == HealthCheckResponse.ServingStatus.SERVING;
} catch (Exception e) {
logger.warn("check health failed, addr: {}, caused by: {}", addressStr, e.getMessage());
backOffer.doBackOff(BackOffFuncType.BoCheckHealth, e);
}
}
}
protected boolean checkHealth(BackOffer backOffer, String addressStr, HostMapping hostMapping) {
protected boolean checkHealth(String addressStr, HostMapping hostMapping) {
ManagedChannel channel = channelFactory.getChannel(addressStr, hostMapping);
HealthGrpc.HealthBlockingStub stub =
HealthGrpc.newBlockingStub(channel).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
HealthCheckRequest req = HealthCheckRequest.newBuilder().build();
try {
return doCheckHealth(backOffer, addressStr, hostMapping);
HealthCheckResponse resp = stub.check(req);
if (resp.getStatus() != HealthCheckResponse.ServingStatus.SERVING) {
return false;
}
} catch (Exception e) {
return false;
}
return true;
}
}

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2020 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2020 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;

View File

@ -1,42 +1,28 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2021 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
import org.tikv.common.util.BackOffer;
import org.tikv.kvproto.Kvrpcpb;
public class ConfigUtils {
public static final String TIKV_CONFIGURATION_FILENAME = "tikv.properties";
public static final String TIKV_PD_ADDRESSES = "tikv.pd.addresses";
public static final String TIKV_GRPC_TIMEOUT = "tikv.grpc.timeout_in_ms";
public static final String TIKV_GRPC_INGEST_TIMEOUT = "tikv.grpc.ingest_timeout_in_ms";
public static final String TIKV_GRPC_FORWARD_TIMEOUT = "tikv.grpc.forward_timeout_in_ms";
public static final String TIKV_GRPC_WARM_UP_TIMEOUT = "tikv.grpc.warm_up_timeout_in_ms";
public static final String TIKV_PD_FIRST_GET_MEMBER_TIMEOUT =
"tikv.grpc.pd_first_get_member_timeout_in_ms";
public static final String TIKV_GRPC_SCAN_TIMEOUT = "tikv.grpc.scan_timeout_in_ms";
public static final String TIKV_GRPC_SCAN_BATCH_SIZE = "tikv.grpc.scan_batch_size";
public static final String TIKV_GRPC_MAX_FRAME_SIZE = "tikv.grpc.max_frame_size";
public static final String TIKV_GRPC_KEEPALIVE_TIME = "tikv.grpc.keepalive_time";
public static final String TIKV_GRPC_KEEPALIVE_TIMEOUT = "tikv.grpc.keepalive_timeout";
public static final String TIKV_GRPC_IDLE_TIMEOUT = "tikv.grpc.idle_timeout";
public static final String TIKV_CONN_RECYCLE_TIME = "tikv.conn.recycle_time";
public static final String TIKV_INDEX_SCAN_BATCH_SIZE = "tikv.index.scan_batch_size";
public static final String TIKV_INDEX_SCAN_CONCURRENCY = "tikv.index.scan_concurrency";
@ -64,79 +50,13 @@ public class ConfigUtils {
public static final String TIKV_NETWORK_MAPPING_NAME = "tikv.network.mapping";
public static final String TIKV_ENABLE_GRPC_FORWARD = "tikv.enable_grpc_forward";
public static final String TIKV_GRPC_HEALTH_CHECK_TIMEOUT = "tikv.grpc.health_check_timeout";
public static final String TIKV_HEALTH_CHECK_PERIOD_DURATION =
"tikv.health_check_period_duration";
public static final String TIKV_ENABLE_ATOMIC_FOR_CAS = "tikv.enable_atomic_for_cas";
public static final String TIKV_IMPORTER_MAX_KV_BATCH_BYTES = "tikv.importer.max_kv_batch_bytes";
public static final String TIKV_IMPORTER_MAX_KV_BATCH_SIZE = "tikv.importer.max_kv_batch_size";
public static final String TIKV_SCATTER_WAIT_SECONDS = "tikv.scatter_wait_seconds";
public static final String TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS = "tikv.rawkv.default_backoff_in_ms";
public static final String TIKV_RAWKV_READ_TIMEOUT_IN_MS = "tikv.rawkv.read_timeout_in_ms";
public static final String TIKV_RAWKV_WRITE_TIMEOUT_IN_MS = "tikv.rawkv.write_timeout_in_ms";
public static final String TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS =
"tikv.rawkv.batch_read_timeout_in_ms";
public static final String TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS =
"tikv.rawkv.batch_write_timeout_in_ms";
public static final String TIKV_RAWKV_SCAN_TIMEOUT_IN_MS = "tikv.rawkv.scan_timeout_in_ms";
public static final String TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS = "tikv.rawkv.clean_timeout_in_ms";
public static final String TIKV_BO_REGION_MISS_BASE_IN_MS = "tikv.bo_region_miss_base_in_ms";
public static final String TIKV_RAWKV_READ_SLOWLOG_IN_MS = "tikv.rawkv.read_slowlog_in_ms";
public static final String TIKV_RAWKV_WRITE_SLOWLOG_IN_MS = "tikv.rawkv.write_slowlog_in_ms";
public static final String TIKV_RAWKV_BATCH_READ_SLOWLOG_IN_MS =
"tikv.rawkv.batch_read_slowlog_in_ms";
public static final String TIKV_RAWKV_BATCH_WRITE_SLOWLOG_IN_MS =
"tikv.rawkv.batch_write_slowlog_in_ms";
public static final String TIKV_RAWKV_SCAN_SLOWLOG_IN_MS = "tikv.rawkv.scan_slowlog_in_ms";
public static final String TIKV_RAWKV_SERVER_SLOWLOG_FACTOR = "tikv.rawkv.server_slowlog_factor";
public static final String TIKV_TLS_ENABLE = "tikv.tls_enable";
public static final String TIKV_TLS_RELOAD_INTERVAL = "tikv.tls.reload_interval";
public static final String TIKV_TRUST_CERT_COLLECTION = "tikv.trust_cert_collection";
public static final String TIKV_KEY_CERT_CHAIN = "tikv.key_cert_chain";
public static final String TIKV_KEY_FILE = "tikv.key_file";
public static final String TIKV_USE_JKS = "tikv.use_jks";
public static final String TIKV_JKS_KEY_PATH = "tikv.jks.key_path";
public static final String TIKV_JKS_KEY_PASSWORD = "tikv.jks.key_password";
public static final String TIKV_JKS_TRUST_PATH = "tikv.jks.trust_path";
public static final String TIKV_JKS_TRUST_PASSWORD = "tikv.jks.trust_password";
public static final String TiKV_CIRCUIT_BREAK_ENABLE = "tikv.circuit_break.enable";
public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS =
"tikv.circuit_break.trigger.availability.window_in_seconds";
public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE =
"tikv.circuit_break.trigger.availability.error_threshold_percentage";
public static final String TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD =
"tikv.circuit_break.trigger.availability.request_volumn_threshold";
public static final String TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS =
"tikv.circuit_break.trigger.sleep_window_in_seconds";
public static final String TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT =
"tikv.circuit_break.trigger.attempt_request_count";
public static final String TIKV_SCAN_REGIONS_LIMIT = "tikv.scan_regions_limit";
public static final String TIFLASH_ENABLE = "tiflash.enable";
public static final String TIKV_WARM_UP_ENABLE = "tikv.warm_up.enable";
public static final String TIKV_API_VERSION = "tikv.api_version";
public static final String DEF_PD_ADDRESSES = "127.0.0.1:2379";
public static final String DEF_TIMEOUT = "200ms";
public static final String DEF_TIKV_GRPC_INGEST_TIMEOUT = "200s";
public static final String DEF_FORWARD_TIMEOUT = "300ms";
public static final String DEF_TIKV_GRPC_WARM_UP_TIMEOUT = "5000ms";
public static final String DEF_TIKV_PD_FIRST_GET_MEMBER_TIMEOUT = "10000ms";
public static final String DEF_TIMEOUT = "600ms";
public static final String DEF_SCAN_TIMEOUT = "20s";
public static final int DEF_CHECK_HEALTH_TIMEOUT = 100;
public static final int DEF_HEALTH_CHECK_PERIOD_DURATION = 300;
public static final int DEF_CHECK_HEALTH_TIMEOUT = 40;
public static final int DEF_SCAN_BATCH_SIZE = 10240;
public static final int DEF_MAX_FRAME_SIZE = 268435456 * 2; // 256 * 2 MB
public static final String DEF_TIKV_CONN_RECYCLE_TIME = "60s";
public static final String DEF_TIKV_TLS_RELOAD_INTERVAL = "10s";
public static final int DEF_INDEX_SCAN_BATCH_SIZE = 20000;
public static final int DEF_REGION_SCAN_DOWNGRADE_THRESHOLD = 10000000;
// if keyRange size per request exceeds this limit, the request might be too large to be accepted
@ -160,22 +80,6 @@ public class ConfigUtils {
public static final int DEF_METRICS_PORT = 3140;
public static final String DEF_TIKV_NETWORK_MAPPING_NAME = "";
public static final boolean DEF_GRPC_FORWARD_ENABLE = true;
public static final boolean DEF_TIKV_ENABLE_ATOMIC_FOR_CAS = false;
public static final int DEF_TIKV_IMPORTER_MAX_KV_BATCH_BYTES = 1024 * 1024;
public static final int DEF_TIKV_IMPORTER_MAX_KV_BATCH_SIZE = 1024 * 32;
public static final int DEF_TIKV_SCATTER_WAIT_SECONDS = 300;
public static final int DEF_TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS = BackOffer.RAWKV_MAX_BACKOFF;
public static final int DEF_TIKV_RAWKV_READ_TIMEOUT_IN_MS = 2000;
public static final int DEF_TIKV_RAWKV_WRITE_TIMEOUT_IN_MS = 2000;
public static final int DEF_TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS = 2000;
public static final int DEF_TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS = 2000;
public static final int DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS = 10000;
public static final int DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS = 600000;
public static final int DEF_TIKV_BO_REGION_MISS_BASE_IN_MS = 20;
public static final String DEF_TIKV_RAWKV_SCAN_SLOWLOG_IN_MS = "5000";
public static final String NORMAL_COMMAND_PRIORITY = "NORMAL";
public static final String LOW_COMMAND_PRIORITY = "LOW";
@ -190,23 +94,4 @@ public class ConfigUtils {
public static final String LEADER = "LEADER";
public static final String FOLLOWER = "FOLLOWER";
public static final String LEADER_AND_FOLLOWER = "LEADER_AND_FOLLOWER";
public static final int DEF_TIKV_GRPC_KEEPALIVE_TIME = 10;
public static final int DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT = 3;
public static final int DEF_TIKV_GRPC_IDLE_TIMEOUT = 60;
public static final boolean DEF_TIKV_TLS_ENABLE = false;
public static final boolean DEF_TIKV_USE_JKS = false;
public static final boolean DEF_TIFLASH_ENABLE = false;
public static final boolean DEF_TIKV_WARM_UP_ENABLE = true;
public static final boolean DEF_TiKV_CIRCUIT_BREAK_ENABLE = false;
public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS = 60;
public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE = 100;
public static final int DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUST_VOLUMN_THRESHOLD = 10;
public static final int DEF_TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS = 20;
public static final int DEF_TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT = 10;
public static final int DEF_TIKV_SCAN_REGIONS_LIMIT = 1000;
public static final int DEF_TIKV_API_VERSION = 1;
}

View File

@ -1,89 +0,0 @@
/*
* Copyright 2021 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
import static org.tikv.common.pd.PDUtils.addrToUri;
import com.google.common.annotations.Beta;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.Client;
import io.etcd.jetcd.KeyValue;
import io.etcd.jetcd.kv.GetResponse;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DefaultHostMapping implements HostMapping {
private static final String NETWORK_MAPPING_PATH = "/client/url-mapping";
private final Client etcdClient;
private final String networkMappingName;
private final ConcurrentMap<String, String> hostMapping;
private final Logger logger = LoggerFactory.getLogger(DefaultHostMapping.class);
public DefaultHostMapping(Client etcdClient, String networkMappingName) {
this.etcdClient = etcdClient;
this.networkMappingName = networkMappingName;
this.hostMapping = new ConcurrentHashMap<>();
}
private ByteSequence hostToNetworkMappingKey(String host) {
String path = NETWORK_MAPPING_PATH + "/" + networkMappingName + "/" + host;
return ByteSequence.from(path, StandardCharsets.UTF_8);
}
@Beta
private String getMappedHostFromPD(String host) {
ByteSequence hostKey = hostToNetworkMappingKey(host);
for (int i = 0; i < 5; i++) {
CompletableFuture<GetResponse> future = etcdClient.getKVClient().get(hostKey);
try {
GetResponse resp = future.get();
List<KeyValue> kvs = resp.getKvs();
if (kvs.size() != 1) {
break;
}
return kvs.get(0).getValue().toString(StandardCharsets.UTF_8);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (ExecutionException e) {
logger.info("failed to get mapped Host from PD: " + host, e);
break;
} catch (Exception ignore) {
// ignore
break;
}
}
return host;
}
public URI getMappedURI(URI uri) {
if (networkMappingName.isEmpty()) {
return uri;
}
return addrToUri(
hostMapping.computeIfAbsent(uri.getHost(), this::getMappedHostFromPD)
+ ":"
+ uri.getPort());
}
}

View File

@ -1,15 +1,15 @@
/*
* Copyright 2021 TiKV Project Authors.
*
* Copyright 2019 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*

View File

@ -1,25 +1,87 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2021 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
import java.io.Serializable;
import java.net.URI;
import static org.tikv.common.pd.PDUtils.addrToUri;
public interface HostMapping extends Serializable {
URI getMappedURI(URI uri);
import com.google.common.annotations.Beta;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.Client;
import io.etcd.jetcd.KeyValue;
import io.etcd.jetcd.kv.GetResponse;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HostMapping {
private static final String NETWORK_MAPPING_PATH = "/client/url-mapping";
private final Client etcdClient;
private final String networkMappingName;
private final ConcurrentMap<String, String> hostMapping;
private final Logger logger = LoggerFactory.getLogger(HostMapping.class);
public HostMapping(Client etcdClient, String networkMappingName) {
this.etcdClient = etcdClient;
this.networkMappingName = networkMappingName;
this.hostMapping = new ConcurrentHashMap<>();
}
private ByteSequence hostToNetworkMappingKey(String host) {
String path = NETWORK_MAPPING_PATH + "/" + networkMappingName + "/" + host;
return ByteSequence.from(path, StandardCharsets.UTF_8);
}
@Beta
private String getMappedHostFromPD(String host) {
ByteSequence hostKey = hostToNetworkMappingKey(host);
for (int i = 0; i < 5; i++) {
CompletableFuture<GetResponse> future = etcdClient.getKVClient().get(hostKey);
try {
GetResponse resp = future.get();
List<KeyValue> kvs = resp.getKvs();
if (kvs.size() != 1) {
break;
}
return kvs.get(0).getValue().toString(StandardCharsets.UTF_8);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (ExecutionException e) {
logger.info("failed to get mapped Host from PD: " + host, e);
break;
} catch (Exception ignore) {
// ignore
break;
}
}
return host;
}
public URI getMappedURI(URI uri) {
if (networkMappingName.isEmpty()) {
return uri;
}
return addrToUri(
hostMapping.computeIfAbsent(uri.getHost(), this::getMappedHostFromPD)
+ ":"
+ uri.getPort());
}
}

View File

@ -1,15 +1,15 @@
/*
* Copyright 2021 TiKV Project Authors.
*
* Copyright 2019 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
@ -65,9 +65,7 @@ public class KVClient implements AutoCloseable {
* @return a ByteString value if key exists, ByteString.EMPTY if key does not exist
*/
public ByteString get(ByteString key, long version) throws GrpcException {
BackOffer backOffer =
ConcreteBackOffer.newGetBackOff(
clientBuilder.getRegionManager().getPDClient().getClusterId());
BackOffer backOffer = ConcreteBackOffer.newGetBackOff();
while (true) {
RegionStoreClient client = clientBuilder.build(key);
try {

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2020 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;

View File

@ -1,83 +0,0 @@
/*
* Copyright 2021 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
import io.prometheus.client.exporter.HTTPServer;
import io.prometheus.client.hotspot.DefaultExports;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MetricsServer {
private static final Logger logger = LoggerFactory.getLogger(MetricsServer.class);
private static MetricsServer METRICS_SERVER_INSTANCE = null;
private static int metricsServerRefCount = 0;
private final int port;
private final HTTPServer server;
public static MetricsServer getInstance(TiConfiguration conf) {
if (!conf.isMetricsEnable()) {
return null;
}
synchronized (MetricsServer.class) {
int port = conf.getMetricsPort();
if (METRICS_SERVER_INSTANCE != null) {
if (port != METRICS_SERVER_INSTANCE.port) {
throw new IllegalArgumentException(
String.format(
"Do dot support multiple tikv.metrics.port, which are %d and %d",
port, METRICS_SERVER_INSTANCE.port));
}
} else {
METRICS_SERVER_INSTANCE = new MetricsServer(port);
}
metricsServerRefCount += 1;
return METRICS_SERVER_INSTANCE;
}
}
private MetricsServer(int port) {
try {
this.port = port;
DefaultExports.initialize();
this.server = new HTTPServer(port, true);
logger.info("http server is up " + this.server.getPort());
} catch (Exception e) {
logger.error("http server not up");
throw new RuntimeException(e);
}
}
public void close() {
synchronized (MetricsServer.class) {
if (metricsServerRefCount == 1) {
if (server != null) {
server.stop();
logger.info("Metrics server on " + server.getPort() + " is stopped");
}
METRICS_SERVER_INSTANCE = null;
}
if (metricsServerRefCount >= 1) {
metricsServerRefCount -= 1;
}
}
}
}

View File

@ -1,48 +0,0 @@
/*
* Copyright 2021 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
public enum PDChecker {
Learner,
Replica,
Rule,
Split,
Merge,
JointState,
Priority;
public String apiName() {
switch (this) {
case Learner:
return "learner";
case Replica:
return "replica";
case Rule:
return "rule";
case Split:
return "split";
case Merge:
return "merge";
case JointState:
return "joint-state";
case Priority:
return "priority";
}
throw new IllegalArgumentException();
}
}

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
@ -23,9 +21,6 @@ import static org.tikv.common.pd.PDError.buildFromPdpbError;
import static org.tikv.common.pd.PDUtils.addrToUri;
import static org.tikv.common.pd.PDUtils.uriToAddr;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.ByteString;
@ -39,50 +34,37 @@ import io.grpc.Metadata;
import io.grpc.stub.MetadataUtils;
import io.prometheus.client.Histogram;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tikv.common.apiversion.RequestKeyCodec;
import org.tikv.common.TiConfiguration.KVMode;
import org.tikv.common.codec.Codec.BytesCodec;
import org.tikv.common.codec.CodecDataOutput;
import org.tikv.common.codec.KeyUtils;
import org.tikv.common.exception.GrpcException;
import org.tikv.common.exception.TiClientInternalException;
import org.tikv.common.meta.TiTimestamp;
import org.tikv.common.operation.NoopHandler;
import org.tikv.common.operation.PDErrorHandler;
import org.tikv.common.region.TiRegion;
import org.tikv.common.util.BackOffFunction.BackOffFuncType;
import org.tikv.common.util.BackOffer;
import org.tikv.common.util.ChannelFactory;
import org.tikv.common.util.ConcreteBackOffer;
import org.tikv.common.util.HistogramUtils;
import org.tikv.common.util.Pair;
import org.tikv.kvproto.Metapb;
import org.tikv.kvproto.Metapb.Store;
import org.tikv.kvproto.PDGrpc;
import org.tikv.kvproto.PDGrpc.PDBlockingStub;
import org.tikv.kvproto.PDGrpc.PDFutureStub;
import org.tikv.kvproto.PDGrpc.PDStub;
import org.tikv.kvproto.Pdpb;
import org.tikv.kvproto.Pdpb.Error;
import org.tikv.kvproto.Pdpb.ErrorType;
@ -104,55 +86,40 @@ import org.tikv.kvproto.Pdpb.ScatterRegionResponse;
import org.tikv.kvproto.Pdpb.Timestamp;
import org.tikv.kvproto.Pdpb.TsoRequest;
import org.tikv.kvproto.Pdpb.TsoResponse;
import org.tikv.kvproto.Pdpb.UpdateServiceGCSafePointRequest;
public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
implements ReadOnlyPDClient {
private static final String TIFLASH_TABLE_SYNC_PROGRESS_PATH = "/tiflash/table/sync";
private static final long MIN_TRY_UPDATE_DURATION = 50;
private static final int PAUSE_CHECKER_TIMEOUT = 300; // in seconds
private static final int KEEP_CHECKER_PAUSE_PERIOD = PAUSE_CHECKER_TIMEOUT / 5; // in seconds
private static final Logger logger = LoggerFactory.getLogger(PDClient.class);
private static final ObjectMapper mapper = new ObjectMapper();
private final RequestKeyCodec codec;
private final Logger logger = LoggerFactory.getLogger(PDClient.class);
private RequestHeader header;
private TsoRequest tsoReq;
private volatile PDClientWrapper pdClientWrapper;
private ScheduledExecutorService service;
private ScheduledExecutorService tiflashReplicaService;
private final HashMap<PDChecker, ScheduledExecutorService> pauseCheckerService = new HashMap<>();
private List<URI> pdAddrs;
private Client etcdClient;
private ConcurrentMap<Long, Double> tiflashReplicaMap;
private HostMapping hostMapping;
private long lastUpdateLeaderTime;
private final ExecutorService updateLeaderService = Executors.newSingleThreadExecutor();
private final AtomicBoolean updateLeaderNotify = new AtomicBoolean();
public static final Histogram PD_GET_REGION_BY_KEY_REQUEST_LATENCY =
HistogramUtils.buildDuration()
Histogram.build()
.name("client_java_pd_get_region_by_requests_latency")
.help("pd getRegionByKey request latency.")
.labelNames("cluster")
.register();
private PDClient(TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
private PDClient(TiConfiguration conf, ChannelFactory channelFactory) {
super(conf, channelFactory);
initCluster();
this.codec = codec;
this.blockingStub = getBlockingStub();
this.asyncStub = getAsyncStub();
}
public static ReadOnlyPDClient create(
TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
return createRaw(conf, codec, channelFactory);
public static ReadOnlyPDClient create(TiConfiguration conf, ChannelFactory channelFactory) {
return createRaw(conf, channelFactory);
}
static PDClient createRaw(
TiConfiguration conf, RequestKeyCodec codec, ChannelFactory channelFactory) {
return new PDClient(conf, codec, channelFactory);
static PDClient createRaw(TiConfiguration conf, ChannelFactory channelFactory) {
return new PDClient(conf, channelFactory);
}
public HostMapping getHostMapping() {
@ -173,75 +140,12 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
return new TiTimestamp(timestamp.getPhysical(), timestamp.getLogical());
}
public synchronized void keepPauseChecker(PDChecker checker) {
if (!this.pauseCheckerService.containsKey(checker)) {
ScheduledExecutorService newService =
Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder()
.setNameFormat(String.format("PDClient-pause-%s-pool-%%d", checker.name()))
.setDaemon(true)
.build());
newService.scheduleAtFixedRate(
() -> pauseChecker(checker, PAUSE_CHECKER_TIMEOUT),
0,
KEEP_CHECKER_PAUSE_PERIOD,
TimeUnit.SECONDS);
this.pauseCheckerService.put(checker, newService);
}
}
public synchronized void stopKeepPauseChecker(PDChecker checker) {
if (this.pauseCheckerService.containsKey(checker)) {
this.pauseCheckerService.get(checker).shutdown();
this.pauseCheckerService.remove(checker);
}
}
public void resumeChecker(PDChecker checker) {
pauseChecker(checker, 0);
}
private void pauseChecker(PDChecker checker, int timeout) {
String verb = timeout == 0 ? "resume" : "pause";
URI url = pdAddrs.get(0);
String api = url.toString() + "/pd/api/v1/checker/" + checker.apiName();
HashMap<String, Integer> arguments = new HashMap<>();
arguments.put("delay", timeout);
try (CloseableHttpClient client = HttpClients.createDefault()) {
JsonMapper jsonMapper = new JsonMapper();
byte[] body = jsonMapper.writeValueAsBytes(arguments);
HttpPost post = new HttpPost(api);
post.setEntity(new ByteArrayEntity(body));
try (CloseableHttpResponse resp = client.execute(post)) {
if (resp.getStatusLine().getStatusCode() != 200) {
logger.error("failed to {} checker.", verb);
}
logger.info("checker {} {}d", checker.apiName(), verb);
}
} catch (Exception e) {
logger.error(String.format("failed to %s checker.", verb), e);
}
}
public Boolean isCheckerPaused(PDChecker checker) {
URI url = pdAddrs.get(0);
String api = url.toString() + "/pd/api/v1/checker/" + checker.apiName();
try {
HashMap<String, Boolean> status =
mapper.readValue(new URL(api), new TypeReference<HashMap<String, Boolean>>() {});
return status.get("paused");
} catch (Exception e) {
logger.error(String.format("failed to get %s checker status.", checker.apiName()), e);
return null;
}
}
/**
* Sends request to pd to scatter region.
*
* @param region represents a region info
*/
void scatterRegion(Metapb.Region region, BackOffer backOffer) {
void scatterRegion(TiRegion region, BackOffer backOffer) {
Supplier<ScatterRegionRequest> request =
() ->
ScatterRegionRequest.newBuilder().setHeader(header).setRegionId(region.getId()).build();
@ -265,7 +169,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
*
* @param region
*/
void waitScatterRegionFinish(Metapb.Region region, BackOffer backOffer) {
void waitScatterRegionFinish(TiRegion region, BackOffer backOffer) {
for (; ; ) {
GetOperatorResponse resp = getOperator(region.getId());
if (resp != null) {
@ -280,7 +184,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
"wait scatter region %d at key %s is %s",
region.getId(),
KeyUtils.formatBytes(resp.getDesc().toByteArray()),
resp.getStatus()));
resp.getStatus().toString()));
}
}
}
@ -291,7 +195,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
() -> GetOperatorRequest.newBuilder().setHeader(header).setRegionId(regionId).build();
// get operator no need to handle error and no need back offer.
return callWithRetry(
ConcreteBackOffer.newCustomBackOff(0, getClusterId()),
ConcreteBackOffer.newCustomBackOff(0),
PDGrpc.getGetOperatorMethod(),
request,
new NoopHandler<>());
@ -318,30 +222,39 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
}
@Override
public Pair<Metapb.Region, Metapb.Peer> getRegionByKey(BackOffer backOffer, ByteString key) {
Histogram.Timer requestTimer =
PD_GET_REGION_BY_KEY_REQUEST_LATENCY.labels(getClusterId().toString()).startTimer();
public TiRegion getRegionByKey(BackOffer backOffer, ByteString key) {
Histogram.Timer requestTimer = PD_GET_REGION_BY_KEY_REQUEST_LATENCY.startTimer();
try {
if (conf.getKvMode() == KVMode.TXN) {
CodecDataOutput cdo = new CodecDataOutput();
BytesCodec.writeBytes(cdo, key.toByteArray());
key = cdo.toByteString();
}
ByteString queryKey = key;
Supplier<GetRegionRequest> request =
() ->
GetRegionRequest.newBuilder()
.setHeader(header)
.setRegionKey(codec.encodePdQuery(key))
.build();
() -> GetRegionRequest.newBuilder().setHeader(header).setRegionKey(queryKey).build();
PDErrorHandler<GetRegionResponse> handler =
new PDErrorHandler<>(getRegionResponseErrorExtractor, this);
GetRegionResponse resp =
callWithRetry(backOffer, PDGrpc.getGetRegionMethod(), request, handler);
return new Pair<>(codec.decodeRegion(resp.getRegion()), resp.getLeader());
return new TiRegion(
resp.getRegion(),
resp.getLeader(),
null,
conf.getIsolationLevel(),
conf.getCommandPriority(),
conf.getKvMode(),
conf.getReplicaSelector());
} finally {
requestTimer.observeDuration();
}
}
@Override
public Pair<Metapb.Region, Metapb.Peer> getRegionByID(BackOffer backOffer, long id) {
public TiRegion getRegionByID(BackOffer backOffer, long id) {
Supplier<GetRegionByIDRequest> request =
() -> GetRegionByIDRequest.newBuilder().setHeader(header).setRegionId(id).build();
PDErrorHandler<GetRegionResponse> handler =
@ -349,31 +262,15 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
GetRegionResponse resp =
callWithRetry(backOffer, PDGrpc.getGetRegionByIDMethod(), request, handler);
return new Pair<Metapb.Region, Metapb.Peer>(
codec.decodeRegion(resp.getRegion()), resp.getLeader());
}
@Override
public List<Pdpb.Region> scanRegions(
BackOffer backOffer, ByteString startKey, ByteString endKey, int limit) {
// no need to backoff because ScanRegions is just for optimization
// introduce a warm-up timeout for ScanRegions requests
PDGrpc.PDBlockingStub stub =
getBlockingStub().withDeadlineAfter(conf.getWarmUpTimeout(), TimeUnit.MILLISECONDS);
Pair<ByteString, ByteString> range = codec.encodePdQueryRange(startKey, endKey);
Pdpb.ScanRegionsRequest request =
Pdpb.ScanRegionsRequest.newBuilder()
.setHeader(header)
.setStartKey(range.first)
.setEndKey(range.second)
.setLimit(limit)
.build();
Pdpb.ScanRegionsResponse resp = stub.scanRegions(request);
if (resp == null) {
return null;
}
return codec.decodePdRegions(resp.getRegionsList());
// Instead of using default leader instance, explicitly set no leader to null
return new TiRegion(
resp.getRegion(),
resp.getLeader(),
null,
conf.getIsolationLevel(),
conf.getCommandPriority(),
conf.getKvMode(),
conf.getReplicaSelector());
}
private Supplier<GetStoreRequest> buildGetStoreReq(long storeId) {
@ -384,17 +281,6 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
return () -> GetAllStoresRequest.newBuilder().setHeader(header).build();
}
private Supplier<UpdateServiceGCSafePointRequest> buildUpdateServiceGCSafePointRequest(
ByteString serviceId, long ttl, long safePoint) {
return () ->
UpdateServiceGCSafePointRequest.newBuilder()
.setHeader(header)
.setSafePoint(safePoint)
.setServiceId(serviceId)
.setTTL(ttl)
.build();
}
private <T> PDErrorHandler<GetStoreResponse> buildPDErrorHandler() {
return new PDErrorHandler<>(
r -> r.getHeader().hasError() ? buildFromPdpbError(r.getHeader().getError()) : null, this);
@ -402,16 +288,9 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
@Override
public Store getStore(BackOffer backOffer, long storeId) {
GetStoreResponse resp =
callWithRetry(
backOffer,
PDGrpc.getGetStoreMethod(),
buildGetStoreReq(storeId),
buildPDErrorHandler());
if (resp != null) {
return resp.getStore();
}
return null;
return callWithRetry(
backOffer, PDGrpc.getGetStoreMethod(), buildGetStoreReq(storeId), buildPDErrorHandler())
.getStore();
}
@Override
@ -431,20 +310,6 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
return conf.getReplicaRead();
}
@Override
public Long updateServiceGCSafePoint(
String serviceId, long ttl, long safePoint, BackOffer backOffer) {
return callWithRetry(
backOffer,
PDGrpc.getUpdateServiceGCSafePointMethod(),
buildUpdateServiceGCSafePointRequest(
ByteString.copyFromUtf8(serviceId), ttl, safePoint),
new PDErrorHandler<>(
r -> r.getHeader().hasError() ? buildFromPdpbError(r.getHeader().getError()) : null,
this))
.getMinSafePoint();
}
@Override
public void close() throws InterruptedException {
etcdClient.close();
@ -457,8 +322,6 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
if (channelFactory != null) {
channelFactory.close();
}
updateLeaderService.shutdownNow();
}
@VisibleForTesting
@ -471,36 +334,22 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
return pdClientWrapper;
}
private GetMembersResponse doGetMembers(BackOffer backOffer, URI uri) {
while (true) {
backOffer.checkTimeout();
try {
ManagedChannel probChan = channelFactory.getChannel(uriToAddr(uri), hostMapping);
PDGrpc.PDBlockingStub stub =
PDGrpc.newBlockingStub(probChan).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS);
GetMembersRequest request =
GetMembersRequest.newBuilder().setHeader(RequestHeader.getDefaultInstance()).build();
GetMembersResponse resp = stub.getMembers(request);
// check if the response contains a valid leader
if (resp != null && resp.getLeader().getMemberId() == 0) {
return null;
}
return resp;
} catch (Exception e) {
logger.warn(
"failed to get member from pd server from {}, caused by: {}", uri, e.getMessage());
backOffer.doBackOff(BackOffFuncType.BoPDRPC, e);
}
}
}
private GetMembersResponse getMembers(BackOffer backOffer, URI uri) {
private GetMembersResponse getMembers(URI uri) {
try {
return doGetMembers(backOffer, uri);
ManagedChannel probChan = channelFactory.getChannel(uriToAddr(uri), hostMapping);
PDGrpc.PDBlockingStub stub = PDGrpc.newBlockingStub(probChan);
GetMembersRequest request =
GetMembersRequest.newBuilder().setHeader(RequestHeader.getDefaultInstance()).build();
GetMembersResponse resp = stub.getMembers(request);
// check if the response contains a valid leader
if (resp != null && resp.getLeader().getMemberId() == 0) {
return null;
}
return resp;
} catch (Exception e) {
return null;
logger.warn("failed to get member from pd server.", e);
}
return null;
}
// return whether the leader has changed to target address `leaderUrlStr`.
@ -512,7 +361,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
return true;
}
}
// If leader has transferred to another member, we can create another leaderWrapper.
// If leader has transfered to another member, we can create another leaderwrapper.
}
// switch leader
return createLeaderClientWrapper(leaderUrlStr);
@ -524,65 +373,37 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
ManagedChannel clientChannel = channelFactory.getChannel(leaderUrlStr, hostMapping);
pdClientWrapper =
new PDClientWrapper(leaderUrlStr, leaderUrlStr, clientChannel, System.nanoTime());
timeout = conf.getTimeout();
} catch (IllegalArgumentException e) {
logger.error("Error updating leader. " + leaderUrlStr, e);
return false;
}
logger.info(String.format("Switched to new leader: %s", pdClientWrapper));
return true;
}
synchronized boolean createFollowerClientWrapper(
BackOffer backOffer, String followerUrlStr, String leaderUrls) {
synchronized boolean createFollowerClientWrapper(String followerUrlStr, String leaderUrls) {
// TODO: Why not strip protocol info on server side since grpc does not need it
try {
if (!checkHealth(backOffer, followerUrlStr, hostMapping)) {
if (!checkHealth(followerUrlStr, hostMapping)) {
return false;
}
// create new Leader
ManagedChannel channel = channelFactory.getChannel(followerUrlStr, hostMapping);
pdClientWrapper = new PDClientWrapper(leaderUrls, followerUrlStr, channel, System.nanoTime());
timeout = conf.getForwardTimeout();
} catch (IllegalArgumentException e) {
logger.error("Error updating follower. " + followerUrlStr, e);
return false;
}
logger.info(String.format("Switched to new leader by follower forward: %s", pdClientWrapper));
return true;
}
public void tryUpdateLeaderOrForwardFollower() {
if (updateLeaderNotify.compareAndSet(false, true)) {
try {
updateLeaderService.submit(
() -> {
try {
updateLeaderOrForwardFollower();
} catch (Exception e) {
logger.info("update leader or forward follower failed", e);
throw e;
} finally {
updateLeaderNotify.set(false);
logger.info("updating leader finish");
}
});
} catch (RejectedExecutionException e) {
logger.error("PDClient is shutdown", e);
updateLeaderNotify.set(false);
}
}
}
private synchronized void updateLeaderOrForwardFollower() {
logger.warn("updating leader or forward follower");
if (System.currentTimeMillis() - lastUpdateLeaderTime < MIN_TRY_UPDATE_DURATION) {
return;
}
public synchronized void updateLeaderOrforwardFollower() {
for (URI url : this.pdAddrs) {
BackOffer backOffer = this.probeBackOffer();
// since resp is null, we need update leader's address by walking through all pd server.
GetMembersResponse resp = getMembers(backOffer, url);
GetMembersResponse resp = getMembers(url);
if (resp == null) {
continue;
}
@ -594,9 +415,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
// if leader is switched, just return.
if (checkHealth(backOffer, leaderUrlStr, hostMapping)
&& createLeaderClientWrapper(leaderUrlStr)) {
lastUpdateLeaderTime = System.currentTimeMillis();
if (checkHealth(leaderUrlStr, hostMapping) && trySwitchLeader(leaderUrlStr)) {
return;
}
@ -604,12 +423,13 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
continue;
}
logger.info(String.format("can not switch to new leader, try follower forward"));
List<Pdpb.Member> members = resp.getMembersList();
boolean hasReachNextMember = false;
// If we have not used follower forward, try the first follower.
boolean hasReachNextMember =
pdClientWrapper != null && pdClientWrapper.getStoreAddress().equals(leaderUrlStr);
if (pdClientWrapper != null && pdClientWrapper.getStoreAddress().equals(leaderUrlStr)) {
hasReachNextMember = true;
}
for (int i = 0; i < members.size() * 2; i++) {
Pdpb.Member member = members.get(i % members.size());
@ -622,15 +442,11 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
hasReachNextMember = true;
continue;
}
if (hasReachNextMember
&& createFollowerClientWrapper(backOffer, followerUrlStr, leaderUrlStr)) {
logger.warn(
String.format("forward request to pd [%s] by pd [%s]", leaderUrlStr, followerUrlStr));
if (hasReachNextMember && createFollowerClientWrapper(followerUrlStr, leaderUrlStr)) {
return;
}
}
}
lastUpdateLeaderTime = System.currentTimeMillis();
if (pdClientWrapper == null) {
throw new TiClientInternalException(
"already tried all address on file, but not leader found yet.");
@ -638,11 +454,9 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
}
public void tryUpdateLeader() {
logger.info("try update leader");
for (URI url : this.pdAddrs) {
BackOffer backOffer = this.probeBackOffer();
// since resp is null, we need update leader's address by walking through all pd server.
GetMembersResponse resp = getMembers(backOffer, url);
GetMembersResponse resp = getMembers(url);
if (resp == null) {
continue;
}
@ -655,18 +469,15 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
// If leader is not change but becomes available, we can cancel follower forward.
if (checkHealth(backOffer, leaderUrlStr, hostMapping) && trySwitchLeader(leaderUrlStr)) {
if (checkHealth(leaderUrlStr, hostMapping) && trySwitchLeader(leaderUrlStr)) {
if (!urls.equals(this.pdAddrs)) {
tryUpdateMembers(urls);
}
return;
}
}
lastUpdateLeaderTime = System.currentTimeMillis();
if (pdClientWrapper == null) {
throw new TiClientInternalException(
"already tried all address on file, but not leader found yet.");
}
throw new TiClientInternalException(
"already tried all address on file, but not leader found yet.");
}
private synchronized void tryUpdateMembers(List<URI> members) {
@ -735,7 +546,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
}
@Override
protected PDFutureStub getAsyncStub() {
protected PDStub getAsyncStub() {
if (pdClientWrapper == null) {
throw new GrpcException("PDClient may not be initialized");
}
@ -743,11 +554,8 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
}
private void initCluster() {
logger.info("init cluster: start");
GetMembersResponse resp = null;
List<URI> pdAddrs = new ArrayList<>(getConf().getPdAddrs());
// shuffle PD addresses so that clients call getMembers from different PD
Collections.shuffle(pdAddrs);
List<URI> pdAddrs = getConf().getPdAddrs();
this.pdAddrs = pdAddrs;
this.etcdClient =
Client.builder()
@ -759,27 +567,13 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
.setDaemon(true)
.build()))
.build();
logger.info("init host mapping: start");
this.hostMapping =
Optional.ofNullable(getConf().getHostMapping())
.orElseGet(() -> new DefaultHostMapping(this.etcdClient, conf.getNetworkMappingName()));
logger.info("init host mapping: end");
// The first request may cost too much latency
long originTimeout = this.timeout;
this.timeout = conf.getPdFirstGetMemberTimeout();
this.hostMapping = new HostMapping(this.etcdClient, conf.getNetworkMappingName());
for (URI u : pdAddrs) {
logger.info("get members with pd " + u + ": start");
resp = getMembers(defaultBackOffer(), u);
logger.info("get members with pd " + u + ": end");
resp = getMembers(u);
if (resp != null) {
break;
}
}
if (resp == null) {
logger.error("Could not get leader member with: " + pdAddrs);
}
this.timeout = originTimeout;
checkNotNull(resp, "Failed to init client for PD cluster.");
long clusterId = resp.getHeader().getClusterId();
header = RequestHeader.newBuilder().setClusterId(clusterId).build();
@ -794,9 +588,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
String leaderUrlStr = resp.getLeader().getClientUrls(0);
leaderUrlStr = uriToAddr(addrToUri(leaderUrlStr));
logger.info("createLeaderClientWrapper with leader " + leaderUrlStr + ": start");
createLeaderClientWrapper(leaderUrlStr);
logger.info("createLeaderClientWrapper with leader " + leaderUrlStr + ": end");
service =
Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder()
@ -815,23 +607,20 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
10,
10,
TimeUnit.SECONDS);
if (conf.isTiFlashEnabled()) {
tiflashReplicaService =
Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder()
.setNameFormat("PDClient-tiflash-replica-pool-%d")
.setDaemon(true)
.build());
tiflashReplicaService.scheduleAtFixedRate(
this::updateTiFlashReplicaStatus, 10, 10, TimeUnit.SECONDS);
}
logger.info("init cluster: finish");
tiflashReplicaService =
Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder()
.setNameFormat("PDClient-tiflash-replica-pool-%d")
.setDaemon(true)
.build());
tiflashReplicaService.scheduleAtFixedRate(
this::updateTiFlashReplicaStatus, 10, 10, TimeUnit.SECONDS);
}
static class PDClientWrapper {
private final String leaderInfo;
private final PDBlockingStub blockingStub;
private final PDFutureStub asyncStub;
private final PDStub asyncStub;
private final long createTime;
private final String storeAddress;
@ -841,14 +630,11 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
Metadata header = new Metadata();
header.put(TiConfiguration.PD_FORWARD_META_DATA_KEY, addrToUri(leaderInfo).toString());
this.blockingStub =
PDGrpc.newBlockingStub(clientChannel)
.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header));
this.asyncStub =
PDGrpc.newFutureStub(clientChannel)
.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header));
MetadataUtils.attachHeaders(PDGrpc.newBlockingStub(clientChannel), header);
this.asyncStub = MetadataUtils.attachHeaders(PDGrpc.newStub(clientChannel), header);
} else {
this.blockingStub = PDGrpc.newBlockingStub(clientChannel);
this.asyncStub = PDGrpc.newFutureStub(clientChannel);
this.asyncStub = PDGrpc.newStub(clientChannel);
}
this.leaderInfo = leaderInfo;
this.storeAddress = storeAddress;
@ -867,7 +653,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
return blockingStub;
}
PDFutureStub getAsyncStub() {
PDStub getAsyncStub() {
return asyncStub;
}
@ -877,28 +663,7 @@ public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDFutureStub>
@Override
public String toString() {
return "[leaderInfo: " + leaderInfo + ", storeAddress: " + storeAddress + "]";
return "[leaderInfo: " + leaderInfo + "]";
}
}
public Long getClusterId() {
return header.getClusterId();
}
public List<URI> getPdAddrs() {
return pdAddrs;
}
public RequestKeyCodec getCodec() {
return codec;
}
private static BackOffer defaultBackOffer() {
return ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF);
}
private BackOffer probeBackOffer() {
int maxSleep = (int) getTimeout() * 2;
return ConcreteBackOffer.newCustomBackOff(maxSleep);
}
}

View File

@ -1,31 +1,26 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
import com.google.protobuf.ByteString;
import java.util.List;
import org.tikv.common.apiversion.RequestKeyCodec;
import org.tikv.common.meta.TiTimestamp;
import org.tikv.common.region.TiRegion;
import org.tikv.common.util.BackOffer;
import org.tikv.common.util.Pair;
import org.tikv.kvproto.Metapb;
import org.tikv.kvproto.Metapb.Store;
import org.tikv.kvproto.Pdpb;
/** Readonly PD client including only reading related interface Supposed for TiDB-like use cases */
public interface ReadOnlyPDClient {
@ -42,7 +37,7 @@ public interface ReadOnlyPDClient {
* @param key key in bytes for locating a region
* @return the region whose startKey and endKey range covers the given key
*/
Pair<Metapb.Region, Metapb.Peer> getRegionByKey(BackOffer backOffer, ByteString key);
TiRegion getRegionByKey(BackOffer backOffer, ByteString key);
/**
* Get Region by Region Id
@ -50,10 +45,7 @@ public interface ReadOnlyPDClient {
* @param id Region Id
* @return the region corresponding to the given Id
*/
Pair<Metapb.Region, Metapb.Peer> getRegionByID(BackOffer backOffer, long id);
List<Pdpb.Region> scanRegions(
BackOffer backOffer, ByteString startKey, ByteString endKey, int limit);
TiRegion getRegionByID(BackOffer backOffer, long id);
HostMapping getHostMapping();
@ -68,20 +60,4 @@ public interface ReadOnlyPDClient {
List<Store> getAllStores(BackOffer backOffer);
TiConfiguration.ReplicaRead getReplicaRead();
Long getClusterId();
RequestKeyCodec getCodec();
/**
* Update ServiceGCSafePoint
*
* @param serviceId ServiceId
* @param ttl TTL in seconds
* @param safePoint The TiTimestamp you want to set. Set to start_ts.getPrevious() is a good
* practice
* @return the MinSafePoint of all services. If this value is greater than safePoint, it means
* update failed
*/
Long updateServiceGCSafePoint(String serviceId, long ttl, long safePoint, BackOffer backOffer);
}

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
@ -80,9 +78,7 @@ public class Snapshot {
try (KVClient client = new KVClient(session, session.getRegionStoreClientBuilder())) {
List<KvPair> kvPairList =
client.batchGet(
ConcreteBackOffer.newCustomBackOff(backOffer, session.getPDClient().getClusterId()),
list,
timestamp.getVersion());
ConcreteBackOffer.newCustomBackOff(backOffer), list, timestamp.getVersion());
return kvPairList
.stream()
.map(

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2020 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
@ -27,6 +25,7 @@ import org.tikv.common.util.ConcreteBackOffer;
import org.tikv.kvproto.Metapb;
public class StoreVersion {
private static final int SCALE = 10000;
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private int v0 = 9999;
@ -61,8 +60,7 @@ public class StoreVersion {
public static boolean minTiKVVersion(String version, PDClient pdClient) {
StoreVersion storeVersion = new StoreVersion(version);
BackOffer bo =
ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF, pdClient.getClusterId());
BackOffer bo = ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF);
List<Metapb.Store> storeList =
pdClient
.getAllStores(bo)

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2019 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;

View File

@ -1,201 +1,46 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
import static org.tikv.common.ConfigUtils.DEF_BATCH_DELETE_CONCURRENCY;
import static org.tikv.common.ConfigUtils.DEF_BATCH_GET_CONCURRENCY;
import static org.tikv.common.ConfigUtils.DEF_BATCH_PUT_CONCURRENCY;
import static org.tikv.common.ConfigUtils.DEF_BATCH_SCAN_CONCURRENCY;
import static org.tikv.common.ConfigUtils.DEF_CHECK_HEALTH_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_DB_PREFIX;
import static org.tikv.common.ConfigUtils.DEF_DELETE_RANGE_CONCURRENCY;
import static org.tikv.common.ConfigUtils.DEF_FORWARD_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_GRPC_FORWARD_ENABLE;
import static org.tikv.common.ConfigUtils.DEF_HEALTH_CHECK_PERIOD_DURATION;
import static org.tikv.common.ConfigUtils.DEF_INDEX_SCAN_BATCH_SIZE;
import static org.tikv.common.ConfigUtils.DEF_INDEX_SCAN_CONCURRENCY;
import static org.tikv.common.ConfigUtils.DEF_KV_CLIENT_CONCURRENCY;
import static org.tikv.common.ConfigUtils.DEF_MAX_FRAME_SIZE;
import static org.tikv.common.ConfigUtils.DEF_METRICS_ENABLE;
import static org.tikv.common.ConfigUtils.DEF_METRICS_PORT;
import static org.tikv.common.ConfigUtils.DEF_PD_ADDRESSES;
import static org.tikv.common.ConfigUtils.DEF_REPLICA_READ;
import static org.tikv.common.ConfigUtils.DEF_SCAN_BATCH_SIZE;
import static org.tikv.common.ConfigUtils.DEF_SCAN_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_SHOW_ROWID;
import static org.tikv.common.ConfigUtils.DEF_TABLE_SCAN_CONCURRENCY;
import static org.tikv.common.ConfigUtils.DEF_TIFLASH_ENABLE;
import static org.tikv.common.ConfigUtils.DEF_TIKV_API_VERSION;
import static org.tikv.common.ConfigUtils.DEF_TIKV_BO_REGION_MISS_BASE_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_CONN_RECYCLE_TIME;
import static org.tikv.common.ConfigUtils.DEF_TIKV_ENABLE_ATOMIC_FOR_CAS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_IDLE_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_INGEST_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_KEEPALIVE_TIME;
import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_TIKV_GRPC_WARM_UP_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_TIKV_IMPORTER_MAX_KV_BATCH_BYTES;
import static org.tikv.common.ConfigUtils.DEF_TIKV_IMPORTER_MAX_KV_BATCH_SIZE;
import static org.tikv.common.ConfigUtils.DEF_TIKV_NETWORK_MAPPING_NAME;
import static org.tikv.common.ConfigUtils.DEF_TIKV_PD_FIRST_GET_MEMBER_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_READ_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_SCAN_SLOWLOG_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_RAWKV_WRITE_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_SCAN_REGIONS_LIMIT;
import static org.tikv.common.ConfigUtils.DEF_TIKV_SCATTER_WAIT_SECONDS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_TLS_ENABLE;
import static org.tikv.common.ConfigUtils.DEF_TIKV_TLS_RELOAD_INTERVAL;
import static org.tikv.common.ConfigUtils.DEF_TIKV_USE_JKS;
import static org.tikv.common.ConfigUtils.DEF_TIKV_WARM_UP_ENABLE;
import static org.tikv.common.ConfigUtils.DEF_TIMEOUT;
import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT;
import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE;
import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUST_VOLUMN_THRESHOLD;
import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS;
import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_ENABLE;
import static org.tikv.common.ConfigUtils.DEF_TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS;
import static org.tikv.common.ConfigUtils.FOLLOWER;
import static org.tikv.common.ConfigUtils.HIGH_COMMAND_PRIORITY;
import static org.tikv.common.ConfigUtils.LEADER_AND_FOLLOWER;
import static org.tikv.common.ConfigUtils.LOW_COMMAND_PRIORITY;
import static org.tikv.common.ConfigUtils.NORMAL_COMMAND_PRIORITY;
import static org.tikv.common.ConfigUtils.RAW_KV_MODE;
import static org.tikv.common.ConfigUtils.READ_COMMITTED_ISOLATION_LEVEL;
import static org.tikv.common.ConfigUtils.SNAPSHOT_ISOLATION_LEVEL;
import static org.tikv.common.ConfigUtils.TIFLASH_ENABLE;
import static org.tikv.common.ConfigUtils.TIKV_API_VERSION;
import static org.tikv.common.ConfigUtils.TIKV_BATCH_DELETE_CONCURRENCY;
import static org.tikv.common.ConfigUtils.TIKV_BATCH_GET_CONCURRENCY;
import static org.tikv.common.ConfigUtils.TIKV_BATCH_PUT_CONCURRENCY;
import static org.tikv.common.ConfigUtils.TIKV_BATCH_SCAN_CONCURRENCY;
import static org.tikv.common.ConfigUtils.TIKV_BO_REGION_MISS_BASE_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_CONN_RECYCLE_TIME;
import static org.tikv.common.ConfigUtils.TIKV_DB_PREFIX;
import static org.tikv.common.ConfigUtils.TIKV_DELETE_RANGE_CONCURRENCY;
import static org.tikv.common.ConfigUtils.TIKV_ENABLE_ATOMIC_FOR_CAS;
import static org.tikv.common.ConfigUtils.TIKV_ENABLE_GRPC_FORWARD;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_FORWARD_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_HEALTH_CHECK_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_IDLE_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_INGEST_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_KEEPALIVE_TIME;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_KEEPALIVE_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_MAX_FRAME_SIZE;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_SCAN_BATCH_SIZE;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_SCAN_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_GRPC_WARM_UP_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_HEALTH_CHECK_PERIOD_DURATION;
import static org.tikv.common.ConfigUtils.TIKV_IMPORTER_MAX_KV_BATCH_BYTES;
import static org.tikv.common.ConfigUtils.TIKV_IMPORTER_MAX_KV_BATCH_SIZE;
import static org.tikv.common.ConfigUtils.TIKV_INDEX_SCAN_BATCH_SIZE;
import static org.tikv.common.ConfigUtils.TIKV_INDEX_SCAN_CONCURRENCY;
import static org.tikv.common.ConfigUtils.TIKV_JKS_KEY_PASSWORD;
import static org.tikv.common.ConfigUtils.TIKV_JKS_KEY_PATH;
import static org.tikv.common.ConfigUtils.TIKV_JKS_TRUST_PASSWORD;
import static org.tikv.common.ConfigUtils.TIKV_JKS_TRUST_PATH;
import static org.tikv.common.ConfigUtils.TIKV_KEY_CERT_CHAIN;
import static org.tikv.common.ConfigUtils.TIKV_KEY_FILE;
import static org.tikv.common.ConfigUtils.TIKV_KV_CLIENT_CONCURRENCY;
import static org.tikv.common.ConfigUtils.TIKV_KV_MODE;
import static org.tikv.common.ConfigUtils.TIKV_METRICS_ENABLE;
import static org.tikv.common.ConfigUtils.TIKV_METRICS_PORT;
import static org.tikv.common.ConfigUtils.TIKV_NETWORK_MAPPING_NAME;
import static org.tikv.common.ConfigUtils.TIKV_PD_ADDRESSES;
import static org.tikv.common.ConfigUtils.TIKV_PD_FIRST_GET_MEMBER_TIMEOUT;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_BATCH_READ_SLOWLOG_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_BATCH_WRITE_SLOWLOG_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_READ_SLOWLOG_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_READ_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_SCAN_SLOWLOG_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_SCAN_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_SERVER_SLOWLOG_FACTOR;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_WRITE_SLOWLOG_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_RAWKV_WRITE_TIMEOUT_IN_MS;
import static org.tikv.common.ConfigUtils.TIKV_REPLICA_READ;
import static org.tikv.common.ConfigUtils.TIKV_REQUEST_COMMAND_PRIORITY;
import static org.tikv.common.ConfigUtils.TIKV_REQUEST_ISOLATION_LEVEL;
import static org.tikv.common.ConfigUtils.TIKV_SCAN_REGIONS_LIMIT;
import static org.tikv.common.ConfigUtils.TIKV_SCATTER_WAIT_SECONDS;
import static org.tikv.common.ConfigUtils.TIKV_SHOW_ROWID;
import static org.tikv.common.ConfigUtils.TIKV_TABLE_SCAN_CONCURRENCY;
import static org.tikv.common.ConfigUtils.TIKV_TLS_ENABLE;
import static org.tikv.common.ConfigUtils.TIKV_TLS_RELOAD_INTERVAL;
import static org.tikv.common.ConfigUtils.TIKV_TRUST_CERT_COLLECTION;
import static org.tikv.common.ConfigUtils.TIKV_USE_JKS;
import static org.tikv.common.ConfigUtils.TIKV_WARM_UP_ENABLE;
import static org.tikv.common.ConfigUtils.TXN_KV_MODE;
import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT;
import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE;
import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD;
import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS;
import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_ENABLE;
import static org.tikv.common.ConfigUtils.TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS;
import static org.tikv.common.ConfigUtils.*;
import com.google.protobuf.ByteString;
import io.grpc.Metadata;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tikv.common.pd.PDUtils;
import org.tikv.common.replica.ReplicaSelector;
import org.tikv.kvproto.Kvrpcpb;
import org.tikv.kvproto.Kvrpcpb.CommandPri;
import org.tikv.kvproto.Kvrpcpb.IsolationLevel;
public class TiConfiguration implements Serializable {
private static final Logger logger = LoggerFactory.getLogger(TiConfiguration.class);
private static final ConcurrentHashMap<String, String> settings = new ConcurrentHashMap<>();
public static final Metadata.Key<String> FORWARD_META_DATA_KEY =
public static final Metadata.Key FORWARD_META_DATA_KEY =
Metadata.Key.of("tikv-forwarded-host", Metadata.ASCII_STRING_MARSHALLER);
public static final Metadata.Key<String> PD_FORWARD_META_DATA_KEY =
public static final Metadata.Key PD_FORWARD_META_DATA_KEY =
Metadata.Key.of("pd-forwarded-host", Metadata.ASCII_STRING_MARSHALLER);
public static final ByteString API_V2_RAW_PREFIX = ByteString.copyFromUtf8("r");
public static final ByteString API_V2_TXN_PREFIX = ByteString.copyFromUtf8("x");
static {
// priority: system environment > config file > default
loadFromSystemProperties();
loadFromConfigurationFile();
loadFromDefaultProperties();
listAll();
}
private static void loadFromSystemProperties() {
@ -206,43 +51,12 @@ public class TiConfiguration implements Serializable {
}
}
private static void loadFromConfigurationFile() {
try (InputStream input =
TiConfiguration.class
.getClassLoader()
.getResourceAsStream(ConfigUtils.TIKV_CONFIGURATION_FILENAME)) {
Properties properties = new Properties();
if (input == null) {
logger.warn("Unable to find " + ConfigUtils.TIKV_CONFIGURATION_FILENAME);
return;
}
logger.info("loading " + ConfigUtils.TIKV_CONFIGURATION_FILENAME);
properties.load(input);
for (String key : properties.stringPropertyNames()) {
if (key.startsWith("tikv.")) {
String value = properties.getProperty(key);
setIfMissing(key, value);
}
}
} catch (IOException e) {
logger.error("load config file error", e);
}
}
private static void loadFromDefaultProperties() {
setIfMissing(TIKV_PD_ADDRESSES, DEF_PD_ADDRESSES);
setIfMissing(TIKV_GRPC_TIMEOUT, DEF_TIMEOUT);
setIfMissing(TIKV_GRPC_INGEST_TIMEOUT, DEF_TIKV_GRPC_INGEST_TIMEOUT);
setIfMissing(TIKV_GRPC_FORWARD_TIMEOUT, DEF_FORWARD_TIMEOUT);
setIfMissing(TIKV_GRPC_WARM_UP_TIMEOUT, DEF_TIKV_GRPC_WARM_UP_TIMEOUT);
setIfMissing(TIKV_PD_FIRST_GET_MEMBER_TIMEOUT, DEF_TIKV_PD_FIRST_GET_MEMBER_TIMEOUT);
setIfMissing(TIKV_GRPC_SCAN_TIMEOUT, DEF_SCAN_TIMEOUT);
setIfMissing(TIKV_GRPC_SCAN_BATCH_SIZE, DEF_SCAN_BATCH_SIZE);
setIfMissing(TIKV_GRPC_MAX_FRAME_SIZE, DEF_MAX_FRAME_SIZE);
setIfMissing(TIKV_CONN_RECYCLE_TIME, DEF_TIKV_CONN_RECYCLE_TIME);
setIfMissing(TIKV_TLS_RELOAD_INTERVAL, DEF_TIKV_TLS_RELOAD_INTERVAL);
setIfMissing(TIKV_INDEX_SCAN_BATCH_SIZE, DEF_INDEX_SCAN_BATCH_SIZE);
setIfMissing(TIKV_INDEX_SCAN_CONCURRENCY, DEF_INDEX_SCAN_CONCURRENCY);
setIfMissing(TIKV_TABLE_SCAN_CONCURRENCY, DEF_TABLE_SCAN_CONCURRENCY);
@ -265,48 +79,10 @@ public class TiConfiguration implements Serializable {
setIfMissing(TIKV_NETWORK_MAPPING_NAME, DEF_TIKV_NETWORK_MAPPING_NAME);
setIfMissing(TIKV_ENABLE_GRPC_FORWARD, DEF_GRPC_FORWARD_ENABLE);
setIfMissing(TIKV_GRPC_HEALTH_CHECK_TIMEOUT, DEF_CHECK_HEALTH_TIMEOUT);
setIfMissing(TIKV_HEALTH_CHECK_PERIOD_DURATION, DEF_HEALTH_CHECK_PERIOD_DURATION);
setIfMissing(TIKV_ENABLE_ATOMIC_FOR_CAS, DEF_TIKV_ENABLE_ATOMIC_FOR_CAS);
setIfMissing(TIKV_IMPORTER_MAX_KV_BATCH_BYTES, DEF_TIKV_IMPORTER_MAX_KV_BATCH_BYTES);
setIfMissing(TIKV_IMPORTER_MAX_KV_BATCH_SIZE, DEF_TIKV_IMPORTER_MAX_KV_BATCH_SIZE);
setIfMissing(TIKV_SCATTER_WAIT_SECONDS, DEF_TIKV_SCATTER_WAIT_SECONDS);
setIfMissing(TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS, DEF_TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS);
setIfMissing(TIKV_GRPC_KEEPALIVE_TIME, DEF_TIKV_GRPC_KEEPALIVE_TIME);
setIfMissing(TIKV_GRPC_KEEPALIVE_TIMEOUT, DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT);
setIfMissing(TIKV_GRPC_IDLE_TIMEOUT, DEF_TIKV_GRPC_IDLE_TIMEOUT);
setIfMissing(TIKV_TLS_ENABLE, DEF_TIKV_TLS_ENABLE);
setIfMissing(TIKV_USE_JKS, DEF_TIKV_USE_JKS);
setIfMissing(TIFLASH_ENABLE, DEF_TIFLASH_ENABLE);
setIfMissing(TIKV_WARM_UP_ENABLE, DEF_TIKV_WARM_UP_ENABLE);
setIfMissing(TIKV_RAWKV_READ_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_READ_TIMEOUT_IN_MS);
setIfMissing(TIKV_RAWKV_WRITE_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_WRITE_TIMEOUT_IN_MS);
setIfMissing(TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS);
setIfMissing(TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS);
setIfMissing(TIKV_RAWKV_SCAN_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS);
setIfMissing(TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS, DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS);
setIfMissing(TIKV_BO_REGION_MISS_BASE_IN_MS, DEF_TIKV_BO_REGION_MISS_BASE_IN_MS);
setIfMissing(TIKV_RAWKV_SCAN_SLOWLOG_IN_MS, DEF_TIKV_RAWKV_SCAN_SLOWLOG_IN_MS);
setIfMissing(TiKV_CIRCUIT_BREAK_ENABLE, DEF_TiKV_CIRCUIT_BREAK_ENABLE);
setIfMissing(
TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS,
DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS);
setIfMissing(
TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE,
DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE);
setIfMissing(
TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD,
DEF_TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUST_VOLUMN_THRESHOLD);
setIfMissing(
TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS, DEF_TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS);
setIfMissing(
TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT, DEF_TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT);
setIfMissing(TIKV_SCAN_REGIONS_LIMIT, DEF_TIKV_SCAN_REGIONS_LIMIT);
setIfMissing(TIKV_API_VERSION, DEF_TIKV_API_VERSION);
}
public static void listAll() {
logger.info("static configurations are:" + new ArrayList<>(settings.entrySet()));
logger.info(new ArrayList<>(settings.entrySet()).toString());
}
private static void set(String key, String value) {
@ -349,14 +125,10 @@ public class TiConfiguration implements Serializable {
return option.get();
}
public static int getInt(String key) {
private static int getInt(String key) {
return Integer.parseInt(get(key));
}
public static Optional<Integer> getIntOption(String key) {
return getOption(key).map(Integer::parseInt);
}
private static int getInt(String key, int defaultValue) {
try {
return getOption(key).map(Integer::parseInt).orElse(defaultValue);
@ -464,13 +236,8 @@ public class TiConfiguration implements Serializable {
}
private long timeout = getTimeAsMs(TIKV_GRPC_TIMEOUT);
private long ingestTimeout = getTimeAsMs(TIKV_GRPC_INGEST_TIMEOUT);
private long forwardTimeout = getTimeAsMs(TIKV_GRPC_FORWARD_TIMEOUT);
private long warmUpTimeout = getTimeAsMs(TIKV_GRPC_WARM_UP_TIMEOUT);
private long pdFirstGetMemberTimeout = getTimeAsMs(TIKV_PD_FIRST_GET_MEMBER_TIMEOUT);
private long scanTimeout = getTimeAsMs(TIKV_GRPC_SCAN_TIMEOUT);
private int maxFrameSize = getInt(TIKV_GRPC_MAX_FRAME_SIZE);
private long connRecycleTime = getTimeAsSeconds(TIKV_CONN_RECYCLE_TIME);
private List<URI> pdAddrs = getPdAddrs(TIKV_PD_ADDRESSES);
private int indexScanBatchSize = getInt(TIKV_INDEX_SCAN_BATCH_SIZE);
private int indexScanConcurrency = getInt(TIKV_INDEX_SCAN_CONCURRENCY);
@ -495,70 +262,8 @@ public class TiConfiguration implements Serializable {
private boolean metricsEnable = getBoolean(TIKV_METRICS_ENABLE);
private int metricsPort = getInt(TIKV_METRICS_PORT);
private int grpcHealthCheckTimeout = getInt(TIKV_GRPC_HEALTH_CHECK_TIMEOUT);
private int healthCheckPeriodDuration = getInt(TIKV_HEALTH_CHECK_PERIOD_DURATION);
private final String networkMappingName = get(TIKV_NETWORK_MAPPING_NAME);
private HostMapping hostMapping = null;
private boolean enableAtomicForCAS = getBoolean(TIKV_ENABLE_ATOMIC_FOR_CAS);
private int importerMaxKVBatchBytes = getInt(TIKV_IMPORTER_MAX_KV_BATCH_BYTES);
private int importerMaxKVBatchSize = getInt(TIKV_IMPORTER_MAX_KV_BATCH_SIZE);
private int scatterWaitSeconds = getInt(TIKV_SCATTER_WAIT_SECONDS);
private int rawKVDefaultBackoffInMS = getInt(TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS);
private int rawKVReadTimeoutInMS = getInt(TIKV_RAWKV_READ_TIMEOUT_IN_MS);
private int rawKVWriteTimeoutInMS = getInt(TIKV_RAWKV_WRITE_TIMEOUT_IN_MS);
private int rawKVBatchReadTimeoutInMS = getInt(TIKV_RAWKV_BATCH_READ_TIMEOUT_IN_MS);
private int rawKVBatchWriteTimeoutInMS = getInt(TIKV_RAWKV_BATCH_WRITE_TIMEOUT_IN_MS);
private int rawKVScanTimeoutInMS = getInt(TIKV_RAWKV_SCAN_TIMEOUT_IN_MS);
private int rawKVCleanTimeoutInMS = getInt(TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS);
private Integer rawKVReadSlowLogInMS = getIntOption(TIKV_RAWKV_READ_SLOWLOG_IN_MS).orElse(null);
private Integer rawKVWriteSlowLogInMS = getIntOption(TIKV_RAWKV_WRITE_SLOWLOG_IN_MS).orElse(null);
private Integer rawKVBatchReadSlowLogInMS =
getIntOption(TIKV_RAWKV_BATCH_READ_SLOWLOG_IN_MS).orElse(null);
private Integer rawKVBatchWriteSlowLogInMS =
getIntOption(TIKV_RAWKV_BATCH_WRITE_SLOWLOG_IN_MS).orElse(null);
private int rawKVScanSlowLogInMS = getInt(TIKV_RAWKV_SCAN_SLOWLOG_IN_MS);
private double rawKVServerSlowLogFactor = getDouble(TIKV_RAWKV_SERVER_SLOWLOG_FACTOR, 0.5);
private boolean tlsEnable = getBoolean(TIKV_TLS_ENABLE);
private long certReloadInterval = getTimeAsSeconds(TIKV_TLS_RELOAD_INTERVAL);
private String trustCertCollectionFile = getOption(TIKV_TRUST_CERT_COLLECTION).orElse(null);
private String keyCertChainFile = getOption(TIKV_KEY_CERT_CHAIN).orElse(null);
private String keyFile = getOption(TIKV_KEY_FILE).orElse(null);
private boolean useJks = getBoolean(TIKV_USE_JKS);
private String jksKeyPath = getOption(TIKV_JKS_KEY_PATH).orElse(null);
private String jksKeyPassword = getOption(TIKV_JKS_KEY_PASSWORD).orElse(null);
private String jksTrustPath = getOption(TIKV_JKS_TRUST_PATH).orElse(null);
private String jksTrustPassword = getOption(TIKV_JKS_TRUST_PASSWORD).orElse(null);
private final boolean tiFlashEnable = getBoolean(TIFLASH_ENABLE);
private boolean warmUpEnable = getBoolean(TIKV_WARM_UP_ENABLE);
private boolean isTest = false;
private int keepaliveTime = getInt(TIKV_GRPC_KEEPALIVE_TIME);
private int keepaliveTimeout = getInt(TIKV_GRPC_KEEPALIVE_TIMEOUT);
private int idleTimeout = getInt(TIKV_GRPC_IDLE_TIMEOUT);
private boolean circuitBreakEnable = getBoolean(TiKV_CIRCUIT_BREAK_ENABLE);
private int circuitBreakAvailabilityWindowInSeconds =
getInt(TiKV_CIRCUIT_BREAK_AVAILABILITY_WINDOW_IN_SECONDS);
private int circuitBreakAvailabilityErrorThresholdPercentage =
getInt(TiKV_CIRCUIT_BREAK_AVAILABILITY_ERROR_THRESHOLD_PERCENTAGE);
private int circuitBreakAvailabilityRequestVolumnThreshold =
getInt(TiKV_CIRCUIT_BREAK_AVAILABILITY_REQUEST_VOLUMN_THRESHOLD);
private int circuitBreakSleepWindowInSeconds = getInt(TiKV_CIRCUIT_BREAK_SLEEP_WINDOW_IN_SECONDS);
private int circuitBreakAttemptRequestCount = getInt(TiKV_CIRCUIT_BREAK_ATTEMPT_REQUEST_COUNT);
private int scanRegionsLimit = getInt(TIKV_SCAN_REGIONS_LIMIT);
private ApiVersion apiVersion = ApiVersion.fromInt(getInt(TIKV_API_VERSION));
public enum KVMode {
TXN,
@ -571,12 +276,6 @@ public class TiConfiguration implements Serializable {
LEADER_AND_FOLLOWER
}
public TiConfiguration() {
if (rawKVServerSlowLogFactor < 0 || rawKVServerSlowLogFactor > 1) {
throw new IllegalArgumentException("rawkv_server_slowlog_factor must be in range [0, 1]");
}
}
public static TiConfiguration createDefault() {
return new TiConfiguration();
}
@ -631,40 +330,6 @@ public class TiConfiguration implements Serializable {
return this;
}
public long getIngestTimeout() {
return ingestTimeout;
}
public void setIngestTimeout(long ingestTimeout) {
this.ingestTimeout = ingestTimeout;
}
public long getForwardTimeout() {
return forwardTimeout;
}
public TiConfiguration setForwardTimeout(long timeout) {
this.forwardTimeout = timeout;
return this;
}
public long getWarmUpTimeout() {
return warmUpTimeout;
}
public TiConfiguration setWarmUpTimeout(long timeout) {
this.warmUpTimeout = timeout;
return this;
}
public long getPdFirstGetMemberTimeout() {
return pdFirstGetMemberTimeout;
}
public void setPdFirstGetMemberTimeout(long pdFirstGetMemberTimeout) {
this.pdFirstGetMemberTimeout = pdFirstGetMemberTimeout;
}
public long getScanTimeout() {
return scanTimeout;
}
@ -695,15 +360,6 @@ public class TiConfiguration implements Serializable {
return this;
}
public long getConnRecycleTimeInSeconds() {
return connRecycleTime;
}
public TiConfiguration setConnRecycleTimeInSeconds(int connRecycleTime) {
this.connRecycleTime = connRecycleTime;
return this;
}
public int getIndexScanBatchSize() {
return indexScanBatchSize;
}
@ -816,14 +472,6 @@ public class TiConfiguration implements Serializable {
return kvMode;
}
public boolean isRawKVMode() {
return getKvMode() == TiConfiguration.KVMode.RAW;
}
public boolean isTxnKVMode() {
return getKvMode() == KVMode.TXN;
}
public TiConfiguration setKvMode(String kvMode) {
this.kvMode = KVMode.valueOf(kvMode);
return this;
@ -894,404 +542,11 @@ public class TiConfiguration implements Serializable {
return this.networkMappingName;
}
public HostMapping getHostMapping() {
return hostMapping;
}
public void setHostMapping(HostMapping mapping) {
this.hostMapping = mapping;
}
public boolean getEnableGrpcForward() {
return this.enableGrpcForward;
}
public void setEnableGrpcForward(boolean enableGrpcForward) {
this.enableGrpcForward = enableGrpcForward;
}
public long getGrpcHealthCheckTimeout() {
return this.grpcHealthCheckTimeout;
}
public void setGrpcHealthCheckTimeout(int grpcHealthCheckTimeout) {
this.grpcHealthCheckTimeout = grpcHealthCheckTimeout;
}
public long getHealthCheckPeriodDuration() {
return this.healthCheckPeriodDuration;
}
public void setHealthCheckPeriodDuration(int healthCheckPeriodDuration) {
this.healthCheckPeriodDuration = healthCheckPeriodDuration;
}
public boolean isEnableAtomicForCAS() {
return enableAtomicForCAS;
}
public void setEnableAtomicForCAS(boolean enableAtomicForCAS) {
this.enableAtomicForCAS = enableAtomicForCAS;
}
public int getImporterMaxKVBatchBytes() {
return importerMaxKVBatchBytes;
}
public void setImporterMaxKVBatchBytes(int importerMaxKVBatchBytes) {
this.importerMaxKVBatchBytes = importerMaxKVBatchBytes;
}
public int getImporterMaxKVBatchSize() {
return importerMaxKVBatchSize;
}
public void setImporterMaxKVBatchSize(int importerMaxKVBatchSize) {
this.importerMaxKVBatchSize = importerMaxKVBatchSize;
}
public int getScatterWaitSeconds() {
return scatterWaitSeconds;
}
public void setScatterWaitSeconds(int scatterWaitSeconds) {
this.scatterWaitSeconds = scatterWaitSeconds;
}
public int getRawKVDefaultBackoffInMS() {
return rawKVDefaultBackoffInMS;
}
public void setRawKVDefaultBackoffInMS(int rawKVDefaultBackoffInMS) {
this.rawKVDefaultBackoffInMS = rawKVDefaultBackoffInMS;
}
public boolean isTest() {
return isTest;
}
public void setTest(boolean test) {
isTest = test;
}
public int getKeepaliveTime() {
return keepaliveTime;
}
public void setKeepaliveTime(int keepaliveTime) {
this.keepaliveTime = keepaliveTime;
}
public int getKeepaliveTimeout() {
return keepaliveTimeout;
}
public void setKeepaliveTimeout(int timeout) {
this.keepaliveTimeout = timeout;
}
public int getIdleTimeout() {
return idleTimeout;
}
public void setIdleTimeout(int timeout) {
this.idleTimeout = timeout;
}
public boolean isTiFlashEnabled() {
return tiFlashEnable;
}
public boolean isWarmUpEnable() {
return warmUpEnable;
}
public void setWarmUpEnable(boolean warmUpEnable) {
this.warmUpEnable = warmUpEnable;
}
public boolean isTlsEnable() {
return tlsEnable;
}
public long getCertReloadIntervalInSeconds() {
return certReloadInterval;
}
public TiConfiguration setCertReloadIntervalInSeconds(long interval) {
this.certReloadInterval = interval;
return this;
}
public void setTlsEnable(boolean tlsEnable) {
this.tlsEnable = tlsEnable;
}
public String getTrustCertCollectionFile() {
return trustCertCollectionFile;
}
public void setTrustCertCollectionFile(String trustCertCollectionFile) {
this.trustCertCollectionFile = trustCertCollectionFile;
}
public String getKeyCertChainFile() {
return keyCertChainFile;
}
public void setKeyCertChainFile(String keyCertChainFile) {
this.keyCertChainFile = keyCertChainFile;
}
public String getKeyFile() {
return keyFile;
}
public void setKeyFile(String keyFile) {
this.keyFile = keyFile;
}
public boolean isJksEnable() {
return useJks;
}
public void setJksEnable(boolean useJks) {
this.useJks = useJks;
}
public String getJksKeyPath() {
return jksKeyPath;
}
public void setJksKeyPath(String jksKeyPath) {
this.jksKeyPath = jksKeyPath;
}
public String getJksKeyPassword() {
return jksKeyPassword;
}
public void setJksKeyPassword(String jksKeyPassword) {
this.jksKeyPassword = jksKeyPassword;
}
public String getJksTrustPath() {
return jksTrustPath;
}
public void setJksTrustPath(String jksTrustPath) {
this.jksTrustPath = jksTrustPath;
}
public String getJksTrustPassword() {
return jksTrustPassword;
}
public void setJksTrustPassword(String jksTrustPassword) {
this.jksTrustPassword = jksTrustPassword;
}
public int getRawKVReadTimeoutInMS() {
return rawKVReadTimeoutInMS;
}
public void setRawKVReadTimeoutInMS(int rawKVReadTimeoutInMS) {
this.rawKVReadTimeoutInMS = rawKVReadTimeoutInMS;
}
public int getRawKVWriteTimeoutInMS() {
return rawKVWriteTimeoutInMS;
}
public void setRawKVWriteTimeoutInMS(int rawKVWriteTimeoutInMS) {
this.rawKVWriteTimeoutInMS = rawKVWriteTimeoutInMS;
}
public int getRawKVBatchReadTimeoutInMS() {
return rawKVBatchReadTimeoutInMS;
}
public void setRawKVBatchReadTimeoutInMS(int rawKVBatchReadTimeoutInMS) {
this.rawKVBatchReadTimeoutInMS = rawKVBatchReadTimeoutInMS;
}
public int getRawKVBatchWriteTimeoutInMS() {
return rawKVBatchWriteTimeoutInMS;
}
public void setRawKVBatchWriteTimeoutInMS(int rawKVBatchWriteTimeoutInMS) {
this.rawKVBatchWriteTimeoutInMS = rawKVBatchWriteTimeoutInMS;
}
public int getRawKVScanTimeoutInMS() {
return rawKVScanTimeoutInMS;
}
public void setRawKVScanTimeoutInMS(int rawKVScanTimeoutInMS) {
this.rawKVScanTimeoutInMS = rawKVScanTimeoutInMS;
}
public int getRawKVCleanTimeoutInMS() {
return rawKVCleanTimeoutInMS;
}
public void setRawKVCleanTimeoutInMS(int rawKVCleanTimeoutInMS) {
this.rawKVCleanTimeoutInMS = rawKVCleanTimeoutInMS;
}
public Integer getRawKVReadSlowLogInMS() {
return Optional.ofNullable(rawKVReadSlowLogInMS).orElse((int) (getTimeout() * 2));
}
public void setRawKVReadSlowLogInMS(Integer rawKVReadSlowLogInMS) {
this.rawKVReadSlowLogInMS = rawKVReadSlowLogInMS;
}
public Integer getRawKVWriteSlowLogInMS() {
return Optional.ofNullable(rawKVWriteSlowLogInMS).orElse((int) (getTimeout() * 2));
}
public void setRawKVWriteSlowLogInMS(Integer rawKVWriteSlowLogInMS) {
this.rawKVWriteSlowLogInMS = rawKVWriteSlowLogInMS;
}
public Integer getRawKVBatchReadSlowLogInMS() {
return Optional.ofNullable(rawKVBatchReadSlowLogInMS).orElse((int) (getTimeout() * 2));
}
public void setRawKVBatchReadSlowLogInMS(Integer rawKVBatchReadSlowLogInMS) {
this.rawKVBatchReadSlowLogInMS = rawKVBatchReadSlowLogInMS;
}
public Integer getRawKVBatchWriteSlowLogInMS() {
return Optional.ofNullable(rawKVBatchWriteSlowLogInMS).orElse((int) (getTimeout() * 2));
}
public void setRawKVBatchWriteSlowLogInMS(Integer rawKVBatchWriteSlowLogInMS) {
this.rawKVBatchWriteSlowLogInMS = rawKVBatchWriteSlowLogInMS;
}
public int getRawKVScanSlowLogInMS() {
return rawKVScanSlowLogInMS;
}
public void setRawKVScanSlowLogInMS(int rawKVScanSlowLogInMS) {
this.rawKVScanSlowLogInMS = rawKVScanSlowLogInMS;
}
public double getRawKVServerSlowLogFactor() {
return rawKVServerSlowLogFactor;
}
public void setRawKVServerSlowLogFactor(double rawKVServerSlowLogFactor) {
if (rawKVServerSlowLogFactor < 0 || rawKVServerSlowLogFactor > 1) {
throw new IllegalArgumentException("rawkv_server_slowlog_factor must be in range [0, 1]");
}
this.rawKVServerSlowLogFactor = rawKVServerSlowLogFactor;
}
public boolean isCircuitBreakEnable() {
return circuitBreakEnable;
}
public void setCircuitBreakEnable(boolean circuitBreakEnable) {
this.circuitBreakEnable = circuitBreakEnable;
}
public int getCircuitBreakAvailabilityWindowInSeconds() {
return circuitBreakAvailabilityWindowInSeconds;
}
public void setCircuitBreakAvailabilityWindowInSeconds(
int circuitBreakAvailabilityWindowInSeconds) {
this.circuitBreakAvailabilityWindowInSeconds = circuitBreakAvailabilityWindowInSeconds;
}
public int getCircuitBreakAvailabilityErrorThresholdPercentage() {
return circuitBreakAvailabilityErrorThresholdPercentage;
}
public void setCircuitBreakAvailabilityErrorThresholdPercentage(
int circuitBreakAvailabilityErrorThresholdPercentage) {
this.circuitBreakAvailabilityErrorThresholdPercentage =
circuitBreakAvailabilityErrorThresholdPercentage;
}
public int getCircuitBreakAvailabilityRequestVolumnThreshold() {
return circuitBreakAvailabilityRequestVolumnThreshold;
}
public void setCircuitBreakAvailabilityRequestVolumnThreshold(
int circuitBreakAvailabilityRequestVolumnThreshold) {
this.circuitBreakAvailabilityRequestVolumnThreshold =
circuitBreakAvailabilityRequestVolumnThreshold;
}
public int getCircuitBreakSleepWindowInSeconds() {
return circuitBreakSleepWindowInSeconds;
}
public void setCircuitBreakSleepWindowInSeconds(int circuitBreakSleepWindowInSeconds) {
this.circuitBreakSleepWindowInSeconds = circuitBreakSleepWindowInSeconds;
}
public int getCircuitBreakAttemptRequestCount() {
return circuitBreakAttemptRequestCount;
}
public void setCircuitBreakAttemptRequestCount(int circuitBreakAttemptRequestCount) {
this.circuitBreakAttemptRequestCount = circuitBreakAttemptRequestCount;
}
public int getScanRegionsLimit() {
return scanRegionsLimit;
}
public void setScanRegionsLimit(int scanRegionsLimit) {
this.scanRegionsLimit = scanRegionsLimit;
}
public ApiVersion getApiVersion() {
return apiVersion;
}
public TiConfiguration setApiVersion(ApiVersion version) {
this.apiVersion = version;
return this;
}
public enum ApiVersion {
V1,
V2;
public static ApiVersion fromInt(int version) {
switch (version) {
case 1:
return V1;
case 2:
return V2;
default:
throw new IllegalArgumentException("unknown api version " + version);
}
}
public boolean isV1() {
return this == V1;
}
public boolean isV2() {
return this == V2;
}
public Kvrpcpb.APIVersion toPb() {
switch (this) {
case V1:
return Kvrpcpb.APIVersion.V1;
case V2:
return Kvrpcpb.APIVersion.V2;
default:
throw new IllegalArgumentException("unknown api version " + this);
}
}
}
}

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2019 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
@ -41,7 +39,6 @@ public class TiDBJDBCClient implements AutoCloseable {
private static final int DELAY_CLEAN_TABLE_LOCK_DEFAULT = 0;
private static final String TIDB_ROW_FORMAT_VERSION_SQL = "select @@tidb_row_format_version";
private static final int TIDB_ROW_FORMAT_VERSION_DEFAULT = 1;
private static final ObjectMapper objectMapper = new ObjectMapper();
private final Logger logger = LoggerFactory.getLogger(getClass().getName());
private final Connection connection;
@ -121,6 +118,7 @@ public class TiDBJDBCClient implements AutoCloseable {
private Map<String, Object> readConfMapFromTiDB() throws SQLException, IOException {
String configJSON = (String) queryTiDBViaJDBC(SELECT_TIDB_CONFIG_SQL).get(0).get(0);
ObjectMapper objectMapper = new ObjectMapper();
TypeReference<HashMap<String, Object>> typeRef =
new TypeReference<HashMap<String, Object>>() {};
return objectMapper.readValue(configJSON, typeRef);

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;
@ -22,61 +20,46 @@ import static org.tikv.common.util.ClientUtils.groupKeysByRegion;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.ByteString;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.exporter.HTTPServer;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tikv.common.apiversion.RequestKeyCodec;
import org.tikv.common.apiversion.RequestKeyV1RawCodec;
import org.tikv.common.apiversion.RequestKeyV1TxnCodec;
import org.tikv.common.apiversion.RequestKeyV2RawCodec;
import org.tikv.common.apiversion.RequestKeyV2TxnCodec;
import org.tikv.common.catalog.Catalog;
import org.tikv.common.event.CacheInvalidateEvent;
import org.tikv.common.exception.TiKVException;
import org.tikv.common.importer.ImporterStoreClient;
import org.tikv.common.importer.SwitchTiKVModeClient;
import org.tikv.common.key.Key;
import org.tikv.common.meta.TiTimestamp;
import org.tikv.common.policy.RetryPolicy;
import org.tikv.common.region.RegionManager;
import org.tikv.common.region.RegionStoreClient;
import org.tikv.common.region.RegionStoreClient.RegionStoreClientBuilder;
import org.tikv.common.region.TiRegion;
import org.tikv.common.region.TiStore;
import org.tikv.common.util.BackOffFunction;
import org.tikv.common.util.BackOffer;
import org.tikv.common.util.ChannelFactory;
import org.tikv.common.util.ConcreteBackOffer;
import org.tikv.common.util.Pair;
import org.tikv.kvproto.Errorpb;
import org.tikv.kvproto.ImportSstpb;
import org.tikv.kvproto.Metapb;
import org.tikv.kvproto.Pdpb;
import org.tikv.common.util.*;
import org.tikv.raw.RawKVClient;
import org.tikv.raw.SmartRawKVClient;
import org.tikv.service.failsafe.CircuitBreaker;
import org.tikv.service.failsafe.CircuitBreakerImpl;
import org.tikv.txn.KVClient;
import org.tikv.txn.TxnKVClient;
/**
* TiSession is the holder for PD Client, Store pdClient and PD Cache All sessions share common
* region store connection pool but separated PD conn and cache for better concurrency
*
* <p>TiSession is thread-safe but it's also recommended to have multiple session avoiding lock
* contention
* region store connection pool but separated PD conn and cache for better concurrency TiSession is
* thread-safe but it's also recommended to have multiple session avoiding lock contention
*/
public class TiSession implements AutoCloseable {
private static final Logger logger = LoggerFactory.getLogger(TiSession.class);
private static final Map<String, TiSession> sessionCachedMap = new HashMap<>();
private final TiConfiguration conf;
private final RequestKeyCodec keyCodec;
private final ChannelFactory channelFactory;
private Function<CacheInvalidateEvent, Void> cacheInvalidateCallback;
// below object creation is either heavy or making connection (pd), pending for lazy loading
private volatile PDClient client;
private volatile Catalog catalog;
@ -88,184 +71,38 @@ public class TiSession implements AutoCloseable {
private volatile ExecutorService batchScanThreadPool;
private volatile ExecutorService deleteRangeThreadPool;
private volatile RegionManager regionManager;
private final boolean enableGrpcForward;
private volatile boolean enableGrpcForward;
private volatile RegionStoreClient.RegionStoreClientBuilder clientBuilder;
private volatile ImporterStoreClient.ImporterStoreClientBuilder importerClientBuilder;
private volatile boolean isClosed = false;
private volatile SwitchTiKVModeClient switchTiKVModeClient;
private final MetricsServer metricsServer;
private final CircuitBreaker circuitBreaker;
private static final int MAX_SPLIT_REGION_STACK_DEPTH = 6;
static {
logger.info("Welcome to TiKV Java Client {}", getVersionInfo());
}
private static class VersionInfo {
private final String buildVersion;
private final String commitHash;
public VersionInfo(String buildVersion, String commitHash) {
this.buildVersion = buildVersion;
this.commitHash = commitHash;
}
@Override
public String toString() {
return buildVersion + "@" + commitHash;
}
}
private boolean isClosed = false;
private HTTPServer server;
private CollectorRegistry collectorRegistry;
public TiSession(TiConfiguration conf) {
// may throw org.tikv.common.MetricsServer - http server not up
// put it at the beginning of this function to avoid unclosed Thread
this.metricsServer = MetricsServer.getInstance(conf);
this.conf = conf;
if (conf.getApiVersion().isV1()) {
if (conf.isRawKVMode()) {
keyCodec = new RequestKeyV1RawCodec();
} else {
keyCodec = new RequestKeyV1TxnCodec();
}
} else {
if (conf.isRawKVMode()) {
keyCodec = new RequestKeyV2RawCodec();
} else {
keyCodec = new RequestKeyV2TxnCodec();
}
}
if (conf.isTlsEnable()) {
if (conf.isJksEnable()) {
this.channelFactory =
new ChannelFactory(
conf.getMaxFrameSize(),
conf.getKeepaliveTime(),
conf.getKeepaliveTimeout(),
conf.getIdleTimeout(),
conf.getConnRecycleTimeInSeconds(),
conf.getCertReloadIntervalInSeconds(),
conf.getJksKeyPath(),
conf.getJksKeyPassword(),
conf.getJksTrustPath(),
conf.getJksTrustPassword());
} else {
this.channelFactory =
new ChannelFactory(
conf.getMaxFrameSize(),
conf.getKeepaliveTime(),
conf.getKeepaliveTimeout(),
conf.getIdleTimeout(),
conf.getConnRecycleTimeInSeconds(),
conf.getCertReloadIntervalInSeconds(),
conf.getTrustCertCollectionFile(),
conf.getKeyCertChainFile(),
conf.getKeyFile());
}
} else {
this.channelFactory =
new ChannelFactory(
conf.getMaxFrameSize(),
conf.getKeepaliveTime(),
conf.getKeepaliveTimeout(),
conf.getIdleTimeout());
}
this.client = PDClient.createRaw(conf, keyCodec, channelFactory);
if (conf.getApiVersion().isV2() && !StoreVersion.minTiKVVersion(Version.API_V2, client)) {
throw new IllegalStateException(
"With API v2, store versions should not older than " + Version.API_V2);
}
this.channelFactory = new ChannelFactory(conf.getMaxFrameSize());
this.client = PDClient.createRaw(conf, channelFactory);
this.enableGrpcForward = conf.getEnableGrpcForward();
if (this.enableGrpcForward) {
logger.info("enable grpc forward for high available");
}
if (conf.isWarmUpEnable() && conf.isRawKVMode()) {
warmUp();
}
this.circuitBreaker = new CircuitBreakerImpl(conf, client.getClusterId());
logger.info(
"TiSession initialized in "
+ conf.getKvMode()
+ " mode in API version: "
+ conf.getApiVersion());
}
private static VersionInfo getVersionInfo() {
VersionInfo info;
try {
final Properties properties = new Properties();
properties.load(TiSession.class.getClassLoader().getResourceAsStream("git.properties"));
String version = properties.getProperty("git.build.version");
String commitHash = properties.getProperty("git.commit.id.full");
info = new VersionInfo(version, commitHash);
} catch (Exception e) {
logger.info("Fail to read package info: " + e.getMessage());
info = new VersionInfo("unknown", "unknown");
}
return info;
}
@VisibleForTesting
public synchronized void warmUp() {
long warmUpStartTime = System.nanoTime();
BackOffer backOffer = ConcreteBackOffer.newRawKVBackOff(getPDClient().getClusterId());
try {
// let JVM ClassLoader load gRPC error related classes
// this operation may cost 100ms
Errorpb.Error.newBuilder().setNotLeader(Errorpb.NotLeader.newBuilder().build()).build();
this.client = getPDClient();
this.regionManager = getRegionManager();
List<Metapb.Store> stores = this.client.getAllStores(backOffer);
// warm up store cache
for (Metapb.Store store : stores) {
this.regionManager.updateStore(
null, new TiStore(this.client.getStore(backOffer, store.getId())));
if (conf.isMetricsEnable()) {
try {
this.collectorRegistry = new CollectorRegistry();
this.collectorRegistry.register(RawKVClient.RAW_REQUEST_LATENCY);
this.collectorRegistry.register(RawKVClient.RAW_REQUEST_FAILURE);
this.collectorRegistry.register(RawKVClient.RAW_REQUEST_SUCCESS);
this.collectorRegistry.register(RegionStoreClient.GRPC_RAW_REQUEST_LATENCY);
this.collectorRegistry.register(RetryPolicy.GRPC_SINGLE_REQUEST_LATENCY);
this.collectorRegistry.register(RegionManager.GET_REGION_BY_KEY_REQUEST_LATENCY);
this.collectorRegistry.register(PDClient.PD_GET_REGION_BY_KEY_REQUEST_LATENCY);
this.enableGrpcForward = conf.getEnableGrpcForward();
this.server =
new HTTPServer(
new InetSocketAddress(conf.getMetricsPort()), this.collectorRegistry, true);
logger.info("http server is up " + this.server.getPort());
} catch (Exception e) {
logger.error("http server not up");
throw new RuntimeException(e);
}
// use scan region to load region cache with limit
ByteString startKey = ByteString.EMPTY;
do {
List<Pdpb.Region> regions =
regionManager.scanRegions(
backOffer, startKey, ByteString.EMPTY, conf.getScanRegionsLimit());
if (regions == null || regions.isEmpty()) {
// something went wrong, but the warm-up process could continue
break;
}
for (Pdpb.Region region : regions) {
regionManager.insertRegionToCache(
regionManager.createRegion(region.getRegion(), backOffer));
}
startKey = regions.get(regions.size() - 1).getRegion().getEndKey();
} while (!startKey.isEmpty());
try (RawKVClient rawKVClient = createRawClient()) {
ByteString exampleKey = ByteString.EMPTY;
Optional<ByteString> prev = rawKVClient.get(exampleKey);
if (prev.isPresent()) {
rawKVClient.delete(exampleKey);
rawKVClient.putIfAbsent(exampleKey, prev.get());
rawKVClient.put(exampleKey, prev.get());
} else {
rawKVClient.putIfAbsent(exampleKey, ByteString.EMPTY);
rawKVClient.put(exampleKey, ByteString.EMPTY);
rawKVClient.delete(exampleKey);
}
}
} catch (Exception e) {
// ignore error
logger.info("warm up fails, ignored ", e);
} finally {
logger.info(
String.format(
"warm up duration %d ms", (System.nanoTime() - warmUpStartTime) / 1_000_000));
}
logger.info("TiSession initialized in " + conf.getKvMode() + " mode");
}
@VisibleForTesting
@ -288,66 +125,31 @@ public class TiSession implements AutoCloseable {
}
public RawKVClient createRawClient() {
checkIsClosed();
return new RawKVClient(this, this.getRegionStoreClientBuilder());
}
public SmartRawKVClient createSmartRawClient() {
RawKVClient rawKVClient = createRawClient();
return new SmartRawKVClient(rawKVClient, circuitBreaker);
RegionStoreClientBuilder builder =
new RegionStoreClientBuilder(conf, channelFactory, this.getRegionManager(), client);
return new RawKVClient(this, builder);
}
public KVClient createKVClient() {
checkIsClosed();
return new KVClient(this.conf, this.getRegionStoreClientBuilder(), this);
RegionStoreClientBuilder builder =
new RegionStoreClientBuilder(conf, channelFactory, this.getRegionManager(), client);
return new KVClient(conf, builder);
}
public TxnKVClient createTxnClient() {
checkIsClosed();
return new TxnKVClient(conf, this.getRegionStoreClientBuilder(), this.getPDClient());
}
public RegionStoreClient.RegionStoreClientBuilder getRegionStoreClientBuilder() {
checkIsClosed();
if (this.clientBuilder != null) {
return this.clientBuilder;
}
// lazily create the clientBuilder for the current TiSession
synchronized (this) {
if (this.clientBuilder == null) {
this.clientBuilder =
new RegionStoreClient.RegionStoreClientBuilder(
this.conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
}
}
return this.clientBuilder;
}
public ImporterStoreClient.ImporterStoreClientBuilder getImporterRegionStoreClientBuilder() {
checkIsClosed();
ImporterStoreClient.ImporterStoreClientBuilder res = importerClientBuilder;
RegionStoreClient.RegionStoreClientBuilder res = clientBuilder;
if (res == null) {
synchronized (this) {
if (importerClientBuilder == null) {
if (conf.isTxnKVMode()) {
importerClientBuilder =
new ImporterStoreClient.ImporterStoreClientBuilder<
ImportSstpb.WriteRequest, ImportSstpb.WriteRequest>(
conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
} else {
importerClientBuilder =
new ImporterStoreClient.ImporterStoreClientBuilder<
ImportSstpb.RawWriteRequest, ImportSstpb.RawWriteResponse>(
conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
}
if (clientBuilder == null) {
clientBuilder =
new RegionStoreClient.RegionStoreClientBuilder(
conf, this.channelFactory, this.getRegionManager(), this.getPDClient());
}
res = importerClientBuilder;
res = clientBuilder;
}
}
return res;
@ -358,32 +160,23 @@ public class TiSession implements AutoCloseable {
}
public TiTimestamp getTimestamp() {
checkIsClosed();
return getPDClient()
.getTimestamp(ConcreteBackOffer.newTsoBackOff(getPDClient().getClusterId()));
return getPDClient().getTimestamp(ConcreteBackOffer.newTsoBackOff());
}
public Snapshot createSnapshot() {
checkIsClosed();
return new Snapshot(getTimestamp(), this);
}
public Snapshot createSnapshot(TiTimestamp ts) {
checkIsClosed();
return new Snapshot(ts, this);
}
public PDClient getPDClient() {
checkIsClosed();
PDClient res = client;
if (res == null) {
synchronized (this) {
if (client == null) {
client = PDClient.createRaw(this.getConf(), keyCodec, channelFactory);
client = PDClient.createRaw(this.getConf(), channelFactory);
}
res = client;
}
@ -392,8 +185,6 @@ public class TiSession implements AutoCloseable {
}
public Catalog getCatalog() {
checkIsClosed();
Catalog res = catalog;
if (res == null) {
synchronized (this) {
@ -407,13 +198,16 @@ public class TiSession implements AutoCloseable {
}
public RegionManager getRegionManager() {
checkIsClosed();
RegionManager res = regionManager;
if (res == null) {
synchronized (this) {
if (regionManager == null) {
regionManager = new RegionManager(getConf(), getPDClient(), this.channelFactory);
regionManager =
new RegionManager(
getPDClient(),
this.cacheInvalidateCallback,
this.channelFactory,
this.enableGrpcForward);
}
res = regionManager;
}
@ -422,8 +216,6 @@ public class TiSession implements AutoCloseable {
}
public ExecutorService getThreadPoolForIndexScan() {
checkIsClosed();
ExecutorService res = indexScanThreadPool;
if (res == null) {
synchronized (this) {
@ -443,8 +235,6 @@ public class TiSession implements AutoCloseable {
}
public ExecutorService getThreadPoolForTableScan() {
checkIsClosed();
ExecutorService res = tableScanThreadPool;
if (res == null) {
synchronized (this) {
@ -461,8 +251,6 @@ public class TiSession implements AutoCloseable {
}
public ExecutorService getThreadPoolForBatchPut() {
checkIsClosed();
ExecutorService res = batchPutThreadPool;
if (res == null) {
synchronized (this) {
@ -482,8 +270,6 @@ public class TiSession implements AutoCloseable {
}
public ExecutorService getThreadPoolForBatchGet() {
checkIsClosed();
ExecutorService res = batchGetThreadPool;
if (res == null) {
synchronized (this) {
@ -503,8 +289,6 @@ public class TiSession implements AutoCloseable {
}
public ExecutorService getThreadPoolForBatchDelete() {
checkIsClosed();
ExecutorService res = batchDeleteThreadPool;
if (res == null) {
synchronized (this) {
@ -524,8 +308,6 @@ public class TiSession implements AutoCloseable {
}
public ExecutorService getThreadPoolForBatchScan() {
checkIsClosed();
ExecutorService res = batchScanThreadPool;
if (res == null) {
synchronized (this) {
@ -545,8 +327,6 @@ public class TiSession implements AutoCloseable {
}
public ExecutorService getThreadPoolForDeleteRange() {
checkIsClosed();
ExecutorService res = deleteRangeThreadPool;
if (res == null) {
synchronized (this) {
@ -567,67 +347,49 @@ public class TiSession implements AutoCloseable {
@VisibleForTesting
public ChannelFactory getChannelFactory() {
checkIsClosed();
return channelFactory;
}
/**
* SwitchTiKVModeClient is used for SST Ingest.
*
* @return a SwitchTiKVModeClient
*/
public SwitchTiKVModeClient getSwitchTiKVModeClient() {
checkIsClosed();
public CollectorRegistry getCollectorRegistry() {
return collectorRegistry;
}
SwitchTiKVModeClient res = switchTiKVModeClient;
if (res == null) {
synchronized (this) {
if (switchTiKVModeClient == null) {
switchTiKVModeClient =
new SwitchTiKVModeClient(getPDClient(), getImporterRegionStoreClientBuilder());
}
res = switchTiKVModeClient;
}
}
return res;
/**
* This is used for setting call back function to invalidate cache information
*
* @param callBackFunc callback function
*/
public void injectCallBackFunc(Function<CacheInvalidateEvent, Void> callBackFunc) {
this.cacheInvalidateCallback = callBackFunc;
}
/**
* split region and scatter
*
* @param splitKeys
* @param splitRegionBackoffMS
* @param scatterRegionBackoffMS
* @param scatterWaitMS
*/
public void splitRegionAndScatter(
List<byte[]> splitKeys,
int splitRegionBackoffMS,
int scatterRegionBackoffMS,
int scatterWaitMS) {
checkIsClosed();
logger.info(String.format("split key's size is %d", splitKeys.size()));
long startMS = System.currentTimeMillis();
// split region
List<Metapb.Region> newRegions =
List<TiRegion> newRegions =
splitRegion(
splitKeys
.stream()
.map(k -> Key.toRawKey(k).toByteString())
.map(k -> Key.toRawKey(k).next().toByteString())
.collect(Collectors.toList()),
ConcreteBackOffer.newCustomBackOff(splitRegionBackoffMS, getPDClient().getClusterId()));
ConcreteBackOffer.newCustomBackOff(splitRegionBackoffMS));
// scatter region
for (Metapb.Region newRegion : newRegions) {
for (TiRegion newRegion : newRegions) {
try {
getPDClient()
.scatterRegion(
newRegion,
ConcreteBackOffer.newCustomBackOff(
scatterRegionBackoffMS, getPDClient().getClusterId()));
.scatterRegion(newRegion, ConcreteBackOffer.newCustomBackOff(scatterRegionBackoffMS));
} catch (Exception e) {
logger.warn(String.format("failed to scatter region: %d", newRegion.getId()), e);
}
@ -637,16 +399,14 @@ public class TiSession implements AutoCloseable {
if (scatterWaitMS > 0) {
logger.info("start to wait scatter region finish");
long scatterRegionStartMS = System.currentTimeMillis();
for (Metapb.Region newRegion : newRegions) {
for (TiRegion newRegion : newRegions) {
long remainMS = (scatterRegionStartMS + scatterWaitMS) - System.currentTimeMillis();
if (remainMS <= 0) {
logger.warn("wait scatter region timeout");
return;
}
getPDClient()
.waitScatterRegionFinish(
newRegion,
ConcreteBackOffer.newCustomBackOff((int) remainMS, getPDClient().getClusterId()));
.waitScatterRegionFinish(newRegion, ConcreteBackOffer.newCustomBackOff((int) remainMS));
}
} else {
logger.info("skip to wait scatter region finish");
@ -656,30 +416,11 @@ public class TiSession implements AutoCloseable {
logger.info("splitRegionAndScatter cost {} seconds", (endMS - startMS) / 1000);
}
/**
* split region and scatter
*
* @param splitKeys
*/
public void splitRegionAndScatter(List<byte[]> splitKeys) {
checkIsClosed();
int splitRegionBackoffMS = BackOffer.SPLIT_REGION_BACKOFF;
int scatterRegionBackoffMS = BackOffer.SCATTER_REGION_BACKOFF;
int scatterWaitMS = conf.getScatterWaitSeconds() * 1000;
splitRegionAndScatter(splitKeys, splitRegionBackoffMS, scatterRegionBackoffMS, scatterWaitMS);
}
private List<Metapb.Region> splitRegion(List<ByteString> splitKeys, BackOffer backOffer) {
return splitRegion(splitKeys, backOffer, 1);
}
private List<Metapb.Region> splitRegion(
List<ByteString> splitKeys, BackOffer backOffer, int depth) {
List<Metapb.Region> regions = new ArrayList<>();
private List<TiRegion> splitRegion(List<ByteString> splitKeys, BackOffer backOffer) {
List<TiRegion> regions = new ArrayList<>();
Map<TiRegion, List<ByteString>> groupKeys =
groupKeysByRegion(getRegionManager(), splitKeys, backOffer);
groupKeysByRegion(regionManager, splitKeys, backOffer);
for (Map.Entry<TiRegion, List<ByteString>> entry : groupKeys.entrySet()) {
Pair<TiRegion, TiStore> pair =
@ -698,25 +439,15 @@ public class TiSession implements AutoCloseable {
"split key equal to region start key or end key. Region splitting is not needed.");
} else {
logger.info("start to split region id={}, split size={}", region.getId(), splits.size());
List<Metapb.Region> newRegions;
List<TiRegion> newRegions;
try {
newRegions = getRegionStoreClientBuilder().build(region, store).splitRegion(splits);
// invalidate old region
getRegionManager().invalidateRegion(region);
} catch (final TiKVException e) {
// retry
logger.warn("ReSplitting ranges for splitRegion", e);
getRegionManager().invalidateRegion(region);
clientBuilder.getRegionManager().invalidateRegion(region);
backOffer.doBackOff(BackOffFunction.BackOffFuncType.BoRegionMiss, e);
if (depth >= MAX_SPLIT_REGION_STACK_DEPTH) {
logger.warn(
String.format(
"Skip split region because MAX_SPLIT_REGION_STACK_DEPTH(%d) reached!",
MAX_SPLIT_REGION_STACK_DEPTH));
newRegions = new ArrayList<>();
} else {
newRegions = splitRegion(splits, backOffer, depth + 1);
}
newRegions = splitRegion(splits, backOffer);
}
logger.info("region id={}, new region size={}", region.getId(), newRegions.size());
regions.addAll(newRegions);
@ -727,119 +458,51 @@ public class TiSession implements AutoCloseable {
return regions;
}
private void checkIsClosed() {
if (isClosed) {
throw new RuntimeException("this TiSession is closed!");
}
}
public synchronized void closeAwaitTermination(long timeoutMS) throws Exception {
shutdown(false);
long startMS = System.currentTimeMillis();
while (true) {
if (isTerminatedExecutorServices()) {
cleanAfterTerminated();
return;
}
if (System.currentTimeMillis() - startMS > timeoutMS) {
shutdown(true);
return;
}
Thread.sleep(500);
}
}
@Override
public synchronized void close() throws Exception {
shutdown(true);
}
private synchronized void shutdown(boolean now) throws Exception {
if (!isClosed) {
isClosed = true;
synchronized (sessionCachedMap) {
sessionCachedMap.remove(conf.getPdAddrsString());
}
if (metricsServer != null) {
metricsServer.close();
}
if (circuitBreaker != null) {
circuitBreaker.close();
}
if (isClosed) {
logger.warn("this TiSession is already closed!");
return;
}
if (now) {
shutdownNowExecutorServices();
cleanAfterTerminated();
} else {
shutdownExecutorServices();
if (server != null) {
server.stop();
logger.info("Metrics server on " + server.getPort() + " is stopped");
}
}
private synchronized void cleanAfterTerminated() throws InterruptedException {
isClosed = true;
synchronized (sessionCachedMap) {
sessionCachedMap.remove(conf.getPdAddrsString());
}
if (regionManager != null) {
regionManager.close();
}
if (client != null) {
client.close();
}
if (catalog != null) {
catalog.close();
}
if (switchTiKVModeClient != null) {
switchTiKVModeClient.stopKeepTiKVToImportMode();
}
}
private List<ExecutorService> getExecutorServices() {
List<ExecutorService> executorServiceList = new ArrayList<>();
if (tableScanThreadPool != null) {
executorServiceList.add(tableScanThreadPool);
tableScanThreadPool.shutdownNow();
}
if (indexScanThreadPool != null) {
executorServiceList.add(indexScanThreadPool);
indexScanThreadPool.shutdownNow();
}
if (batchGetThreadPool != null) {
executorServiceList.add(batchGetThreadPool);
batchGetThreadPool.shutdownNow();
}
if (batchPutThreadPool != null) {
executorServiceList.add(batchPutThreadPool);
batchPutThreadPool.shutdownNow();
}
if (batchDeleteThreadPool != null) {
executorServiceList.add(batchDeleteThreadPool);
batchDeleteThreadPool.shutdownNow();
}
if (batchScanThreadPool != null) {
executorServiceList.add(batchScanThreadPool);
batchScanThreadPool.shutdownNow();
}
if (deleteRangeThreadPool != null) {
executorServiceList.add(deleteRangeThreadPool);
deleteRangeThreadPool.shutdownNow();
}
return executorServiceList;
}
private void shutdownExecutorServices() {
for (ExecutorService executorService : getExecutorServices()) {
executorService.shutdown();
if (client != null) {
getPDClient().close();
}
}
private void shutdownNowExecutorServices() {
for (ExecutorService executorService : getExecutorServices()) {
executorService.shutdownNow();
if (catalog != null) {
getCatalog().close();
}
}
private boolean isTerminatedExecutorServices() {
for (ExecutorService executorService : getExecutorServices()) {
if (!executorService.isTerminated()) {
return false;
}
}
return true;
}
}

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2021 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common;

View File

@ -1,15 +1,15 @@
/*
* Copyright 2021 TiKV Project Authors.
*
* Copyright 2020 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
@ -25,6 +25,4 @@ public class Version {
public static final String RESOLVE_LOCK_V4 = "4.0.0";
public static final String BATCH_WRITE = "3.0.14";
public static final String API_V2 = "6.1.0";
}

View File

@ -1,36 +0,0 @@
/*
* Copyright 2022 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.apiversion;
import com.google.protobuf.ByteString;
import org.tikv.common.codec.Codec.BytesCodec;
import org.tikv.common.codec.CodecDataInput;
import org.tikv.common.codec.CodecDataOutput;
// TODO(iosmanthus): use ByteString.wrap to avoid once more copying.
public class CodecUtils {
public static ByteString encode(ByteString key) {
CodecDataOutput cdo = new CodecDataOutput();
BytesCodec.writeBytes(cdo, key.toByteArray());
return cdo.toByteString();
}
public static ByteString decode(ByteString key) {
return ByteString.copyFrom(BytesCodec.readBytes(new CodecDataInput(key)));
}
}

View File

@ -1,72 +0,0 @@
/*
* Copyright 2022 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.apiversion;
import com.google.protobuf.ByteString;
import java.util.List;
import java.util.stream.Collectors;
import org.tikv.common.util.Pair;
import org.tikv.kvproto.Kvrpcpb.KvPair;
import org.tikv.kvproto.Kvrpcpb.Mutation;
import org.tikv.kvproto.Metapb;
import org.tikv.kvproto.Pdpb;
public interface RequestKeyCodec {
ByteString encodeKey(ByteString key);
default List<ByteString> encodeKeys(List<ByteString> keys) {
return keys.stream().map(this::encodeKey).collect(Collectors.toList());
}
default List<Mutation> encodeMutations(List<Mutation> mutations) {
return mutations
.stream()
.map(mut -> Mutation.newBuilder().mergeFrom(mut).setKey(encodeKey(mut.getKey())).build())
.collect(Collectors.toList());
}
ByteString decodeKey(ByteString key);
default KvPair decodeKvPair(KvPair pair) {
return KvPair.newBuilder().mergeFrom(pair).setKey(decodeKey(pair.getKey())).build();
}
default List<KvPair> decodeKvPairs(List<KvPair> pairs) {
return pairs.stream().map(this::decodeKvPair).collect(Collectors.toList());
}
Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end);
ByteString encodePdQuery(ByteString key);
Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end);
Metapb.Region decodeRegion(Metapb.Region region);
default List<Pdpb.Region> decodePdRegions(List<Pdpb.Region> regions) {
return regions
.stream()
.map(
r ->
Pdpb.Region.newBuilder()
.mergeFrom(r)
.setRegion(this.decodeRegion(r.getRegion()))
.build())
.collect(Collectors.toList());
}
}

View File

@ -1,83 +0,0 @@
/*
* Copyright 2022 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.apiversion;
import com.google.protobuf.ByteString;
import java.util.List;
import org.tikv.common.util.Pair;
import org.tikv.kvproto.Kvrpcpb.KvPair;
import org.tikv.kvproto.Kvrpcpb.Mutation;
import org.tikv.kvproto.Metapb.Region;
import org.tikv.kvproto.Pdpb;
public class RequestKeyV1Codec implements RequestKeyCodec {
@Override
public ByteString encodeKey(ByteString key) {
return key;
}
@Override
public List<ByteString> encodeKeys(List<ByteString> keys) {
return keys;
}
@Override
public List<Mutation> encodeMutations(List<Mutation> mutations) {
return mutations;
}
@Override
public ByteString decodeKey(ByteString key) {
return key;
}
@Override
public KvPair decodeKvPair(KvPair pair) {
return pair;
}
@Override
public List<KvPair> decodeKvPairs(List<KvPair> pairs) {
return pairs;
}
@Override
public Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end) {
return Pair.create(start, end);
}
@Override
public ByteString encodePdQuery(ByteString key) {
return key;
}
@Override
public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
return Pair.create(start, end);
}
@Override
public Region decodeRegion(Region region) {
return region;
}
@Override
public List<Pdpb.Region> decodePdRegions(List<Pdpb.Region> regions) {
return regions;
}
}

View File

@ -1,22 +0,0 @@
/*
* Copyright 2022 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.apiversion;
public class RequestKeyV1RawCodec extends RequestKeyV1Codec implements RequestKeyCodec {
public RequestKeyV1RawCodec() {}
}

View File

@ -1,61 +0,0 @@
/*
* Copyright 2022 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.apiversion;
import com.google.protobuf.ByteString;
import org.tikv.common.util.Pair;
import org.tikv.kvproto.Metapb;
public class RequestKeyV1TxnCodec extends RequestKeyV1Codec implements RequestKeyCodec {
public RequestKeyV1TxnCodec() {}
@Override
public ByteString encodePdQuery(ByteString key) {
return CodecUtils.encode(key);
}
@Override
public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
if (!start.isEmpty()) {
start = CodecUtils.encode(start);
}
if (!end.isEmpty()) {
end = CodecUtils.encode(end);
}
return Pair.create(start, end);
}
@Override
public Metapb.Region decodeRegion(Metapb.Region region) {
Metapb.Region.Builder builder = Metapb.Region.newBuilder().mergeFrom(region);
ByteString start = region.getStartKey();
ByteString end = region.getEndKey();
if (!start.isEmpty()) {
start = CodecUtils.decode(start);
}
if (!end.isEmpty()) {
end = CodecUtils.decode(end);
}
return builder.setStartKey(start).setEndKey(end).build();
}
}

View File

@ -1,101 +0,0 @@
/*
* Copyright 2022 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.apiversion;
import com.google.protobuf.ByteString;
import org.tikv.common.util.Pair;
import org.tikv.kvproto.Metapb;
import org.tikv.kvproto.Metapb.Region;
public class RequestKeyV2Codec implements RequestKeyCodec {
protected static final ByteString RAW_DEFAULT_PREFIX =
ByteString.copyFrom(new byte[] {'r', 0, 0, 0});
protected static final ByteString RAW_DEFAULT_END =
ByteString.copyFrom(new byte[] {'r', 0, 0, 1});
protected static final ByteString TXN_DEFAULT_PREFIX =
ByteString.copyFrom(new byte[] {'x', 0, 0, 0});
protected static final ByteString TXN_DEFAULT_END =
ByteString.copyFrom(new byte[] {'x', 0, 0, 1});
protected ByteString keyPrefix;
protected ByteString infiniteEndKey;
@Override
public ByteString encodeKey(ByteString key) {
return keyPrefix.concat(key);
}
@Override
public ByteString decodeKey(ByteString key) {
if (key.isEmpty()) {
return key;
}
if (!key.startsWith(keyPrefix)) {
throw new IllegalArgumentException("key corrupted, wrong prefix");
}
return key.substring(keyPrefix.size());
}
@Override
public Pair<ByteString, ByteString> encodeRange(ByteString start, ByteString end) {
start = encodeKey(start);
end = end.isEmpty() ? infiniteEndKey : encodeKey(end);
return Pair.create(start, end);
}
@Override
public ByteString encodePdQuery(ByteString key) {
return CodecUtils.encode(encodeKey(key));
}
@Override
public Pair<ByteString, ByteString> encodePdQueryRange(ByteString start, ByteString end) {
Pair<ByteString, ByteString> range = encodeRange(start, end);
return Pair.create(CodecUtils.encode(range.first), CodecUtils.encode(range.second));
}
@Override
public Region decodeRegion(Region region) {
Metapb.Region.Builder builder = Metapb.Region.newBuilder().mergeFrom(region);
ByteString start = region.getStartKey();
ByteString end = region.getEndKey();
if (!start.isEmpty()) {
start = CodecUtils.decode(start);
}
if (!end.isEmpty()) {
end = CodecUtils.decode(end);
}
if (ByteString.unsignedLexicographicalComparator().compare(start, infiniteEndKey) >= 0
|| (!end.isEmpty()
&& ByteString.unsignedLexicographicalComparator().compare(end, keyPrefix) <= 0)) {
throw new IllegalArgumentException("region out of keyspace" + region.toString());
}
start = start.startsWith(keyPrefix) ? start.substring(keyPrefix.size()) : ByteString.EMPTY;
end = end.startsWith(keyPrefix) ? end.substring(keyPrefix.size()) : ByteString.EMPTY;
return builder.setStartKey(start).setEndKey(end).build();
}
}

View File

@ -1,27 +0,0 @@
/*
* Copyright 2022 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.apiversion;
public class RequestKeyV2RawCodec extends RequestKeyV2Codec {
public RequestKeyV2RawCodec() {
super();
this.keyPrefix = RAW_DEFAULT_PREFIX;
this.infiniteEndKey = RAW_DEFAULT_END;
}
}

View File

@ -1,27 +0,0 @@
/*
* Copyright 2022 TiKV Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.apiversion;
public class RequestKeyV2TxnCodec extends RequestKeyV2Codec {
public RequestKeyV2TxnCodec() {
super();
this.keyPrefix = TXN_DEFAULT_PREFIX;
this.infiniteEndKey = TXN_DEFAULT_END;
}
}

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.catalog;

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.catalog;
@ -40,7 +38,6 @@ import org.tikv.common.util.Pair;
public class CatalogTransaction {
protected static final Logger logger = LoggerFactory.getLogger(CatalogTransaction.class);
private static final ObjectMapper mapper = new ObjectMapper();
private final Snapshot snapshot;
CatalogTransaction(Snapshot snapshot) {
@ -52,6 +49,7 @@ public class CatalogTransaction {
Objects.requireNonNull(cls, "cls is null");
logger.debug(String.format("Parse Json %s : %s", cls.getSimpleName(), json.toStringUtf8()));
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(json.toStringUtf8(), cls);
} catch (JsonParseException | JsonMappingException e) {

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.codec;

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.codec;

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2020 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.codec;

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.codec;

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2020 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.codec;

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.codec;

View File

@ -1,18 +1,16 @@
/*
* Copyright 2021 TiKV Project Authors.
* Copyright 2020 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.tikv.common.codec;

Some files were not shown because too many files have changed in this diff Show More