tikv
/
client-java
mirror of https://github.com/tikv/client-java.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[close #425] make test stable (#429)

This commit is contained in:
Liangliang Gu 2021-12-23 11:36:13 +08:00 committed by GitHub
parent 80a262390a
commit 2a04b91be7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 121 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
.ci/integration_test.groovy
View File

@ -1,17 +1,8 @@
def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) {
def TIDB_BRANCH = "master"
def TIKV_BRANCH = "master"
def PD_BRANCH = "master"
// parse tidb branch
def m1 = ghprbCommentBody =~ /tidb\s*=\s*([^\s\\]+)(\s|\\|$)/
if (m1) {
TIDB_BRANCH = "${m1[0][1]}"
}
m1 = null
println "TIDB_BRANCH=${TIDB_BRANCH}"
// parse pd branch
def m2 = ghprbCommentBody =~ /pd\s*=\s*([^\s\\]+)(\s|\\|$)/
if (m2) {
@ -48,9 +39,6 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
}
dir("/home/jenkins/agent/git/client-java/_run") {
// tidb
def tidb_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tidb/${TIDB_BRANCH}/sha1").trim()
sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz | tar xz"
// tikv
def tikv_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tikv/${TIKV_BRANCH}/sha1").trim()
sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz | tar xz"
@ -59,20 +47,33 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz | tar xz"
sh """
killall -9 tidb-server || true
killall -9 tikv-server || true
killall -9 pd-server || true
killall -9 java || true
sleep 10
bin/pd-server --name=pd --data-dir=pd --config=../config/pd.toml &>pd.log &
"""
sh """
echo "start TiKV for RawKV test"
bin/pd-server --name=pd_rawkv --data-dir=pd_rawkv --client-urls="http://0.0.0.0:2379" --advertise-client-urls="http://127.0.0.1:2379" --peer-urls="http://0.0.0.0:2380" --advertise-peer-urls="http://127.0.0.1:2380" --config=../config/pd.toml &>pd_rawkv.log &
sleep 10
bin/tikv-server --pd=127.0.0.1:2379 -s tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 --config=../config/tikv.toml &>tikv.log &
bin/tikv-server --pd 127.0.0.1:2379 --data-dir tikv_rawkv --addr 0.0.0.0:20160 --advertise-addr 127.0.0.1:20160 --status-addr 0.0.0.0:20180 --config ../config/tikv_rawkv.toml &>tikv_rawkv.log &
sleep 10
ps aux | grep '-server' || true
curl -s 127.0.0.1:2379/pd/api/v1/status || true
bin/tidb-server --store=tikv --path="127.0.0.1:2379" --config=../config/tidb.toml &>tidb.log &
sleep 60
"""
sh """
echo "start TiKV for TxnKV test"
bin/pd-server --name=pd_txnkv --data-dir=pd_txnkv --client-urls="http://0.0.0.0:3379" --advertise-client-urls="http://127.0.0.1:3379" --peer-urls="http://0.0.0.0:3380" --advertise-peer-urls="http://127.0.0.1:3380" --config=../config/pd.toml &>pd_txnkv.log &
sleep 10
bin/tikv-server --pd 127.0.0.1:3379 --data-dir tikv_txnkv --addr 0.0.0.0:21160 --advertise-addr 127.0.0.1:21160 --status-addr 0.0.0.0:21180 --config ../config/tikv_txnkv.toml &>tikv_txnkv.log &
sleep 10
ps aux | grep '-server' || true
curl -s 127.0.0.1:3379/pd/api/v1/status || true
"""
sh "sleep 30"
}
}
@ -87,9 +88,10 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
ps aux | grep '-server' || true
curl -s 127.0.0.1:2379/pd/api/v1/status || true
"""
sh "cat _run/pd.log"
sh "cat _run/tikv.log"
sh "cat _run/tidb.log"
sh "cat _run/pd_rawkv.log"
sh "cat _run/tikv_rawkv.log"
sh "cat _run/pd_txnkv.log"
sh "cat _run/tikv_txnkv.log"
throw err
}
}
@ -102,14 +104,14 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
stage('Summary') {
def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" +
"${ghprbPullLink}" + "\n" +
"${ghprbPullDescription}" + "\n" +
"Integration Common Test Result: `${currentBuild.result}`" + "\n" +
"Elapsed Time: `${duration} mins` " + "\n" +
"${env.RUN_DISPLAY_URL}"
"${ghprbPullLink}" + "\n" +
"${ghprbPullDescription}" + "\n" +
"Integration Common Test Result: `${currentBuild.result}`" + "\n" +
"Elapsed Time: `${duration} mins` " + "\n" +
"${env.RUN_DISPLAY_URL}"
print msg
}
}
return this
return this

2
.ci/test.sh
View File

@ -3,4 +3,6 @@
set -x
set -euo pipefail
export RAWKV_PD_ADDRESSES=127.0.0.1:2379
export TXNKV_PD_ADDRESSES=127.0.0.1:3379
mvn clean test

1
config/tidb.toml
View File

@ -1 +0,0 @@
# TiDB Configuration.

8
config/tikv_rawkv.toml Normal file
View File

@ -0,0 +1,8 @@
# TiKV Configuration.
[raftstore]
# set store capacity, if no set, use disk capacity.
capacity = "8G"
[storage]
enable-ttl = true

2
config/tikv.toml → config/tikv_txnkv.toml
View File

@ -2,4 +2,4 @@
[raftstore]
# set store capacity, if no set, use disk capacity.
capacity = "8G"
capacity = "8G"

4
src/main/java/org/tikv/common/TiConfiguration.java
View File

@ -651,6 +651,10 @@ public class TiConfiguration implements Serializable {
return this.enableGrpcForward;
}
public void setEnableGrpcForward(boolean enableGrpcForward) {
this.enableGrpcForward = enableGrpcForward;
}
public long getGrpcHealthCheckTimeout() {
return this.grpcHealthCheckTimeout;
}

17
src/test/java/org/tikv/BaseRawKVTest.java Normal file
View File

@ -0,0 +1,17 @@
package org.tikv;
import org.tikv.common.TiConfiguration;
import org.tikv.util.TestUtils;
public class BaseRawKVTest {
protected TiConfiguration createTiConfiguration() {
String pdAddrsStr = TestUtils.getEnv("RAWKV_PD_ADDRESSES");
TiConfiguration conf =
pdAddrsStr == null
? TiConfiguration.createRawDefault()
: TiConfiguration.createRawDefault(pdAddrsStr);
conf.setEnableGrpcForward(false);
return conf;
}
}

17
src/test/java/org/tikv/BaseTxnKVTest.java Normal file
View File

@ -0,0 +1,17 @@
package org.tikv;
import org.tikv.common.TiConfiguration;
import org.tikv.util.TestUtils;
public class BaseTxnKVTest {
protected TiConfiguration createTiConfiguration() {
String pdAddrsStr = TestUtils.getEnv("TXNKV_PD_ADDRESSES");
TiConfiguration conf =
pdAddrsStr == null
? TiConfiguration.createDefault()
: TiConfiguration.createDefault(pdAddrsStr);
conf.setEnableGrpcForward(false);
return conf;
}
}

7
src/test/java/org/tikv/common/TiConfigurationTest.java
View File

@ -18,18 +18,19 @@ package org.tikv.common;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
import org.tikv.BaseRawKVTest;
public class TiConfigurationTest {
public class TiConfigurationTest extends BaseRawKVTest {
@Test
public void configFileTest() {
TiConfiguration conf = TiConfiguration.createRawDefault();
TiConfiguration conf = createTiConfiguration();
assertEquals("configFileTest", conf.getDBPrefix());
}
@Test
public void testGrpcIdleTimeoutValue() {
TiConfiguration conf = TiConfiguration.createDefault();
TiConfiguration conf = createTiConfiguration();
// default value
assertEquals(TiConfiguration.getInt(ConfigUtils.TIKV_GRPC_IDLE_TIMEOUT), conf.getIdleTimeout());
// new value

7
src/test/java/org/tikv/common/TiSessionTest.java
View File

@ -13,10 +13,11 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.After;
import org.junit.Test;
import org.tikv.BaseRawKVTest;
import org.tikv.common.region.TiRegion;
import org.tikv.raw.RawKVClient;
public class TiSessionTest {
public class TiSessionTest extends BaseRawKVTest {
private TiSession session;
@After
@ -37,7 +38,7 @@ public class TiSessionTest {
}
private void doCloseWithRunningTaskTest(boolean now, long timeoutMS) throws Exception {
TiConfiguration conf = TiConfiguration.createRawDefault();
TiConfiguration conf = createTiConfiguration();
session = TiSession.create(conf);
ExecutorService executorService = session.getThreadPoolForBatchGet();
@ -82,7 +83,7 @@ public class TiSessionTest {
}
private void doCloseTest(boolean now, long timeoutMS) throws Exception {
TiConfiguration conf = TiConfiguration.createRawDefault();
TiConfiguration conf = createTiConfiguration();
session = TiSession.create(conf);
RawKVClient client = session.createRawClient();

11
src/test/java/org/tikv/raw/MetricsTest.java
View File

@ -7,10 +7,11 @@ import java.util.ArrayList;
import java.util.List;
import org.junit.After;
import org.junit.Test;
import org.tikv.BaseRawKVTest;
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
public class MetricsTest {
public class MetricsTest extends BaseRawKVTest {
private List<TiSession> sessionList = new ArrayList<>();
@After
@ -24,7 +25,7 @@ public class MetricsTest {
@Test
public void oneTiSession() throws Exception {
TiConfiguration conf = TiConfiguration.createRawDefault();
TiConfiguration conf = createTiConfiguration();
conf.setMetricsEnable(true);
TiSession session = TiSession.create(conf);
sessionList.add(session);
@ -38,7 +39,7 @@ public class MetricsTest {
@Test
public void twoTiSession() throws Exception {
TiConfiguration conf = TiConfiguration.createRawDefault();
TiConfiguration conf = createTiConfiguration();
conf.setMetricsEnable(true);
TiSession session1 = TiSession.create(conf);
@ -63,13 +64,13 @@ public class MetricsTest {
@Test
public void twoTiSessionWithDifferentPort() {
TiConfiguration conf1 = TiConfiguration.createRawDefault();
TiConfiguration conf1 = createTiConfiguration();
conf1.setMetricsEnable(true);
conf1.setMetricsPort(12345);
TiSession session1 = TiSession.create(conf1);
sessionList.add(session1);
TiConfiguration conf2 = TiConfiguration.createRawDefault();
TiConfiguration conf2 = createTiConfiguration();
conf2.setMetricsEnable(true);
conf2.setMetricsPort(54321);
try {

22
src/test/java/org/tikv/raw/RawKVClientTest.java
View File

@ -10,10 +10,10 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tikv.BaseRawKVTest;
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
import org.tikv.common.codec.KeyUtils;
@ -27,7 +27,7 @@ import org.tikv.common.util.FastByteComparisons;
import org.tikv.common.util.ScanOption;
import org.tikv.kvproto.Kvrpcpb;
public class RawKVClientTest {
public class RawKVClientTest extends BaseRawKVTest {
private static final String RAW_PREFIX = "raw_\u0001_";
private static final int KEY_POOL_SIZE = 1000000;
private static final int TEST_CASES = 10000;
@ -41,7 +41,6 @@ public class RawKVClientTest {
private static final List<ByteString> values;
private static final int limit;
private TreeMap<ByteString, ByteString> data;
private boolean initialized;
private final Random r = new Random(1234);
private static final ByteStringComparator bsc = new ByteStringComparator();
private static final ExecutorService executors = Executors.newFixedThreadPool(WORKER_CNT);
@ -77,14 +76,12 @@ public class RawKVClientTest {
@Before
public void setup() throws IOException {
try {
TiConfiguration conf = TiConfiguration.createRawDefault();
TiConfiguration conf = createTiConfiguration();
session = TiSession.create(conf);
initialized = false;
if (client == null) {
client = session.createRawClient();
}
data = new TreeMap<>(bsc);
initialized = true;
} catch (Exception e) {
logger.warn(
"Cannot initialize raw client, please check whether TiKV is running. Test skipped.", e);
@ -98,10 +95,8 @@ public class RawKVClientTest {
}
}
// tikv-4.0 does not support atomic api
@Ignore
@Test
public void atomicAPITest() {
if (!initialized) return;
long ttl = 10;
ByteString key = ByteString.copyFromUtf8("key_atomic");
ByteString value = ByteString.copyFromUtf8("value");
@ -120,10 +115,8 @@ public class RawKVClientTest {
assertTrue(res3.isEmpty());
}
// tikv-4.0 doest not support ttl
@Ignore
@Test
public void getKeyTTLTest() {
if (!initialized) return;
long ttl = 10;
ByteString key = ByteString.copyFromUtf8("key_ttl");
ByteString value = ByteString.copyFromUtf8("value");
@ -223,7 +216,6 @@ public class RawKVClientTest {
@Test
public void batchPutTest() {
if (!initialized) return;
ExecutorService executors = Executors.newFixedThreadPool(200);
ExecutorCompletionService<Object> completionService =
new ExecutorCompletionService<>(executors);
@ -312,13 +304,11 @@ public class RawKVClientTest {
@Test
public void deleteRangeTest() {
if (!initialized) return;
client.deleteRange(ByteString.EMPTY, ByteString.EMPTY);
}
@Test
public void simpleTest() {
if (!initialized) return;
ByteString key = rawKey("key");
ByteString key0 = rawKey("key0");
ByteString key1 = rawKey("key1");
@ -372,14 +362,12 @@ public class RawKVClientTest {
@Test
public void validate() {
if (!initialized) return;
baseTest(100, 100, 100, 100, false, false, false, false, false);
baseTest(100, 100, 100, 100, false, true, true, true, true);
}
/** Example of benchmarking base test */
public void benchmark() {
if (!initialized) return;
baseTest(TEST_CASES, TEST_CASES, 200, 5000, true, false, false, false, false);
baseTest(TEST_CASES, TEST_CASES, 200, 5000, true, true, true, true, true);
}

5
src/test/java/org/tikv/raw/SmartRawKVClientTest.java
View File

@ -6,11 +6,12 @@ import com.google.protobuf.ByteString;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.tikv.BaseRawKVTest;
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
import org.tikv.common.exception.CircuitBreakerOpenException;
public class SmartRawKVClientTest {
public class SmartRawKVClientTest extends BaseRawKVTest {
private boolean enable = true;
private int windowInSeconds = 2;
private int errorThresholdPercentage = 100;
@ -25,7 +26,7 @@ public class SmartRawKVClientTest {
@Before
public void setup() {
TiConfiguration conf = TiConfiguration.createRawDefault();
TiConfiguration conf = createTiConfiguration();
conf.setCircuitBreakEnable(enable);
conf.setCircuitBreakAvailabilityWindowInSeconds(windowInSeconds);
conf.setCircuitBreakAvailabilityErrorThresholdPercentage(errorThresholdPercentage);

4
src/test/java/org/tikv/txn/ReplicaReadTest.java
View File

@ -37,7 +37,7 @@ public class ReplicaReadTest extends TXNTest {
@Test
public void replicaSelectorTest() {
TiConfiguration conf = TiConfiguration.createDefault();
TiConfiguration conf = createTiConfiguration();
conf.setReplicaSelector(
new ReplicaSelector() {
@ -58,7 +58,7 @@ public class ReplicaReadTest extends TXNTest {
}
private void doTest(TiConfiguration.ReplicaRead replicaRead) {
TiConfiguration conf = TiConfiguration.createDefault();
TiConfiguration conf = createTiConfiguration();
conf.setReplicaRead(replicaRead);
session = TiSession.create(conf);

5
src/test/java/org/tikv/txn/TXNTest.java
View File

@ -11,6 +11,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import org.junit.After;
import org.junit.Before;
import org.tikv.BaseTxnKVTest;
import org.tikv.common.TiConfiguration;
import org.tikv.common.TiSession;
import org.tikv.common.exception.RegionException;
@ -21,14 +22,14 @@ import org.tikv.common.util.BackOffer;
import org.tikv.common.util.ConcreteBackOffer;
import org.tikv.kvproto.Kvrpcpb;
public class TXNTest {
public class TXNTest extends BaseTxnKVTest {
static final int DEFAULT_TTL = 10;
private TiSession session;
RegionStoreClient.RegionStoreClientBuilder builder;
@Before
public void setUp() {
TiConfiguration conf = TiConfiguration.createDefault();
TiConfiguration conf = createTiConfiguration();
try {
session = TiSession.create(conf);
this.builder = session.getRegionStoreClientBuilder();

17
src/test/java/org/tikv/util/TestUtils.java Normal file
View File

@ -0,0 +1,17 @@
package org.tikv.util;
public class TestUtils {
public static String getEnv(String key) {
String tmp = System.getenv(key);
if (tmp != null && !tmp.equals("")) {
return tmp;
}
tmp = System.getProperty(key);
if (tmp != null && !tmp.equals("")) {
return tmp;
}
return null;
}
}