diff --git a/README.md b/README.md index 1ac3594820..f911a7fd20 100644 --- a/README.md +++ b/README.md @@ -9,28 +9,59 @@ It is supposed to: ## How to build +### Gradle + +Alternatively, you can build `tikv-client-java` with gradle. + +The following command will build the project. + +``` +gradle init +gradle clean build -x test +``` + +To make a jar with dependencies + +``` +gradle clean fatJar -x test +``` + +The jar can be found in `./build/libs/` + +### Maven + The alternative way to build a usable jar for testing will be + ``` mvn clean install -Dmaven.test.skip=true ``` The following command can install dependencies for you. + ``` mvn package ``` +The jar can be found in `./target/` + +### Bazel + Alternatively, you can use `bazel` for much faster build. When you try this approach, you should run `git submodule update --init --recursive` before you build project. Making a uber jar: + ``` make uber_jar ``` + run Main class: + ``` make run ``` run test cases: + ``` make test ``` @@ -39,8 +70,9 @@ this project is designed to hook with `pd` and `tikv` which you can find in `Pin When you work with this project, you have to communicate with `pd` and `tikv`. Please run TiKV and PD in advance. -## Raw TiKV-Client in Java -Java Implementation of Raw TiKV-Client +## Component: Raw Ti-Client in Java + +Java Implementation of Raw TiKV-Client to support RawKVClient commands. Demo is avaliable in [KVRawClientTest](https://github.com/birdstorm/KVRawClientTest/) @@ -49,79 +81,94 @@ Demo is avaliable in [KVRawClientTest](https://github.com/birdstorm/KVRawClientT mvn clean install -Dmaven.test.skip=true ``` -### Use as maven dependency -After building, add following lines into your `pom.xml` +### Add to dependency + +#### Use jar for binary + +Add your jar built with all dependencies into you project's library to use `tikv-client-java` as dependency + +#### Use as maven dependency + +After building, add following lines into your `pom.xml` if you are using Maven + ```xml - org.tikv - tikv-client-java - 2.0-SNAPSHOT + org.tikv + tikv-client-java + 2.0-SNAPSHOT ``` ### Entrance -`com.pingcap.tikv.RawKVClient` +`org.tikv.raw.RawKVClient` + +### Create a RawKVClient + +```java +import org.tikv.common.TiSession; +import org.tikv.raw.RawKVClient; + +public class Main { + public static void main() { + // You MUST create a raw configuration if you are using RawKVClient. + TiConfiguration conf = TiConfiguration.createRawDefault(YOUR_PD_ADDRESSES); + TiSession session = TiSession.create(conf); + RawKVClient = session.createRawKVClient(); + } +} +``` ### API ```java - /** - * create a RawKVClient using specific pd addresses - * - * @param address pd addresses(comma seperated) - */ - static RawKVClient create(String address) +/** + * Put a raw key-value pair to TiKV + * + * @param key raw key + * @param value raw value + */ +void put(ByteString key, ByteString value) ``` ```java - /** - * Put a raw key-value pair to TiKV - * - * @param key raw key - * @param value raw value - */ - void put(ByteString key, ByteString value) +/** + * Get a raw key-value pair from TiKV if key exists + * + * @param key raw key + * @return a ByteString value if key exists, ByteString.EMPTY if key does not exist + */ +ByteString get(ByteString key) ``` ```java - /** - * Get a raw key-value pair from TiKV if key exists - * - * @param key raw key - * @return a ByteString value if key exists, ByteString.EMPTY if key does not exist - */ - ByteString get(ByteString key) +/** + * Scan raw key-value pairs from TiKV in range [startKey, endKey) + * + * @param startKey raw start key, inclusive + * @param endKey raw end key, exclusive + * @return list of key-value pairs in range + */ +List scan(ByteString startKey, ByteString endKey) ``` ```java - /** - * Scan raw key-value pairs from TiKV in range [startKey, endKey) - * - * @param startKey raw start key, inclusive - * @param endKey raw end key, exclusive - * @return list of key-value pairs in range - */ - List scan(ByteString startKey, ByteString endKey) +/** + * Scan raw key-value pairs from TiKV in range [startKey, endKey) + * + * @param startKey raw start key, inclusive + * @param limit limit of key-value pairs + * @return list of key-value pairs in range + */ +List scan(ByteString startKey, int limit) ``` ```java - /** - * Scan raw key-value pairs from TiKV in range [startKey, endKey) - * - * @param startKey raw start key, inclusive - * @param limit limit of key-value pairs - * @return list of key-value pairs in range - */ - List scan(ByteString startKey, int limit) -``` - -```java - /** - * Delete a raw key-value pair from TiKV if key exists - * - * @param key raw key to be deleted - */ - void delete(ByteString key) +/** + * Delete a raw key-value pair from TiKV if key exists + * + * @param key raw key to be deleted + */ +void delete(ByteString key) ``` diff --git a/build.gradle b/build.gradle index dc05a39222..4c89d38ed1 100644 --- a/build.gradle +++ b/build.gradle @@ -15,47 +15,47 @@ tasks.withType(JavaCompile) { } idea { - module { - // Marks the already(!) added srcDir as "generated" - generatedSourceDirs += file('target/generated-sources/protobuf/java') - generatedSourceDirs += file('target/generated-sources/protobuf/grpc-java') - } + module { + // Marks the already(!) added srcDir as "generated" + generatedSourceDirs += file('target/generated-sources/main/java') + generatedSourceDirs += file('target/generated-sources/main/grpc-java') + } } buildscript { - repositories { - maven { url "http://repo.maven.apache.org/maven2" } - } - dependencies { - classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.7' - } + repositories { + maven { url "http://repo.maven.apache.org/maven2" } + } + dependencies { + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.7' + } } sourceSets { - main { - proto { - srcDir {'proto'} - include {'**/*.proto'} - } - java { - // include self written and generated code - srcDirs 'src/main/java', 'target/generated-sources/protobuf/java' , 'target/generated-sources/protobuf/grpc-java' - } - } - test { + main { + proto { + srcDir {'proto'} + include {'**/*.proto'} + } java { - srcDirs 'src/test/java', 'target/generated-sources/protobuf/java' , 'target/generated-sources/protobuf/grpc-java' + // include self written and generated code + srcDirs 'src/main/java', 'target/generated-sources/main/java' , 'target/generated-sources/main/grpc-java' } } - // remove the test configuration - at least in your example you don't have a special test proto file + test { + java { + srcDirs 'src/test/java', 'target/generated-sources/main/java' , 'target/generated-sources/main/grpc-java' + } + } + // remove the test configuration - at least in your example you don't have a special test proto file } protobuf { - // Configure the protoc executable - protoc { - // Download from repositories - artifact = 'com.google.protobuf:protoc:3.1.0' - } + // Configure the protoc executable + protoc { + // Download from repositories + artifact = 'com.google.protobuf:protoc:3.1.0' + } // Locate the codegen plugins plugins { @@ -67,82 +67,83 @@ protobuf { // or // path = 'tools/protoc-gen-grpc-java' } - } + } - generateProtoTasks.generatedFilesBaseDir = 'target/generated-sources' + generateProtoTasks.generatedFilesBaseDir = 'target/generated-sources' - generateProtoTasks { - // all() returns the collection of all protoc tasks - all().each { task -> + generateProtoTasks { + // all() returns the collection of all protoc tasks + all().each { task -> task.plugins { grpc { outputSubDir = 'grpc-java' } } - // Here you can configure the task - } + // Here you can configure the task + } - // In addition to all(), you may get the task collection by various - // criteria: + // In addition to all(), you may get the task collection by various + // criteria: - // (Java only) returns tasks for a sourceSet - ofSourceSet('main') - } + // (Java only) returns tasks for a sourceSet + ofSourceSet('main') + } } clean { delete protobuf.generatedFilesBaseDir + delete project.buildDir } repositories { - maven { url "http://repo.maven.apache.org/maven2" } + maven { url "http://repo.maven.apache.org/maven2" } } dependencies { - compile group: 'log4j', name: 'log4j', version:'1.2.17' - compile group: 'net.sf.trove4j', name: 'trove4j', version:'3.0.1' - compile group: 'com.sangupta', name: 'murmur', version:'1.0.0' - compile group: 'io.grpc', name: 'grpc-netty', version:'1.7.0' - compile group: 'io.grpc', name: 'grpc-protobuf', version:'1.7.0' - compile group: 'io.grpc', name: 'grpc-stub', version:'1.7.0' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version:'2.8.11' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version:'2.8.11.1' - compile(group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.11', version:'2.8.11') { + compile group: 'log4j', name: 'log4j', version:'1.2.17' + compile group: 'net.sf.trove4j', name: 'trove4j', version:'3.0.1' + compile group: 'com.sangupta', name: 'murmur', version:'1.0.0' + compile group: 'io.grpc', name: 'grpc-netty', version:'1.7.0' + compile group: 'io.grpc', name: 'grpc-protobuf', version:'1.7.0' + compile group: 'io.grpc', name: 'grpc-stub', version:'1.7.0' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version:'2.8.11' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version:'2.8.11.1' + compile(group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.11', version:'2.8.11') { exclude(module: 'guava') - } - compile group: 'joda-time', name: 'joda-time', version:'2.9.9' - compile group: 'org.joda', name: 'joda-convert', version:'1.9.2' - testCompile group: 'io.grpc', name: 'grpc-testing', version:'1.7.0' - //remove unused hadoop dependencies - /*compile group: 'org.apache.logging.log4j', name: 'log4j-api', version:'2.8.1' - compile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.8.1' - compile group: 'org.apache.spark', name: 'spark-core_2.11', version:'2.3.2' - compile group: 'org.apache.spark', name: 'spark-catalyst_2.11', version:'2.3.2' - compile group: 'org.apache.spark', name: 'spark-sql_2.11', version:'2.3.2' - compile group: 'org.apache.spark', name: 'spark-hive_2.11', version:'2.3.2' - compile group: 'org.apache.spark', name: 'spark-hive-thriftserver_2.11', version:'2.3.2' - compile group: 'org.apache.spark', name: 'spark-unsafe_2.11', version:'2.3.2'*/ - compile group: 'org.apache.commons', name: 'commons-lang3', version:'3.5' - compile group: 'org.slf4j', name: 'slf4j-api', version:'1.7.16' - compile group: 'org.slf4j', name: 'slf4j-log4j12', version:'1.7.16' - compile group: 'org.slf4j', name: 'jul-to-slf4j', version:'1.7.16' - compile group: 'org.slf4j', name: 'jcl-over-slf4j', version:'1.7.16' + } + compile group: 'joda-time', name: 'joda-time', version:'2.9.9' + compile group: 'org.joda', name: 'joda-convert', version:'1.9.2' + testCompile group: 'io.grpc', name: 'grpc-testing', version:'1.7.0' + //remove unused hadoop dependencies + /*compile group: 'org.apache.logging.log4j', name: 'log4j-api', version:'2.8.1' + compile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.8.1' + compile group: 'org.apache.spark', name: 'spark-core_2.11', version:'2.3.2' + compile group: 'org.apache.spark', name: 'spark-catalyst_2.11', version:'2.3.2' + compile group: 'org.apache.spark', name: 'spark-sql_2.11', version:'2.3.2' + compile group: 'org.apache.spark', name: 'spark-hive_2.11', version:'2.3.2' + compile group: 'org.apache.spark', name: 'spark-hive-thriftserver_2.11', version:'2.3.2' + compile group: 'org.apache.spark', name: 'spark-unsafe_2.11', version:'2.3.2'*/ + compile group: 'org.apache.commons', name: 'commons-lang3', version:'3.5' + compile group: 'org.slf4j', name: 'slf4j-api', version:'1.7.16' + compile group: 'org.slf4j', name: 'slf4j-log4j12', version:'1.7.16' + compile group: 'org.slf4j', name: 'jul-to-slf4j', version:'1.7.16' + compile group: 'org.slf4j', name: 'jcl-over-slf4j', version:'1.7.16' compile 'com.google.protobuf:protobuf-java:3.1.0' } task updateProtoModule(type: Exec) { - if (System.getProperty('os.name').toLowerCase(Locale.ROOT).contains('windows')) { - commandLine 'cmd', './scripts/proto.sh' - } else { - commandLine 'sh', '-c', './scripts/proto.sh' - } + if (System.getProperty('os.name').toLowerCase(Locale.ROOT).contains('windows')) { + commandLine 'cmd', './scripts/proto.sh' + } else { + commandLine 'sh', '-c', './scripts/proto.sh' + } } task copyResources(type: Copy) { includeEmptyDirs = false copy { from "./kvproto/include/" - include "**/gogoproto/**" + include "**/gogoproto/**" into "proto" } copy { @@ -151,5 +152,16 @@ task copyResources(type: Copy) { } } +//create a single Jar with all dependencies +task fatJar(type: Jar) { + manifest { + attributes 'Title': project.name, + 'Version': version + } + baseName = project.name + '-all' + from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } + with jar +} + compileJava.dependsOn copyResources copyResources.dependsOn updateProtoModule diff --git a/src/main/java/org/tikv/common/TiConfiguration.java b/src/main/java/org/tikv/common/TiConfiguration.java index ed20c6cc8b..735457a252 100644 --- a/src/main/java/org/tikv/common/TiConfiguration.java +++ b/src/main/java/org/tikv/common/TiConfiguration.java @@ -16,7 +16,6 @@ package org.tikv.common; import com.google.common.collect.ImmutableList; - import java.io.Serializable; import java.net.URI; import java.util.ArrayList; diff --git a/src/main/java/org/tikv/common/util/ChannelFactory.java b/src/main/java/org/tikv/common/util/ChannelFactory.java index f7bbc7fb60..60447e2a41 100644 --- a/src/main/java/org/tikv/common/util/ChannelFactory.java +++ b/src/main/java/org/tikv/common/util/ChannelFactory.java @@ -17,7 +17,6 @@ package org.tikv.common.util; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; - import java.net.URI; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/src/main/java/org/tikv/common/util/ConcreteBackOffer.java b/src/main/java/org/tikv/common/util/ConcreteBackOffer.java index b60a654aaa..0f6ebffdf9 100644 --- a/src/main/java/org/tikv/common/util/ConcreteBackOffer.java +++ b/src/main/java/org/tikv/common/util/ConcreteBackOffer.java @@ -86,7 +86,7 @@ public class ConcreteBackOffer implements BackOffer { BackOffFunction backOffFunction = null; switch (funcType) { case BoUpdateLeader: - //fix: reference from go client + // fix: reference from go client backOffFunction = BackOffFunction.create(1, 10, BackOffStrategy.NoJitter); break; case BoTxnLockFast: @@ -96,7 +96,7 @@ public class ConcreteBackOffer implements BackOffer { backOffFunction = BackOffFunction.create(2000, 10000, BackOffStrategy.EqualJitter); break; case BoRegionMiss: - //fix: reference from go client + // fix: reference from go client // change base time to 2ms, because it may recover soon. backOffFunction = BackOffFunction.create(2, 500, BackOffStrategy.NoJitter); break; @@ -108,7 +108,7 @@ public class ConcreteBackOffer implements BackOffer { backOffFunction = BackOffFunction.create(500, 3000, BackOffStrategy.EqualJitter); break; case BoTiKVRPC: - //fix: reference from go client + // fix: reference from go client backOffFunction = BackOffFunction.create(100, 2000, BackOffStrategy.EqualJitter); break; case BoStoreNotMatch: diff --git a/src/test/java/org/tikv/common/KVMockServer.java b/src/test/java/org/tikv/common/KVMockServer.java index 0d90015a8b..743620eb3e 100644 --- a/src/test/java/org/tikv/common/KVMockServer.java +++ b/src/test/java/org/tikv/common/KVMockServer.java @@ -33,10 +33,10 @@ import org.tikv.common.key.Key; import org.tikv.common.region.TiRegion; import org.tikv.kvproto.Coprocessor; import org.tikv.kvproto.Errorpb; +import org.tikv.kvproto.Errorpb.EpochNotMatch; import org.tikv.kvproto.Errorpb.Error; import org.tikv.kvproto.Errorpb.NotLeader; import org.tikv.kvproto.Errorpb.ServerIsBusy; -import org.tikv.kvproto.Errorpb.EpochNotMatch; import org.tikv.kvproto.Kvrpcpb; import org.tikv.kvproto.Kvrpcpb.Context; import org.tikv.kvproto.TikvGrpc; diff --git a/src/test/java/org/tikv/common/PDClientTest.java b/src/test/java/org/tikv/common/PDClientTest.java index 07bd57b46c..278c88f118 100644 --- a/src/test/java/org/tikv/common/PDClientTest.java +++ b/src/test/java/org/tikv/common/PDClientTest.java @@ -89,7 +89,8 @@ public class PDClientTest { session = TiSession.create(conf); try (PDClient client = session.getPDClient()) { client.switchLeader(ImmutableList.of("http://" + LOCAL_ADDR_IPV6 + ":" + (server.port + 2))); - assertEquals(client.getLeaderWrapper().getLeaderInfo(), LOCAL_ADDR_IPV6 + ":" + (server.port + 2)); + assertEquals( + client.getLeaderWrapper().getLeaderInfo(), LOCAL_ADDR_IPV6 + ":" + (server.port + 2)); } }